6b8d9efc-be89-4667-9ad3-9014df4869b9

Committed 31 May 2026 08:32PM UTC coverage: 94.056% (+0.002%) from 94.054%

Build # 6b8d9efc-be89-4667-9ad3-9014df4869b9

Build Type

push

circleci

Committed by

snarfed

Commit Message

Revert "admin: add memory profiling"

This reverts commit e15669204.

it works, but it increases CPU and latency way too much to keep on for any length of time

for #1149, #2488

Coverage Stats

7690 of 8176 relevant lines covered (94.06%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.71

/admin.py

"""Admin pages and endpoints: admin UI, hub status, memcache API, etc."""
from datetime import datetime
import logging
from urllib.parse import quote

from google.cloud import ndb
from google.cloud.ndb import Key

import arroba.server
import common
from common import (
    secret_key_auth,
)
from flask import redirect, request
from flask_app import app
import filters
from granary import as1, microformats2
import memcache
import models
from models import Object, PROTOCOLS, User
import pytz
from webutil import flask_util, logs, util
from webutil.flask_util import flash

from activitypub import ActivityPub, FEDI_URL_RE
from atproto import ATProto
import ids
from nostr import Nostr
import pages
from web import Web

BLOCKLISTS = {
    bl.key_id: bl for bl in (
        filters.CONTENT_BLOCKLIST,
        filters.MEDIA_BLOCKLIST,
        filters.DOMAIN_BLOCKLIST,
    )
}

logger = logging.getLogger(__name__)


#
# admin UI
#
def render(template, **vars):
    return pages.render(template, **vars)


def format_properties(entity):
    """Generates template variables based on misc User and Object properties:

    * created, updated: to ISO-8601 strings
    * bridged: dict mapping Protocol subclass to string id
    """
    vars = {}

    pt = pytz.timezone('US/Pacific')
    for field in 'created', 'updated':
        # these are proto.datetime_helpers.DatetimeWithNanoseconds. have to recreate
        # them as plain datetimes because otherwise they crash on replace().
        # similar to: https://stackoverflow.com/q/54370012/186123
        vars[field] = datetime.fromtimestamp(getattr(entity, field).timestamp()
                                             ).replace(microsecond=0).astimezone(pt)

    return vars


@app.get('/admin/')
def admin_home():
    for reloader in BLOCKLISTS.values():
        reloader.reload()
    return render('admin.html', filters=filters)


@app.post('/admin/blocklist')
def save_blocklist():
    """
    Form values:
      id (str): blocklist key id
      values (str)
    """
    id = request.values['id']
    values = [v.strip() for v in request.values['values'].splitlines() if v.strip()]
    BLOCKLISTS[id].obj.raw = values
    BLOCKLISTS[id].obj.put()
    flash(f'Saved {id}.')
    return redirect('/admin/')


@app.get('/admin/user')
def admin_user_search():
    """
    Query params:
      query (str)
    """
    query = orig_query = request.values['query'].strip()
    if not query:
        error('empty query')

    # preprocess search query, misc heuristics
    if query.endswith('.ap.brid.gy'):
        query = ids.translate_user_id(id=query, from_=ATProto, to=ActivityPub)
    elif query.endswith('.brid.gy'):
        query = query.rsplit('.', 3)[0]
    elif match := FEDI_URL_RE.fullmatch(query):
        query = ids.handle_as_domain(f'@{match.group("handle")}@{match.group("domain")}')

    queries = [query]
    if '@' in query:
        if handle_as_domain := ids.handle_as_domain(query):
            queries.append(handle_as_domain)

    futures = [
        proto.query(ndb.OR(
            proto.key.IN([proto(id=query).key for query in queries]),
            proto.handle.IN(queries),
            proto.handle_as_domain.IN(queries),
            proto.handle_pay_level_domain.IN(queries))).fetch_async()
        for proto in set(PROTOCOLS.values()) if proto]

    users = []
    for future in futures:
        users.extend(future.get_result())

    if not users:
        if key := models.get_original_user_key(query):
            users = [key.get()]

    for user in users:
        user.bridged_ids = {
            proto: ids.translate_user_id(id=user.key.id(), from_=user, to=proto)
            for proto in (ATProto, ActivityPub, Nostr)
            if not isinstance(user, proto)
        }
        user.sent_dms_ = ', '.join(
            f'{dm.type} ({dm.protocol})' for dm in user.sent_dms)


    return render('admin_users.html', query=orig_query, users=users)


@app.get('/admin/user/<key>')
def admin_user(key):
    user = Key(urlsafe=key).get()
    if not user or not isinstance(user, User):
        flash('user not found')
        return redirect('/admin/')

    return redirect(f'/admin/user?query={quote(user.key.id())}')


@app.post('/admin/object')
def admin_object_lookup():
    """
    Form values:
      id (str)
    """
    id = request.values['id'].strip()
    # ordered
    for proto in ActivityPub, ATProto, Nostr, Web:
        if proto and proto.owns_id(id) is not False:
            if obj := proto.load(id):
                return redirect(f'/admin/object/{obj.key.id()}')

    flash(f"Couldn't resolve {id}")
    return redirect('/admin/')


@app.get('/admin/object/<path:id>')
def admin_object(id):
    if not (obj := Object.get_by_id(id)):
        flash('object not found')
        return redirect('/admin/')

    if obj.as1 and as1.object_type(obj.as1) in as1.CRUD_VERBS:
        if inner_id := as1.get_object(obj.as1).get('id'):
            if inner := Object.get_by_id(inner_id):
                return redirect(f'/admin/object/{inner.key.id()}')

    proto = PROTOCOLS[obj.source_protocol]
    user = None
    if obj.users:
        user = obj.users[0].get()
    elif obj.as1 and proto and (user_id := as1.get_owner(obj.as1)):
        user = proto.get_by_id(user_id)

    bridged_ids = {
        to_proto: ids.translate_object_id(id=obj.key.id(), from_=proto, to=to_proto)
        for to_proto in (ATProto, ActivityPub, Nostr)
        if to_proto != proto and user and user.is_enabled(to_proto)
    }

    return render(
        'admin_object.html',
        obj=obj,
        user=user,
        bridged_ids=bridged_ids,
        **format_properties(obj))


@app.post('/admin/receive')
def admin_receive():
    obj_key = Key(urlsafe=request.values['obj_key'])
    user_key = Key(urlsafe=request.values['user_key'])
    common.create_task(queue='receive', obj_id=obj_key.id(),
                       authed_as=user_key.id(), force='true')
    return redirect(f'/admin/object/{obj_key.id()}')


@app.post('/admin/enable')
def admin_enable():
    """
    Form values:
      key (str): urlsafe user key
      protocol (str)
    """
    key = request.values['key']
    user = Key(urlsafe=key).get()
    proto = PROTOCOLS[request.values['protocol']]
    user.enable_protocol(proto)
    flash(f'Enabled {proto.LABEL} for {user.handle}')
    return redirect(f'/admin/user/{key}')


@app.post('/admin/disable')
def admin_disable():
    """
    Form values:
      key (str): urlsafe user key
      protocol (str)
    """
    key = request.values['key']
    user = Key(urlsafe=key).get()
    proto = PROTOCOLS[request.values['protocol']]
    user.delete(proto)
    user.disable_protocol(proto)
    flash(f'Disabled {proto.LABEL} for {user.handle}')
    return redirect(f'/admin/user/{key}')


#
# internal Memcache API, since we can't connect to our Memorystore instance externally
# https://github.com/snarfed/bridgy-fed/issues/1472
#
@app.get('/admin/memcache/get')
@secret_key_auth
@flask_util.headers({'Content-Type': 'text/plain'})
def memcache_get():
    if key := request.values.get('key'):
        return repr(Key(urlsafe=key).get(use_cache=False, use_datastore=False,
                                         use_global_cache=True))
    elif raw := request.values.get('raw'):
        return repr(memcache.memcache.get(raw))
    else:
        error('either key or raw are required')


@app.post('/admin/memcache/evict')
@secret_key_auth
@flask_util.headers({'Content-Type': 'text/plain'})
def memcache_evict():
    if key := request.values.get('key'):
        memcache.evict(Key(urlsafe=key))
        return ''
    elif raw := request.values.get('raw'):
        deleted = memcache.evict_raw(raw)
        return 'deleted' if deleted else 'not found'
    else:
        error('either key or raw are required')


@app.post('/admin/sequences/alloc')
@secret_key_auth
@flask_util.headers({'Content-Type': 'text/plain'})
def alloc_seq():
    nsid = flask_util.get_required_param('nsid')
    result = arroba.server.storage.sequences.allocate(nsid)
    return str(result)


@app.get('/admin/sequences/last')
@secret_key_auth
@flask_util.headers({'Content-Type': 'text/plain'})
def last_seq():
    nsid = flask_util.get_required_param('nsid')
    result = arroba.server.storage.sequences.last(nsid)
    return str(result)

1	"""Admin pages and endpoints: admin UI, hub status, memcache API, etc."""
2	from datetime import datetime	1✔
3	import logging	1✔
4	from urllib.parse import quote	1✔
5
6	from google.cloud import ndb	1✔
7	from google.cloud.ndb import Key	1✔
8
9	import arroba.server	1✔
10	import common	1✔
11	from common import (	1✔
12	secret_key_auth,
13	)
14	from flask import redirect, request	1✔
15	from flask_app import app	1✔
16	import filters	1✔
17	from granary import as1, microformats2	1✔
18	import memcache	1✔
19	import models	1✔
20	from models import Object, PROTOCOLS, User	1✔
21	import pytz	1✔
22	from webutil import flask_util, logs, util	1✔
23	from webutil.flask_util import flash	1✔
24
25	from activitypub import ActivityPub, FEDI_URL_RE	1✔
26	from atproto import ATProto	1✔
27	import ids	1✔
28	from nostr import Nostr	1✔
29	import pages	1✔
30	from web import Web	1✔
31
32	BLOCKLISTS = {	1✔
33	bl.key_id: bl for bl in (
34	filters.CONTENT_BLOCKLIST,
35	filters.MEDIA_BLOCKLIST,
36	filters.DOMAIN_BLOCKLIST,
37	)
38	}
39
40	logger = logging.getLogger(__name__)	1✔
41
42
43	#
44	# admin UI
45	#
46	def render(template, **vars):	1✔
47	return pages.render(template, **vars)	1✔
48
49
50	def format_properties(entity):	1✔
51	"""Generates template variables based on misc User and Object properties:
52
53	* created, updated: to ISO-8601 strings
54	* bridged: dict mapping Protocol subclass to string id
55	"""
56	vars = {}	1✔
57
58	pt = pytz.timezone('US/Pacific')	1✔
59	for field in 'created', 'updated':	1✔
60	# these are proto.datetime_helpers.DatetimeWithNanoseconds. have to recreate
61	# them as plain datetimes because otherwise they crash on replace().
62	# similar to: https://stackoverflow.com/q/54370012/186123
63	vars[field] = datetime.fromtimestamp(getattr(entity, field).timestamp()	1✔
64	).replace(microsecond=0).astimezone(pt)
65
66	return vars	1✔
67
68
69	@app.get('/admin/')	1✔
70	def admin_home():	1✔
71	for reloader in BLOCKLISTS.values():	1✔
72	reloader.reload()	1✔
73	return render('admin.html', filters=filters)	1✔
74
75
76	@app.post('/admin/blocklist')	1✔
77	def save_blocklist():	1✔
78	"""
79	Form values:
80	id (str): blocklist key id
81	values (str)
82	"""
83	id = request.values['id']	1✔
84	values = [v.strip() for v in request.values['values'].splitlines() if v.strip()]	1✔
85	BLOCKLISTS[id].obj.raw = values	1✔
86	BLOCKLISTS[id].obj.put()	1✔
87	flash(f'Saved {id}.')	1✔
88	return redirect('/admin/')	1✔
89
90
91	@app.get('/admin/user')	1✔
92	def admin_user_search():	1✔
93	"""
94	Query params:
95	query (str)
96	"""
97	query = orig_query = request.values['query'].strip()	1✔
98	if not query:	1✔
99	error('empty query')	×
100
101	# preprocess search query, misc heuristics
102	if query.endswith('.ap.brid.gy'):	1✔
103	query = ids.translate_user_id(id=query, from_=ATProto, to=ActivityPub)	1✔
104	elif query.endswith('.brid.gy'):	1✔
105	query = query.rsplit('.', 3)[0]	1✔
106	elif match := FEDI_URL_RE.fullmatch(query):	1✔
107	query = ids.handle_as_domain(f'@{match.group("handle")}@{match.group("domain")}')	1✔
108
109	queries = [query]	1✔
110	if '@' in query:	1✔
111	if handle_as_domain := ids.handle_as_domain(query):	1✔
112	queries.append(handle_as_domain)	1✔
113
114	futures = [	1✔
115	proto.query(ndb.OR(
116	proto.key.IN([proto(id=query).key for query in queries]),
117	proto.handle.IN(queries),
118	proto.handle_as_domain.IN(queries),
119	proto.handle_pay_level_domain.IN(queries))).fetch_async()
120	for proto in set(PROTOCOLS.values()) if proto]
121
122	users = []	1✔
123	for future in futures:	1✔
124	users.extend(future.get_result())	1✔
125
126	if not users:	1✔
127	if key := models.get_original_user_key(query):	1✔
128	users = [key.get()]	1✔
129
130	for user in users:	1✔
131	user.bridged_ids = {	1✔
132	proto: ids.translate_user_id(id=user.key.id(), from_=user, to=proto)
133	for proto in (ATProto, ActivityPub, Nostr)
134	if not isinstance(user, proto)
135	}
136	user.sent_dms_ = ', '.join(	1✔
137	f'{dm.type} ({dm.protocol})' for dm in user.sent_dms)
138
139
140	return render('admin_users.html', query=orig_query, users=users)	1✔
141
142
143	@app.get('/admin/user/<key>')	1✔
144	def admin_user(key):	1✔
145	user = Key(urlsafe=key).get()	1✔
146	if not user or not isinstance(user, User):	1✔
147	flash('user not found')	1✔
148	return redirect('/admin/')	1✔
149
150	return redirect(f'/admin/user?query={quote(user.key.id())}')	1✔
151
152
153	@app.post('/admin/object')	1✔
154	def admin_object_lookup():	1✔
155	"""
156	Form values:
157	id (str)
158	"""
159	id = request.values['id'].strip()	1✔
160	# ordered
161	for proto in ActivityPub, ATProto, Nostr, Web:	1✔
162	if proto and proto.owns_id(id) is not False:	1✔
163	if obj := proto.load(id):	1✔
164	return redirect(f'/admin/object/{obj.key.id()}')	1✔
165
166	flash(f"Couldn't resolve {id}")	×
167	return redirect('/admin/')	×
168
169
170	@app.get('/admin/object/<path:id>')	1✔
171	def admin_object(id):	1✔
172	if not (obj := Object.get_by_id(id)):	1✔
173	flash('object not found')	1✔
174	return redirect('/admin/')	1✔
175
176	if obj.as1 and as1.object_type(obj.as1) in as1.CRUD_VERBS:	1✔
177	if inner_id := as1.get_object(obj.as1).get('id'):	1✔
178	if inner := Object.get_by_id(inner_id):	1✔
179	return redirect(f'/admin/object/{inner.key.id()}')	1✔
180
181	proto = PROTOCOLS[obj.source_protocol]	1✔
182	user = None	1✔
183	if obj.users:	1✔
184	user = obj.users[0].get()	×
185	elif obj.as1 and proto and (user_id := as1.get_owner(obj.as1)):	1✔
186	user = proto.get_by_id(user_id)	×
187
188	bridged_ids = {	1✔
189	to_proto: ids.translate_object_id(id=obj.key.id(), from_=proto, to=to_proto)
190	for to_proto in (ATProto, ActivityPub, Nostr)
191	if to_proto != proto and user and user.is_enabled(to_proto)
192	}
193
194	return render(	1✔
195	'admin_object.html',
196	obj=obj,
197	user=user,
198	bridged_ids=bridged_ids,
199	**format_properties(obj))
200
201
202	@app.post('/admin/receive')	1✔
203	def admin_receive():	1✔
204	obj_key = Key(urlsafe=request.values['obj_key'])	1✔
205	user_key = Key(urlsafe=request.values['user_key'])	1✔
206	common.create_task(queue='receive', obj_id=obj_key.id(),	1✔
207	authed_as=user_key.id(), force='true')
208	return redirect(f'/admin/object/{obj_key.id()}')	1✔
209
210
211	@app.post('/admin/enable')	1✔
212	def admin_enable():	1✔
213	"""
214	Form values:
215	key (str): urlsafe user key
216	protocol (str)
217	"""
218	key = request.values['key']	1✔
219	user = Key(urlsafe=key).get()	1✔
220	proto = PROTOCOLS[request.values['protocol']]	1✔
221	user.enable_protocol(proto)	1✔
222	flash(f'Enabled {proto.LABEL} for {user.handle}')	1✔
223	return redirect(f'/admin/user/{key}')	1✔
224
225
226	@app.post('/admin/disable')	1✔
227	def admin_disable():	1✔
228	"""
229	Form values:
230	key (str): urlsafe user key
231	protocol (str)
232	"""
233	key = request.values['key']	1✔
234	user = Key(urlsafe=key).get()	1✔
235	proto = PROTOCOLS[request.values['protocol']]	1✔
236	user.delete(proto)	1✔
237	user.disable_protocol(proto)	1✔
238	flash(f'Disabled {proto.LABEL} for {user.handle}')	1✔
239	return redirect(f'/admin/user/{key}')	1✔
240
241
242	#
243	# internal Memcache API, since we can't connect to our Memorystore instance externally
244	# https://github.com/snarfed/bridgy-fed/issues/1472
245	#
246	@app.get('/admin/memcache/get')	1✔
247	@secret_key_auth	1✔
248	@flask_util.headers({'Content-Type': 'text/plain'})	1✔
249	def memcache_get():	1✔
250	if key := request.values.get('key'):	1✔
251	return repr(Key(urlsafe=key).get(use_cache=False, use_datastore=False,	1✔
252	use_global_cache=True))
253	elif raw := request.values.get('raw'):	1✔
254	return repr(memcache.memcache.get(raw))	1✔
255	else:
256	error('either key or raw are required')	×
257
258
259	@app.post('/admin/memcache/evict')	1✔
260	@secret_key_auth	1✔
261	@flask_util.headers({'Content-Type': 'text/plain'})	1✔
262	def memcache_evict():	1✔
263	if key := request.values.get('key'):	1✔
264	memcache.evict(Key(urlsafe=key))	1✔
265	return ''	1✔
266	elif raw := request.values.get('raw'):	1✔
267	deleted = memcache.evict_raw(raw)	1✔
268	return 'deleted' if deleted else 'not found'	1✔
269	else:
270	error('either key or raw are required')	×
271
272
273	@app.post('/admin/sequences/alloc')	1✔
274	@secret_key_auth	1✔
275	@flask_util.headers({'Content-Type': 'text/plain'})	1✔
276	def alloc_seq():	1✔
277	nsid = flask_util.get_required_param('nsid')	1✔
278	result = arroba.server.storage.sequences.allocate(nsid)	1✔
279	return str(result)	1✔
280
281
282	@app.get('/admin/sequences/last')	1✔
283	@secret_key_auth	1✔
284	@flask_util.headers({'Content-Type': 'text/plain'})	1✔
285	def last_seq():	1✔
286	nsid = flask_util.get_required_param('nsid')	1✔
287	result = arroba.server.storage.sequences.last(nsid)	1✔
288	return str(result)	1✔

snarfed / bridgy-fed / 6b8d9efc-be89-4667-9ad3-9014df4869b9

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous