a62808aa-5996-4422-8905-839df0c498c8

Committed 29 Jan 2025 07:57PM UTC coverage: 93.218%. Remained the same

Build # a62808aa-5996-4422-8905-839df0c498c8

Build Type

push

circleci

Committed by

snarfed

Commit Message

cache /convert/ and /r/ endpoints in memcache

...since GAE's edge caching based on Cache-Control doesn't seem to be very effective :/

for #1149

Run Details

7 of 7 new or added lines in 2 files covered. (100.0%)

58 existing lines in 5 files now uncovered.

4536 of 4866 relevant lines covered (93.22%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.0

/convert.py

"""Serves ``/convert/...`` URLs to convert data from one protocol to another.

URL pattern is ``/convert/SOURCE/DEST``, where ``SOURCE`` and ``DEST`` are the
``LABEL`` constants from the :class:`protocol.Protocol` subclasses.
"""
from datetime import timedelta
import logging
import re
from urllib.parse import quote, unquote

from flask import redirect, request
from granary import as1
from oauth_dropins.webutil import flask_util, util
from oauth_dropins.webutil.flask_util import error

from activitypub import ActivityPub
from common import (
    CACHE_CONTROL,
    LOCAL_DOMAINS,
    subdomain_wrap,
    SUPERDOMAIN,
)
from flask_app import app
import memcache
from models import Object, PROTOCOLS
from protocol import Protocol
from web import Web

logger = logging.getLogger(__name__)


@app.get(f'/convert/<to>/<path:_>')
@memcache.memoize(expire=timedelta(hours=1))
@flask_util.headers(CACHE_CONTROL)
def convert(to, _, from_=None):
    """Converts data from one protocol to another and serves it.

    Fetches the source data if it's not already stored.

    Note that we don't do conneg or otherwise care about the Accept header here,
    we always serve the "to" protocol's format.

    Args:
      to (str): protocol
      from_ (str): protocol, only used when called by
        :func:`convert_source_path_redirect`
    """
    if from_:
        from_proto = PROTOCOLS.get(from_)
        if not from_proto:
            error(f'No protocol found for {from_}', status=404)
        logger.info(f'Overriding any domain protocol with {from_}')
    else:
        from_proto = Protocol.for_request(fed=Protocol)
    if not from_proto:
        error(f'Unknown protocol {request.host.removesuffix(SUPERDOMAIN)}', status=404)

    to_proto = PROTOCOLS.get(to)
    if not to_proto:
        error('Unknown protocol {to}', status=404)

    # don't use urllib.parse.urlencode(request.args) because that doesn't
    # guarantee us the same query param string as in the original URL, and we
    # want exactly the same thing since we're looking up the URL's Object by id
    path_prefix = f'convert/{to}/'
    id = unquote(request.url.removeprefix(request.root_url).removeprefix(path_prefix))

    # our redirects evidently collapse :// down to :/ , maybe to prevent URL
    # parsing bugs? if that happened to this URL, expand it back to ://
    id = re.sub(r'^(https?:/)([^/])', r'\1/\2', id)

    logger.debug(f'Converting from {from_proto.LABEL} to {to}: {id}')

    # load, and maybe fetch. if it's a post/update, redirect to inner object.
    obj = from_proto.load(id)
    if not obj:
        error(f"Couldn't load {id}", status=404)
    elif not obj.as1:
        error(f'Stored object for {id} has no data', status=404)

    type = as1.object_type(obj.as1)
    if type in as1.CRUD_VERBS or type == 'share':
        if obj_id := as1.get_object(obj.as1).get('id'):
            if obj_obj := from_proto.load(obj_id, remote=False):
                if type == 'share':
                    # TODO: should this be Source.base_object? That's broad
                    # though, includes inReplyTo
                    check_bridged_to(obj_obj, to_proto=to_proto)
                elif (type in as1.CRUD_VERBS
                      and obj_obj.as1
                      and obj_obj.as1.keys() - set(['id', 'url', 'objectType'])):
                    logger.info(f'{type} activity, redirecting to Object {obj_id}')
                    return redirect(f'/{path_prefix}{obj_id}', code=301)

    check_bridged_to(obj, to_proto=to_proto)

    # convert and serve
    return to_proto.convert(obj), {
        'Content-Type': to_proto.CONTENT_TYPE,
    }


def check_bridged_to(obj, to_proto):
    """If ``object`` or its owner isn't bridged to ``to_proto``, raises :class:`werkzeug.exceptions.HTTPException`.

    Args:
      obj (models.Object)
      to_proto (subclass of protocol.Protocol)
    """
    # don't serve deletes or deleted objects
    if obj.deleted or obj.type == 'delete':
        error('Deleted', status=410)

    # don't serve for a given protocol if we haven't bridged it there
    if to_proto.HAS_COPIES and not obj.get_copy(to_proto):
        error(f"{obj.key.id()} hasn't been bridged to {to_proto.LABEL}", status=404)

    # check that owner has this protocol enabled
    if owner := as1.get_owner(obj.as1):
        if from_proto := Protocol.for_id(owner):
            user = from_proto.get_by_id(owner)
            if not user:
                error(f"{from_proto.LABEL} user {owner} not found", status=404)
            elif not user.is_enabled(to_proto):
                error(f"{from_proto.LABEL} user {owner} isn't bridged to {to_proto.LABEL}", status=404)


@app.get(f'/convert/<from_>/<to>/<path:_>')
def convert_source_path_redirect(from_, to, _):
    """Old route that included source protocol in path instead of subdomain.

    DEPRECATED! Only kept to support old webmention source URLs.
    """
    if Protocol.for_request() not in (None, 'web'):  # no per-protocol subdomains
        error(f'Try again on fed.brid.gy', status=404)

    # in prod, eg gunicorn, the path somehow gets URL-decoded before we see
    # it, so we need to re-encode.
    new_path = quote(request.full_path.rstrip('?').replace(f'/{from_}/', '/'),
                     safe=':/%')

    if request.host in LOCAL_DOMAINS:
        request.url = request.url.replace(f'/{from_}/', '/')
        return convert(to, None, from_=from_)

    proto = PROTOCOLS.get(from_)
    if not proto:
        error(f'No protocol found for {from_}', status=404)

    return redirect(subdomain_wrap(proto, new_path), code=301)

1	"""Serves ``/convert/...`` URLs to convert data from one protocol to another.
2
3	URL pattern is ``/convert/SOURCE/DEST``, where ``SOURCE`` and ``DEST`` are the
4	``LABEL`` constants from the :class:`protocol.Protocol` subclasses.
5	"""
6	from datetime import timedelta	1✔
7	import logging	1✔
8	import re	1✔
9	from urllib.parse import quote, unquote	1✔
10
11	from flask import redirect, request	1✔
12	from granary import as1	1✔
13	from oauth_dropins.webutil import flask_util, util	1✔
14	from oauth_dropins.webutil.flask_util import error	1✔
15
16	from activitypub import ActivityPub	1✔
17	from common import (	1✔
18	CACHE_CONTROL,
19	LOCAL_DOMAINS,
20	subdomain_wrap,
21	SUPERDOMAIN,
22	)
23	from flask_app import app	1✔
24	import memcache	1✔
25	from models import Object, PROTOCOLS	1✔
26	from protocol import Protocol	1✔
27	from web import Web	1✔
28
29	logger = logging.getLogger(__name__)	1✔
30
31
32	@app.get(f'/convert/<to>/<path:_>')	1✔
33	@memcache.memoize(expire=timedelta(hours=1))	1✔
34	@flask_util.headers(CACHE_CONTROL)	1✔
35	def convert(to, _, from_=None):	1✔
36	"""Converts data from one protocol to another and serves it.
37
38	Fetches the source data if it's not already stored.
39
40	Note that we don't do conneg or otherwise care about the Accept header here,
41	we always serve the "to" protocol's format.
42
43	Args:
44	to (str): protocol
45	from_ (str): protocol, only used when called by
46	:func:`convert_source_path_redirect`
47	"""
48	if from_:	1✔
49	from_proto = PROTOCOLS.get(from_)	1✔
50	if not from_proto:	1✔
51	error(f'No protocol found for {from_}', status=404)	1✔
UNCOV 52	logger.info(f'Overriding any domain protocol with {from_}')	×
53	else:
54	from_proto = Protocol.for_request(fed=Protocol)	1✔
55	if not from_proto:	1✔
56	error(f'Unknown protocol {request.host.removesuffix(SUPERDOMAIN)}', status=404)	1✔
57
58	to_proto = PROTOCOLS.get(to)	1✔
59	if not to_proto:	1✔
60	error('Unknown protocol {to}', status=404)	1✔
61
62	# don't use urllib.parse.urlencode(request.args) because that doesn't
63	# guarantee us the same query param string as in the original URL, and we
64	# want exactly the same thing since we're looking up the URL's Object by id
65	path_prefix = f'convert/{to}/'	1✔
66	id = unquote(request.url.removeprefix(request.root_url).removeprefix(path_prefix))	1✔
67
68	# our redirects evidently collapse :// down to :/ , maybe to prevent URL
69	# parsing bugs? if that happened to this URL, expand it back to ://
70	id = re.sub(r'^(https?:/)([^/])', r'\1/\2', id)	1✔
71
72	logger.debug(f'Converting from {from_proto.LABEL} to {to}: {id}')	1✔
73
74	# load, and maybe fetch. if it's a post/update, redirect to inner object.
75	obj = from_proto.load(id)	1✔
76	if not obj:	1✔
UNCOV 77	error(f"Couldn't load {id}", status=404)	×
78	elif not obj.as1:	1✔
79	error(f'Stored object for {id} has no data', status=404)	1✔
80
81	type = as1.object_type(obj.as1)	1✔
82	if type in as1.CRUD_VERBS or type == 'share':	1✔
83	if obj_id := as1.get_object(obj.as1).get('id'):	1✔
84	if obj_obj := from_proto.load(obj_id, remote=False):	1✔
85	if type == 'share':	1✔
86	# TODO: should this be Source.base_object? That's broad
87	# though, includes inReplyTo
88	check_bridged_to(obj_obj, to_proto=to_proto)	1✔
89	elif (type in as1.CRUD_VERBS	1✔
90	and obj_obj.as1
91	and obj_obj.as1.keys() - set(['id', 'url', 'objectType'])):
92	logger.info(f'{type} activity, redirecting to Object {obj_id}')	1✔
93	return redirect(f'/{path_prefix}{obj_id}', code=301)	1✔
94
95	check_bridged_to(obj, to_proto=to_proto)	1✔
96
97	# convert and serve
98	return to_proto.convert(obj), {	1✔
99	'Content-Type': to_proto.CONTENT_TYPE,
100	}
101
102
103	def check_bridged_to(obj, to_proto):	1✔
104	"""If ``object`` or its owner isn't bridged to ``to_proto``, raises :class:`werkzeug.exceptions.HTTPException`.
105
106	Args:
107	obj (models.Object)
108	to_proto (subclass of protocol.Protocol)
109	"""
110	# don't serve deletes or deleted objects
111	if obj.deleted or obj.type == 'delete':	1✔
112	error('Deleted', status=410)	1✔
113
114	# don't serve for a given protocol if we haven't bridged it there
115	if to_proto.HAS_COPIES and not obj.get_copy(to_proto):	1✔
116	error(f"{obj.key.id()} hasn't been bridged to {to_proto.LABEL}", status=404)	1✔
117
118	# check that owner has this protocol enabled
119	if owner := as1.get_owner(obj.as1):	1✔
120	if from_proto := Protocol.for_id(owner):	1✔
121	user = from_proto.get_by_id(owner)	1✔
122	if not user:	1✔
123	error(f"{from_proto.LABEL} user {owner} not found", status=404)	1✔
124	elif not user.is_enabled(to_proto):	1✔
125	error(f"{from_proto.LABEL} user {owner} isn't bridged to {to_proto.LABEL}", status=404)	1✔
126
127
128	@app.get(f'/convert/<from_>/<to>/<path:_>')	1✔
129	def convert_source_path_redirect(from_, to, _):	1✔
130	"""Old route that included source protocol in path instead of subdomain.
131
132	DEPRECATED! Only kept to support old webmention source URLs.
133	"""
134	if Protocol.for_request() not in (None, 'web'): # no per-protocol subdomains	1✔
UNCOV 135	error(f'Try again on fed.brid.gy', status=404)	×
136
137	# in prod, eg gunicorn, the path somehow gets URL-decoded before we see
138	# it, so we need to re-encode.
139	new_path = quote(request.full_path.rstrip('?').replace(f'/{from_}/', '/'),	1✔
140	safe=':/%')
141
142	if request.host in LOCAL_DOMAINS:	1✔
143	request.url = request.url.replace(f'/{from_}/', '/')	1✔
144	return convert(to, None, from_=from_)	1✔
145
146	proto = PROTOCOLS.get(from_)	1✔
147	if not proto:	1✔
148	error(f'No protocol found for {from_}', status=404)	1✔
149
150	return redirect(subdomain_wrap(proto, new_path), code=301)	1✔

snarfed / bridgy-fed / a62808aa-5996-4422-8905-839df0c498c8

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous