• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

snarfed / bridgy-fed / c7343da8-115a-4a86-a425-518c166fdc80

14 Sep 2025 03:03AM UTC coverage: 92.591% (-0.05%) from 92.639%
c7343da8-115a-4a86-a425-518c166fdc80

push

circleci

snarfed
add ATProto.migrate_out, right now a noop

2 of 6 new or added lines in 1 file covered. (33.33%)

35 existing lines in 3 files now uncovered.

5749 of 6209 relevant lines covered (92.59%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.97
/protocol.py
1
"""Base protocol class and common code."""
2
import copy
1✔
3
from datetime import datetime, timedelta, timezone
1✔
4
import logging
1✔
5
import os
1✔
6
import re
1✔
7
from threading import Lock
1✔
8
from urllib.parse import urljoin, urlparse
1✔
9

10
from cachetools import cached, LRUCache
1✔
11
from flask import request
1✔
12
from google.cloud import ndb
1✔
13
from google.cloud.ndb import OR
1✔
14
from google.cloud.ndb.model import _entity_to_protobuf
1✔
15
from granary import as1, as2, source
1✔
16
from granary.source import html_to_text
1✔
17
from oauth_dropins.webutil.appengine_info import DEBUG
1✔
18
from oauth_dropins.webutil.flask_util import cloud_tasks_only
1✔
19
from oauth_dropins.webutil import models
1✔
20
from oauth_dropins.webutil import util
1✔
21
from oauth_dropins.webutil.util import json_dumps, json_loads
1✔
22
from requests import RequestException
1✔
23
import werkzeug.exceptions
1✔
24
from werkzeug.exceptions import BadGateway, HTTPException
1✔
25

26
import common
1✔
27
from common import (
1✔
28
    DOMAIN_BLOCKLIST,
29
    DOMAIN_RE,
30
    DOMAINS,
31
    ErrorButDoNotRetryTask,
32
    PRIMARY_DOMAIN,
33
    PROTOCOL_DOMAINS,
34
    report_error,
35
    subdomain_wrap,
36
)
37
import dms
1✔
38
import ids
1✔
39
from ids import (
1✔
40
    BOT_ACTOR_AP_IDS,
41
    normalize_user_id,
42
    translate_object_id,
43
    translate_user_id,
44
)
45
import memcache
1✔
46
from models import (
1✔
47
    DM,
48
    Follower,
49
    Object,
50
    PROTOCOLS,
51
    PROTOCOLS_BY_KIND,
52
    Target,
53
    User,
54
)
55
import notifications
1✔
56

57
OBJECT_REFRESH_AGE = timedelta(days=30)
1✔
58
DELETE_TASK_DELAY = timedelta(minutes=1)
1✔
59
CREATE_MAX_AGE = timedelta(weeks=2)
1✔
60
# WARNING: keep this below the receive queue's min_backoff_seconds in queue.yaml!
61
MEMCACHE_LEASE_EXPIRATION = timedelta(seconds=25)
1✔
62

63
# require a follow for users on these domains before we deliver anything from
64
# them other than their profile
65
LIMITED_DOMAINS = (os.getenv('LIMITED_DOMAINS', '').split()
1✔
66
                   or util.load_file_lines('limited_domains'))
67

68
DONT_STORE_AS1_TYPES = as1.CRUD_VERBS | set((
1✔
69
    'accept',
70
    'reject',
71
    'stop-following',
72
    'undo',
73
))
74
STORE_AS1_TYPES = (as1.ACTOR_TYPES | as1.POST_TYPES | as1.VERBS_WITH_OBJECT
1✔
75
                   - DONT_STORE_AS1_TYPES)
76

77
logger = logging.getLogger(__name__)
1✔
78

79

80
def error(*args, status=299, **kwargs):
1✔
81
    """Default HTTP status code to 299 to prevent retrying task."""
82
    return common.error(*args, status=status, **kwargs)
1✔
83

84

85
def activity_id_memcache_key(id):
1✔
86
    return memcache.key(f'receive-{id}')
1✔
87

88

89
class Protocol:
1✔
90
    """Base protocol class. Not to be instantiated; classmethods only."""
91
    ABBREV = None
1✔
92
    """str: lower case abbreviation, used in URL paths"""
1✔
93
    PHRASE = None
1✔
94
    """str: human-readable name or phrase. Used in phrases like ``Follow this person on {PHRASE}``"""
1✔
95
    OTHER_LABELS = ()
1✔
96
    """sequence of str: label aliases"""
1✔
97
    LOGO_EMOJI = ''
1✔
98
    """str: logo emoji, if any"""
1✔
99
    LOGO_HTML = ''
1✔
100
    """str: logo ``<img>`` tag, if any"""
1✔
101
    CONTENT_TYPE = None
1✔
102
    """str: MIME type of this protocol's native data format, appropriate for the ``Content-Type`` HTTP header."""
1✔
103
    HAS_COPIES = False
1✔
104
    """bool: whether this protocol is push and needs us to proactively create "copy" users and objects, as opposed to pulling converted objects on demand"""
1✔
105
    DEFAULT_TARGET = None
1✔
106
    """str: optional, the default target URI to send this protocol's activities to. May be used as the "shared" target. Often only set if ``HAS_COPIES`` is true."""
1✔
107
    REQUIRES_AVATAR = False
1✔
108
    """bool: whether accounts on this protocol are required to have a profile picture. If they don't, their ``User.status`` will be ``blocked``."""
1✔
109
    REQUIRES_NAME = False
1✔
110
    """bool: whether accounts on this protocol are required to have a profile name that's different than their handle or id. If they don't, their ``User.status`` will be ``blocked``."""
1✔
111
    REQUIRES_OLD_ACCOUNT = False
1✔
112
    """bool: whether accounts on this protocol are required to be at least :const:`common.OLD_ACCOUNT_AGE` old. If their profile includes creation date and it's not old enough, their ``User.status`` will be ``blocked``."""
1✔
113
    DEFAULT_ENABLED_PROTOCOLS = ()
1✔
114
    """sequence of str: labels of other protocols that are automatically enabled for this protocol to bridge into"""
1✔
115
    DEFAULT_SERVE_USER_PAGES = False
1✔
116
    """bool: whether to serve user pages for all of this protocol's users on the fed.brid.gy. If ``False``, user pages will only be served for users who have explictly opted in."""
1✔
117
    SUPPORTED_AS1_TYPES = ()
1✔
118
    """sequence of str: AS1 objectTypes and verbs that this protocol supports receiving and sending"""
1✔
119
    SUPPORTS_DMS = False
1✔
120
    """bool: whether this protocol can receive DMs (chat messages)"""
1✔
121
    USES_OBJECT_FEED = False
1✔
122
    """bool: whether to store followers on this protocol in :attr:`Object.feed`."""
1✔
123
    HTML_PROFILES = False
1✔
124
    """bool: whether this protocol supports HTML in profile descriptions. If False, profile descriptions should be plain text."""
1✔
125
    SEND_REPLIES_TO_ORIG_POSTS_MENTIONS = False
1✔
126
    """bool: whether replies to this protocol should include the original post's mentions as delivery targets"""
1✔
127

128
    @classmethod
1✔
129
    @property
1✔
130
    def LABEL(cls):
1✔
131
        """str: human-readable lower case name of this protocol, eg ``'activitypub``"""
132
        return cls.__name__.lower()
1✔
133

134
    @staticmethod
1✔
135
    def for_request(fed=None):
1✔
136
        """Returns the protocol for the current request.
137

138
        ...based on the request's hostname.
139

140
        Args:
141
          fed (str or protocol.Protocol): protocol to return if the current
142
            request is on ``fed.brid.gy``
143

144
        Returns:
145
          Protocol: protocol, or None if the provided domain or request hostname
146
          domain is not a subdomain of ``brid.gy`` or isn't a known protocol
147
        """
148
        return Protocol.for_bridgy_subdomain(request.host, fed=fed)
1✔
149

150
    @staticmethod
1✔
151
    def for_bridgy_subdomain(domain_or_url, fed=None):
1✔
152
        """Returns the protocol for a brid.gy subdomain.
153

154
        Args:
155
          domain_or_url (str)
156
          fed (str or protocol.Protocol): protocol to return if the current
157
            request is on ``fed.brid.gy``
158

159
        Returns:
160
          class: :class:`Protocol` subclass, or None if the provided domain or request
161
          hostname domain is not a subdomain of ``brid.gy`` or isn't a known
162
          protocol
163
        """
164
        domain = (util.domain_from_link(domain_or_url, minimize=False)
1✔
165
                  if util.is_web(domain_or_url)
166
                  else domain_or_url)
167

168
        if domain == common.PRIMARY_DOMAIN or domain in common.LOCAL_DOMAINS:
1✔
169
            return PROTOCOLS[fed] if isinstance(fed, str) else fed
1✔
170
        elif domain and domain.endswith(common.SUPERDOMAIN):
1✔
171
            label = domain.removesuffix(common.SUPERDOMAIN)
1✔
172
            return PROTOCOLS.get(label)
1✔
173

174
    @classmethod
1✔
175
    def owns_id(cls, id):
1✔
176
        """Returns whether this protocol owns the id, or None if it's unclear.
177

178
        To be implemented by subclasses.
179

180
        IDs are string identities that uniquely identify users, and are intended
181
        primarily to be machine readable and usable. Compare to handles, which
182
        are human-chosen, human-meaningful, and often but not always unique.
183

184
        Some protocols' ids are more or less deterministic based on the id
185
        format, eg AT Protocol owns ``at://`` URIs. Others, like http(s) URLs,
186
        could be owned by eg Web or ActivityPub.
187

188
        This should be a quick guess without expensive side effects, eg no
189
        external HTTP fetches to fetch the id itself or otherwise perform
190
        discovery.
191

192
        Returns False if the id's domain is in :const:`common.DOMAIN_BLOCKLIST`.
193

194
        Args:
195
          id (str)
196

197
        Returns:
198
          bool or None:
199
        """
200
        return False
1✔
201

202
    @classmethod
1✔
203
    def owns_handle(cls, handle, allow_internal=False):
1✔
204
        """Returns whether this protocol owns the handle, or None if it's unclear.
205

206
        To be implemented by subclasses.
207

208
        Handles are string identities that are human-chosen, human-meaningful,
209
        and often but not always unique. Compare to IDs, which uniquely identify
210
        users, and are intended primarily to be machine readable and usable.
211

212
        Some protocols' handles are more or less deterministic based on the id
213
        format, eg ActivityPub (technically WebFinger) handles are
214
        ``@user@instance.com``. Others, like domains, could be owned by eg Web,
215
        ActivityPub, AT Protocol, or others.
216

217
        This should be a quick guess without expensive side effects, eg no
218
        external HTTP fetches to fetch the id itself or otherwise perform
219
        discovery.
220

221
        Args:
222
          handle (str)
223
          allow_internal (bool): whether to return False for internal domains
224
            like ``fed.brid.gy``, ``bsky.brid.gy``, etc
225

226
        Returns:
227
          bool or None
228
        """
229
        return False
1✔
230

231
    @classmethod
1✔
232
    def handle_to_id(cls, handle):
1✔
233
        """Converts a handle to an id.
234

235
        To be implemented by subclasses.
236

237
        May incur network requests, eg DNS queries or HTTP requests. Avoids
238
        blocked or opted out users.
239

240
        Args:
241
          handle (str)
242

243
        Returns:
244
          str: corresponding id, or None if the handle can't be found
245
        """
246
        raise NotImplementedError()
×
247

248
    @classmethod
1✔
249
    def key_for(cls, id, allow_opt_out=False):
1✔
250
        """Returns the :class:`google.cloud.ndb.Key` for a given id's :class:`models.User`.
251

252
        To be implemented by subclasses. Canonicalizes the id if necessary.
253

254
        If called via `Protocol.key_for`, infers the appropriate protocol with
255
        :meth:`for_id`. If called with a concrete subclass, uses that subclass
256
        as is.
257

258
        Args:
259
          id (str):
260
          allow_opt_out (bool): whether to allow users who are currently opted out
261

262
        Returns:
263
          google.cloud.ndb.Key: matching key, or None if the given id is not a
264
          valid :class:`User` id for this protocol.
265
        """
266
        if cls == Protocol:
1✔
267
            proto = Protocol.for_id(id)
1✔
268
            return proto.key_for(id, allow_opt_out=allow_opt_out) if proto else None
1✔
269

270
        # load user so that we follow use_instead
271
        existing = cls.get_by_id(id, allow_opt_out=True)
1✔
272
        if existing:
1✔
273
            if existing.status and not allow_opt_out:
1✔
274
                return None
1✔
275
            return existing.key
1✔
276

277
        return cls(id=id).key
1✔
278

279
    @staticmethod
1✔
280
    def _for_id_memcache_key(id, remote=None):
1✔
281
        """If id is a URL, uses its domain, otherwise returns None.
282

283
        Args:
284
          id (str)
285

286
        Returns:
287
          (str domain, bool remote) or None
288
        """
289
        domain = util.domain_from_link(id)
1✔
290
        if domain in PROTOCOL_DOMAINS:
1✔
291
            return id
1✔
292
        elif remote and util.is_web(id):
1✔
293
            return domain
1✔
294

295
    @cached(LRUCache(20000), lock=Lock())
1✔
296
    @memcache.memoize(key=_for_id_memcache_key, write=lambda id, remote=True: remote,
1✔
297
                      version=3)
298
    @staticmethod
1✔
299
    def for_id(id, remote=True):
1✔
300
        """Returns the protocol for a given id.
301

302
        Args:
303
          id (str)
304
          remote (bool): whether to perform expensive side effects like fetching
305
            the id itself over the network, or other discovery.
306

307
        Returns:
308
          Protocol subclass: matching protocol, or None if no single known
309
          protocol definitively owns this id
310
        """
311
        logger.debug(f'Determining protocol for id {id}')
1✔
312
        if not id:
1✔
313
            return None
1✔
314

315
        # remove our synthetic id fragment, if any
316
        #
317
        # will this eventually cause false positives for other services that
318
        # include our full ids inside their own ids, non-URL-encoded? guess
319
        # we'll figure that out if/when it happens.
320
        id = id.partition('#bridgy-fed-')[0]
1✔
321
        if not id:
1✔
322
            return None
1✔
323

324
        if util.is_web(id):
1✔
325
            # step 1: check for our per-protocol subdomains
326
            try:
1✔
327
                parsed = urlparse(id)
1✔
328
            except ValueError as e:
1✔
329
                logger.info(f'urlparse ValueError: {e}')
1✔
330
                return None
1✔
331

332
            is_homepage = parsed.path.strip('/') == ''
1✔
333
            is_internal = parsed.path.startswith(ids.INTERNAL_PATH_PREFIX)
1✔
334
            by_subdomain = Protocol.for_bridgy_subdomain(id)
1✔
335
            if by_subdomain and not (is_homepage or is_internal
1✔
336
                                     or id in BOT_ACTOR_AP_IDS):
337
                logger.debug(f'  {by_subdomain.LABEL} owns id {id}')
1✔
338
                return by_subdomain
1✔
339

340
        # step 2: check if any Protocols say conclusively that they own it
341
        # sort to be deterministic
342
        protocols = sorted(set(p for p in PROTOCOLS.values() if p),
1✔
343
                           key=lambda p: p.LABEL)
344
        candidates = []
1✔
345
        for protocol in protocols:
1✔
346
            owns = protocol.owns_id(id)
1✔
347
            if owns:
1✔
348
                logger.debug(f'  {protocol.LABEL} owns id {id}')
1✔
349
                return protocol
1✔
350
            elif owns is not False:
1✔
351
                candidates.append(protocol)
1✔
352

353
        if len(candidates) == 1:
1✔
354
            logger.debug(f'  {candidates[0].LABEL} owns id {id}')
1✔
355
            return candidates[0]
1✔
356

357
        # step 3: look for existing Objects in the datastore
358
        #
359
        # note that we don't currently see if this is a copy id because I have FUD
360
        # over which Protocol for_id should return in that case...and also because a
361
        # protocol may already say definitively above that it owns the id, eg ATProto
362
        # with DIDs and at:// URIs.
363
        obj = Protocol.load(id, remote=False)
1✔
364
        if obj and obj.source_protocol:
1✔
365
            logger.debug(f'  {obj.key.id()} owned by source_protocol {obj.source_protocol}')
1✔
366
            return PROTOCOLS[obj.source_protocol]
1✔
367

368
        # step 4: fetch over the network, if necessary
369
        if not remote:
1✔
370
            return None
1✔
371

372
        for protocol in candidates:
1✔
373
            logger.debug(f'Trying {protocol.LABEL}')
1✔
374
            try:
1✔
375
                obj = protocol.load(id, local=False, remote=True)
1✔
376

377
                if protocol.ABBREV == 'web':
1✔
378
                    # for web, if we fetch and get HTML without microformats,
379
                    # load returns False but the object will be stored in the
380
                    # datastore with source_protocol web, and in cache. load it
381
                    # again manually to check for that.
382
                    obj = Object.get_by_id(id)
1✔
383
                    if obj and obj.source_protocol != 'web':
1✔
384
                        obj = None
×
385

386
                if obj:
1✔
387
                    logger.debug(f'  {protocol.LABEL} owns id {id}')
1✔
388
                    return protocol
1✔
389
            except BadGateway:
1✔
390
                # we tried and failed fetching the id over the network.
391
                # this depends on ActivityPub.fetch raising this!
392
                return None
1✔
393
            except HTTPException as e:
×
394
                # internal error we generated ourselves; try next protocol
395
                pass
×
396
            except Exception as e:
×
397
                code, _ = util.interpret_http_exception(e)
×
398
                if code:
×
399
                    # we tried and failed fetching the id over the network
400
                    return None
×
401
                raise
×
402

403
        logger.info(f'No matching protocol found for {id} !')
1✔
404
        return None
1✔
405

406
    @cached(LRUCache(20000), lock=Lock())
1✔
407
    @staticmethod
1✔
408
    def for_handle(handle):
1✔
409
        """Returns the protocol for a given handle.
410

411
        May incur expensive side effects like resolving the handle itself over
412
        the network or other discovery.
413

414
        Args:
415
          handle (str)
416

417
        Returns:
418
          (Protocol subclass, str) tuple: matching protocol and optional id (if
419
          resolved), or ``(None, None)`` if no known protocol owns this handle
420
        """
421
        # TODO: normalize, eg convert domains to lower case
422
        logger.debug(f'Determining protocol for handle {handle}')
1✔
423
        if not handle:
1✔
424
            return (None, None)
1✔
425

426
        # step 1: check if any Protocols say conclusively that they own it.
427
        # sort to be deterministic.
428
        protocols = sorted(set(p for p in PROTOCOLS.values() if p),
1✔
429
                           key=lambda p: p.LABEL)
430
        candidates = []
1✔
431
        for proto in protocols:
1✔
432
            owns = proto.owns_handle(handle)
1✔
433
            if owns:
1✔
434
                logger.debug(f'  {proto.LABEL} owns handle {handle}')
1✔
435
                return (proto, None)
1✔
436
            elif owns is not False:
1✔
437
                candidates.append(proto)
1✔
438

439
        if len(candidates) == 1:
1✔
440
            logger.debug(f'  {candidates[0].LABEL} owns handle {handle}')
×
441
            return (candidates[0], None)
×
442

443
        # step 2: look for matching User in the datastore
444
        for proto in candidates:
1✔
445
            user = proto.query(proto.handle == handle).get()
1✔
446
            if user:
1✔
447
                if user.status:
1✔
448
                    return (None, None)
1✔
449
                logger.debug(f'  user {user.key} handle {handle}')
1✔
450
                return (proto, user.key.id())
1✔
451

452
        # step 3: resolve handle to id
453
        for proto in candidates:
1✔
454
            id = proto.handle_to_id(handle)
1✔
455
            if id:
1✔
456
                logger.debug(f'  {proto.LABEL} resolved handle {handle} to id {id}')
1✔
457
                return (proto, id)
1✔
458

459
        logger.info(f'No matching protocol found for handle {handle} !')
1✔
460
        return (None, None)
1✔
461

462
    @classmethod
1✔
463
    def is_user_at_domain(cls, handle, allow_internal=False):
1✔
464
        """Returns True if handle is formatted ``user@domain.tld``, False otherwise.
465

466
        Example: ``@user@instance.com``
467

468
        Args:
469
          handle (str)
470
          allow_internal (bool): whether the domain can be a Bridgy Fed domain
471
        """
472
        parts = handle.split('@')
1✔
473
        if len(parts) != 2:
1✔
474
            return False
1✔
475

476
        user, domain = parts
1✔
477
        return bool(user and domain
1✔
478
                    and not cls.is_blocklisted(domain, allow_internal=allow_internal))
479

480
    @classmethod
1✔
481
    def bridged_web_url_for(cls, user, fallback=False):
1✔
482
        """Returns the web URL for a user's bridged profile in this protocol.
483

484
        For example, for Web user ``alice.com``, :meth:`ATProto.bridged_web_url_for`
485
        returns ``https://bsky.app/profile/alice.com.web.brid.gy``
486

487
        Args:
488
          user (models.User)
489
          fallback (bool): if True, and bridged users have no canonical user
490
            profile URL in this protocol, return the native protocol's profile URL
491

492
        Returns:
493
          str, or None if there isn't a canonical URL
494
        """
495
        if fallback:
1✔
496
            return user.web_url()
1✔
497

498
    @classmethod
1✔
499
    def actor_key(cls, obj, allow_opt_out=False):
1✔
500
        """Returns the :class:`User`: key for a given object's author or actor.
501

502
        Args:
503
          obj (models.Object)
504
          allow_opt_out (bool): whether to return a user key if they're opted out
505

506
        Returns:
507
          google.cloud.ndb.key.Key or None:
508
        """
509
        owner = as1.get_owner(obj.as1)
1✔
510
        if owner:
1✔
511
            return cls.key_for(owner, allow_opt_out=allow_opt_out)
1✔
512

513
    @classmethod
1✔
514
    def bot_user_id(cls):
1✔
515
        """Returns the Web user id for the bot user for this protocol.
516

517
        For example, ``'bsky.brid.gy'`` for ATProto.
518

519
        Returns:
520
          str:
521
        """
522
        return f'{cls.ABBREV}{common.SUPERDOMAIN}'
1✔
523

524
    @classmethod
1✔
525
    def create_for(cls, user):
1✔
526
        """Creates or re-activate a copy user in this protocol.
527

528
        Should add the copy user to :attr:`copies`.
529

530
        If the copy user already exists and active, should do nothing.
531

532
        Args:
533
          user (models.User): original source user. Shouldn't already have a
534
            copy user for this protocol in :attr:`copies`.
535

536
        Raises:
537
          ValueError: if we can't create a copy of the given user in this protocol
538
        """
539
        raise NotImplementedError()
×
540

541
    @classmethod
1✔
542
    def send(to_cls, obj, target, from_user=None, orig_obj_id=None):
1✔
543
        """Sends an outgoing activity.
544

545
        To be implemented by subclasses. Should call
546
        ``to_cls.translate_ids(obj.as1)`` before converting it to this Protocol's
547
        format.
548

549
        NOTE: if this protocol's ``HAS_COPIES`` is True, and this method creates
550
        a copy and sends it, it *must* add that copy to the *object*'s (not
551
        activity's) :attr:`copies`, and store it back in the datastore!
552

553
        Args:
554
          obj (models.Object): with activity to send
555
          target (str): destination URL to send to
556
          from_user (models.User): user (actor) this activity is from
557
          orig_obj_id (str): :class:`models.Object` key id of the "original object"
558
            that this object refers to, eg replies to or reposts or likes
559

560
        Returns:
561
          bool: True if the activity is sent successfully, False if it is
562
          ignored or otherwise unsent due to protocol logic, eg no webmention
563
          endpoint, protocol doesn't support the activity type. (Failures are
564
          raised as exceptions.)
565

566
        Raises:
567
          werkzeug.HTTPException if the request fails
568
        """
569
        raise NotImplementedError()
×
570

571
    @classmethod
1✔
572
    def fetch(cls, obj, **kwargs):
1✔
573
        """Fetches a protocol-specific object and populates it in an :class:`Object`.
574

575
        Errors are raised as exceptions. If this method returns False, the fetch
576
        didn't fail but didn't succeed either, eg the id isn't valid for this
577
        protocol, or the fetch didn't return valid data for this protocol.
578

579
        To be implemented by subclasses.
580

581
        Args:
582
          obj (models.Object): with the id to fetch. Data is filled into one of
583
            the protocol-specific properties, eg ``as2``, ``mf2``, ``bsky``.
584
          kwargs: subclass-specific
585

586
        Returns:
587
          bool: True if the object was fetched and populated successfully,
588
          False otherwise
589

590
        Raises:
591
          requests.RequestException, werkzeug.HTTPException,
592
          websockets.WebSocketException, etc: if the fetch fails
593
        """
594
        raise NotImplementedError()
×
595

596
    @classmethod
1✔
597
    def convert(cls, obj, from_user=None, **kwargs):
1✔
598
        """Converts an :class:`Object` to this protocol's data format.
599

600
        For example, an HTML string for :class:`Web`, or a dict with AS2 JSON
601
        and ``application/activity+json`` for :class:`ActivityPub`.
602

603
        Just passes through to :meth:`_convert`, then does minor
604
        protocol-independent postprocessing.
605

606
        Args:
607
          obj (models.Object):
608
          from_user (models.User): user (actor) this activity/object is from
609
          kwargs: protocol-specific, passed through to :meth:`_convert`
610

611
        Returns:
612
          converted object in the protocol's native format, often a dict
613
        """
614
        if not obj or not obj.as1:
1✔
615
            return {}
1✔
616

617
        id = obj.key.id() if obj.key else obj.as1.get('id')
1✔
618
        is_crud = obj.as1.get('verb') in as1.CRUD_VERBS
1✔
619
        base_obj = as1.get_object(obj.as1) if is_crud else obj.as1
1✔
620
        orig_our_as1 = obj.our_as1
1✔
621

622
        # mark bridged actors as bots and add "bridged by Bridgy Fed" to their bios
623
        if (from_user and base_obj
1✔
624
            and base_obj.get('objectType') in as1.ACTOR_TYPES
625
            and PROTOCOLS.get(obj.source_protocol) != cls
626
            and Protocol.for_bridgy_subdomain(id) not in DOMAINS
627
            # Web users are special cased, they don't get the label if they've
628
            # explicitly enabled Bridgy Fed with redirects or webmentions
629
            and not (from_user.LABEL == 'web'
630
                     and (from_user.last_webmention_in or from_user.has_redirects))):
631
            cls.add_source_links(obj=obj, from_user=from_user)
1✔
632

633
        converted = cls._convert(obj, from_user=from_user, **kwargs)
1✔
634
        obj.our_as1 = orig_our_as1
1✔
635
        return converted
1✔
636

637
    @classmethod
1✔
638
    def _convert(cls, obj, from_user=None, **kwargs):
1✔
639
        """Converts an :class:`Object` to this protocol's data format.
640

641
        To be implemented by subclasses. Implementations should generally call
642
        :meth:`Protocol.translate_ids` (as their own class) before converting to
643
        their format.
644

645
        Args:
646
          obj (models.Object):
647
          from_user (models.User): user (actor) this activity/object is from
648
          kwargs: protocol-specific
649

650
        Returns:
651
          converted object in the protocol's native format, often a dict. May
652
            return the ``{}`` empty dict if the object can't be converted.
653
        """
654
        raise NotImplementedError()
×
655

656
    @classmethod
1✔
657
    def add_source_links(cls, obj, from_user):
1✔
658
        """Adds "bridged from ... by Bridgy Fed" to the user's actor's ``summary``.
659

660
        Uses HTML for protocols that support it, plain text otherwise.
661

662
        Args:
663
          obj (models.Object): user's actor/profile object
664
          from_user (models.User): user (actor) this activity/object is from
665
        """
666
        assert obj and obj.as1
1✔
667
        assert from_user
1✔
668

669
        obj.our_as1 = copy.deepcopy(obj.as1)
1✔
670
        actor = (as1.get_object(obj.as1) if obj.as1.get('verb') in as1.CRUD_VERBS
1✔
671
                 else obj.as1)
672
        actor['objectType'] = 'person'
1✔
673

674
        orig_summary = actor.setdefault('summary', '')
1✔
675
        summary_text = html_to_text(orig_summary, ignore_links=True)
1✔
676

677
        # Check if we've already added source links
678
        if '🌉 bridged' in summary_text:
1✔
679
            return
1✔
680

681
        actor_id = actor.get('id')
1✔
682

683
        url = (as1.get_url(actor)
1✔
684
               or (from_user.web_url() if from_user.profile_id() == actor_id
685
                   else actor_id))
686

687
        from web import Web
1✔
688
        bot_user = Web.get_by_id(from_user.bot_user_id())
1✔
689

690
        if cls.HTML_PROFILES:
1✔
691
            if bot_user and from_user.LABEL not in cls.DEFAULT_ENABLED_PROTOCOLS:
1✔
692
                mention = bot_user.user_link(proto=cls, name=False, handle='short')
1✔
693
                suffix = f', follow {mention} to interact'
1✔
694
            else:
695
                suffix = f' by <a href="https://{PRIMARY_DOMAIN}/">Bridgy Fed</a>'
1✔
696

697
            separator = '<br><br>'
1✔
698

699
            is_user = from_user.key and actor_id in (from_user.key.id(),
1✔
700
                                                     from_user.profile_id())
701
            if is_user:
1✔
702
                bridged = f'🌉 <a href="https://{PRIMARY_DOMAIN}{from_user.user_page_path()}">bridged</a>'
1✔
703
                from_ = f'<a href="{from_user.web_url()}">{from_user.handle}</a>'
1✔
704
            else:
705
                bridged = '🌉 bridged'
1✔
706
                from_ = util.pretty_link(url) if url else '?'
1✔
707

708
        else:  # plain text
709
            # TODO: unify with above. which is right?
710
            id = obj.key.id() if obj.key else obj.our_as1.get('id')
1✔
711
            is_user = from_user.key and id in (from_user.key.id(),
1✔
712
                                               from_user.profile_id())
713
            from_ = (from_user.web_url() if is_user else url) or '?'
1✔
714

715
            bridged = '🌉 bridged'
1✔
716
            suffix = (
1✔
717
                f': https://{PRIMARY_DOMAIN}{from_user.user_page_path()}'
718
                # link web users to their user pages
719
                if from_user.LABEL == 'web'
720
                else f', follow @{bot_user.handle_as(cls)} to interact'
721
                if bot_user and from_user.LABEL not in cls.DEFAULT_ENABLED_PROTOCOLS
722
                else f' by https://{PRIMARY_DOMAIN}/')
723
            separator = '\n\n'
1✔
724
            orig_summary = summary_text
1✔
725

726
        logo = f'{from_user.LOGO_EMOJI} ' if from_user.LOGO_EMOJI else ''
1✔
727
        source_links = f'{separator if orig_summary else ""}{bridged} from {logo}{from_}{suffix}'
1✔
728
        actor['summary'] = orig_summary + source_links
1✔
729

730
    @classmethod
1✔
731
    def set_username(to_cls, user, username):
1✔
732
        """Sets a custom username for a user's bridged account in this protocol.
733

734
        Args:
735
          user (models.User)
736
          username (str)
737

738
        Raises:
739
          ValueError: if the username is invalid
740
          RuntimeError: if the username could not be set
741
        """
742
        raise NotImplementedError()
1✔
743

744
    @classmethod
1✔
745
    def migrate_out(cls, user, to_user_id):
1✔
746
        """Migrates a bridged account out to be a native account.
747

748
        Args:
749
          user (models.User)
750
          to_user_id (str)
751

752
        Raises:
753
          ValueError: eg if this protocol doesn't own ``to_user_id``, or if
754
            ``user`` is on this protocol or not bridged to this protocol
755
        """
756
        raise NotImplementedError()
×
757

758
    @classmethod
1✔
759
    def check_can_migrate_out(cls, user, to_user_id):
1✔
760
        """Raises an exception if a user can't yet migrate to a native account.
761

762
        For example, if ``to_user_id`` isn't on this protocol, or if ``user`` is on
763
        this protocol, or isn't bridged to this protocol.
764

765
        If the user is ready to migrate, returns ``None``.
766

767
        Subclasses may override this to add more criteria, but they should call this
768
        implementation first.
769

770
        Args:
771
          user (models.User)
772
          to_user_id (str)
773

774
        Raises:
775
          ValueError: if ``user`` isn't ready to migrate to this protocol yet
776
        """
777
        def _error(msg):
1✔
778
            logger.warning(msg)
1✔
779
            raise ValueError(msg)
1✔
780

781
        if cls.owns_id(to_user_id) is False:
1✔
782
            _error(f"{to_user_id} doesn't look like an {cls.LABEL} id")
1✔
783
        elif isinstance(user, cls):
1✔
784
            _error(f"{user.handle_or_id()} is on {cls.PHRASE}")
1✔
785
        elif not user.is_enabled(cls):
1✔
786
            _error(f"{user.handle_or_id()} isn't currently bridged to {cls.PHRASE}")
1✔
787

788
    @classmethod
1✔
789
    def migrate_in(cls, user, from_user_id, **kwargs):
1✔
790
        """Migrates a native account in to be a bridged account.
791

792
        The protocol independent parts are done here; protocol-specific parts are
793
        done in :meth:`_migrate_in`, which this wraps.
794

795
        Reloads the user's profile before calling :meth:`_migrate_in`.
796

797
        Args:
798
          user (models.User): native user on another protocol to attach the
799
            newly imported bridged account to
800
          from_user_id (str)
801
          kwargs: additional protocol-specific parameters
802

803
        Raises:
804
          ValueError: eg if this protocol doesn't own ``from_user_id``, or if
805
            ``user`` is on this protocol or already bridged to this protocol
806
        """
807
        def _error(msg):
1✔
808
            logger.warning(msg)
1✔
809
            raise ValueError(msg)
1✔
810

811
        logger.info(f"Migrating in {from_user_id} for {user.key.id()}")
1✔
812

813
        # check req'ts
814
        if cls.owns_id(from_user_id) is False:
1✔
815
            _error(f"{from_user_id} doesn't look like an {cls.LABEL} id")
1✔
816
        elif isinstance(user, cls):
1✔
817
            _error(f"{user.handle_or_id()} is on {cls.PHRASE}")
1✔
818
        elif cls.LABEL in user.enabled_protocols:
1✔
819
            _error(f"{user.handle_or_id()} is already bridged to {cls.PHRASE}")
1✔
820

821
        # reload profile
822
        try:
1✔
823
            user.reload_profile()
1✔
824
        except (RequestException, HTTPException) as e:
×
825
            _, msg = util.interpret_http_exception(e)
×
826

827
        # migrate!
828
        cls._migrate_in(user, from_user_id, **kwargs)
1✔
829
        user.add('enabled_protocols', cls.LABEL)
1✔
830
        user.put()
1✔
831

832
        # attach profile object
833
        if user.obj:
1✔
834
            if cls.HAS_COPIES:
1✔
835
                profile_id = ids.profile_id(id=from_user_id, proto=cls)
1✔
836
                user.obj.remove_copies_on(cls)
1✔
837
                user.obj.add('copies', Target(uri=profile_id, protocol=cls.LABEL))
1✔
838
                user.obj.put()
1✔
839

840
            common.create_task(queue='receive', obj_id=user.obj_key.id(),
1✔
841
                               authed_as=user.key.id())
842

843
    @classmethod
1✔
844
    def _migrate_in(cls, user, from_user_id, **kwargs):
1✔
845
        """Protocol-specific parts of migrating in external account.
846

847
        Called by :meth:`migrate_in`, which does most of the work.
848

849
        Args:
850
          user (models.User): native user on another protocol to attach the
851
            newly imported account to. Unused.
852
          from_user_id (str): DID of the account to be migrated in
853
          kwargs: protocol dependent
854
        """
855
        raise NotImplementedError()
×
856

857
    @classmethod
1✔
858
    def target_for(cls, obj, shared=False):
1✔
859
        """Returns an :class:`Object`'s delivery target (endpoint).
860

861
        To be implemented by subclasses.
862

863
        Examples:
864

865
        * If obj has ``source_protocol`` ``web``, returns its URL, as a
866
          webmention target.
867
        * If obj is an ``activitypub`` actor, returns its inbox.
868
        * If obj is an ``activitypub`` object, returns it's author's or actor's
869
          inbox.
870

871
        Args:
872
          obj (models.Object):
873
          shared (bool): optional. If True, returns a common/shared
874
            endpoint, eg ActivityPub's ``sharedInbox``, that can be reused for
875
            multiple recipients for efficiency
876

877
        Returns:
878
          str: target endpoint, or None if not available.
879
        """
880
        raise NotImplementedError()
×
881

882
    @classmethod
1✔
883
    def is_blocklisted(cls, url, allow_internal=False):
1✔
884
        """Returns True if we block the given URL and shouldn't deliver to it.
885

886
        Default implementation here, subclasses may override.
887

888
        Args:
889
          url (str):
890
          allow_internal (bool): whether to return False for internal domains
891
            like ``fed.brid.gy``, ``bsky.brid.gy``, etc
892
        """
893
        blocklist = DOMAIN_BLOCKLIST
1✔
894
        if not DEBUG:
1✔
895
            blocklist += tuple(util.RESERVED_TLDS | util.LOCAL_TLDS)
×
896
        if not allow_internal:
1✔
897
            blocklist += DOMAINS
1✔
898
        return util.domain_or_parent_in(url, blocklist)
1✔
899

900
    @classmethod
1✔
901
    def translate_ids(to_cls, obj):
1✔
902
        """Translates all ids in an AS1 object to a specific protocol.
903

904
        Infers source protocol for each id value separately.
905

906
        For example, if ``proto`` is :class:`ActivityPub`, the ATProto URI
907
        ``at://did:plc:abc/coll/123`` will be converted to
908
        ``https://bsky.brid.gy/ap/at://did:plc:abc/coll/123``.
909

910
        Wraps these AS1 fields:
911

912
        * ``id``
913
        * ``actor``
914
        * ``author``
915
        * ``bcc``
916
        * ``bto``
917
        * ``cc``
918
        * ``featured[].items``, ``featured[].orderedItems``
919
        * ``object``
920
        * ``object.actor``
921
        * ``object.author``
922
        * ``object.id``
923
        * ``object.inReplyTo``
924
        * ``object.object``
925
        * ``attachments[].id``
926
        * ``tags[objectType=mention].url``
927
        * ``to``
928

929
        This is the inverse of :meth:`models.Object.resolve_ids`. Much of the
930
        same logic is duplicated there!
931

932
        TODO: unify with :meth:`Object.resolve_ids`,
933
        :meth:`models.Object.normalize_ids`.
934

935
        Args:
936
          to_proto (Protocol subclass)
937
          obj (dict): AS1 object or activity (not :class:`models.Object`!)
938

939
        Returns:
940
          dict: wrapped AS1 version of ``obj``
941
        """
942
        assert to_cls != Protocol
1✔
943
        if not obj:
1✔
944
            return obj
1✔
945

946
        outer_obj = copy.deepcopy(obj)
1✔
947
        inner_objs = outer_obj['object'] = as1.get_objects(outer_obj)
1✔
948

949
        def translate(elem, field, fn, uri=False):
1✔
950
            elem[field] = as1.get_objects(elem, field)
1✔
951
            for obj in elem[field]:
1✔
952
                if id := obj.get('id'):
1✔
953
                    if field in ('to', 'cc', 'bcc', 'bto') and as1.is_audience(id):
1✔
954
                        continue
1✔
955
                    from_cls = Protocol.for_id(id)
1✔
956
                    # TODO: what if from_cls is None? relax translate_object_id,
957
                    # make it a noop if we don't know enough about from/to?
958
                    if from_cls and from_cls != to_cls:
1✔
959
                        obj['id'] = fn(id=id, from_=from_cls, to=to_cls)
1✔
960
                    if obj['id'] and uri:
1✔
961
                        obj['id'] = to_cls(id=obj['id']).id_uri()
1✔
962

963
            elem[field] = [o['id'] if o.keys() == {'id'} else o
1✔
964
                           for o in elem[field]]
965

966
            if len(elem[field]) == 1 and field not in ('items', 'orderedItems'):
1✔
967
                elem[field] = elem[field][0]
1✔
968

969
        type = as1.object_type(outer_obj)
1✔
970
        translate(outer_obj, 'id',
1✔
971
                  translate_user_id if type in as1.ACTOR_TYPES
972
                  else translate_object_id)
973

974
        for o in inner_objs:
1✔
975
            is_actor = (as1.object_type(o) in as1.ACTOR_TYPES
1✔
976
                        or as1.get_owner(outer_obj) == o.get('id')
977
                        or type in ('follow', 'stop-following'))
978
            translate(o, 'id', translate_user_id if is_actor else translate_object_id)
1✔
979
            obj_is_actor = o.get('verb') in as1.VERBS_WITH_ACTOR_OBJECT
1✔
980
            translate(o, 'object', translate_user_id if obj_is_actor
1✔
981
                      else translate_object_id)
982

983
        for o in [outer_obj] + inner_objs:
1✔
984
            translate(o, 'inReplyTo', translate_object_id)
1✔
985
            for field in 'actor', 'author', 'to', 'cc', 'bto', 'bcc':
1✔
986
                translate(o, field, translate_user_id)
1✔
987
            for tag in as1.get_objects(o, 'tags'):
1✔
988
                if tag.get('objectType') == 'mention':
1✔
989
                    translate(tag, 'url', translate_user_id, uri=True)
1✔
990
            for att in as1.get_objects(o, 'attachments'):
1✔
991
                translate(att, 'id', translate_object_id)
1✔
992
                url = att.get('url')
1✔
993
                if url and not att.get('id'):
1✔
994
                    if from_cls := Protocol.for_id(url):
1✔
995
                        att['id'] = translate_object_id(from_=from_cls, to=to_cls,
1✔
996
                                                        id=url)
997
            if feat := as1.get_object(o, 'featured'):
1✔
998
                translate(feat, 'orderedItems', translate_object_id)
1✔
999
                translate(feat, 'items', translate_object_id)
1✔
1000

1001
        outer_obj = util.trim_nulls(outer_obj)
1✔
1002

1003
        if objs := util.get_list(outer_obj ,'object'):
1✔
1004
            outer_obj['object'] = [o['id'] if o.keys() == {'id'} else o for o in objs]
1✔
1005
            if len(outer_obj['object']) == 1:
1✔
1006
                outer_obj['object'] = outer_obj['object'][0]
1✔
1007

1008
        return outer_obj
1✔
1009

1010
    @classmethod
1✔
1011
    def receive(from_cls, obj, authed_as=None, internal=False, received_at=None):
1✔
1012
        """Handles an incoming activity.
1013

1014
        If ``obj``'s key is unset, ``obj.as1``'s id field is used. If both are
1015
        unset, returns HTTP 299.
1016

1017
        Args:
1018
          obj (models.Object)
1019
          authed_as (str): authenticated actor id who sent this activity
1020
          internal (bool): whether to allow activity ids on internal domains,
1021
            from opted out/blocked users, etc.
1022
          received_at (datetime): when we first saw (received) this activity.
1023
            Right now only used for monitoring.
1024

1025
        Returns:
1026
          (str, int) tuple: (response body, HTTP status code) Flask response
1027

1028
        Raises:
1029
          werkzeug.HTTPException: if the request is invalid
1030
        """
1031
        # check some invariants
1032
        assert from_cls != Protocol
1✔
1033
        assert isinstance(obj, Object), obj
1✔
1034

1035
        if not obj.as1:
1✔
1036
            error('No object data provided')
1✔
1037

1038
        id = None
1✔
1039
        if obj.key and obj.key.id():
1✔
1040
            id = obj.key.id()
1✔
1041

1042
        if not id:
1✔
1043
            id = obj.as1.get('id')
1✔
1044
            obj.key = ndb.Key(Object, id)
1✔
1045

1046
        if not id:
1✔
1047
            error('No id provided')
×
1048
        elif from_cls.owns_id(id) is False:
1✔
1049
            error(f'Protocol {from_cls.LABEL} does not own id {id}')
1✔
1050
        elif from_cls.is_blocklisted(id, allow_internal=internal):
1✔
1051
            error(f'Activity {id} is blocklisted')
1✔
1052

1053
        # does this protocol support this activity/object type?
1054
        from_cls.check_supported(obj, 'receive')
1✔
1055

1056
        # lease this object, atomically
1057
        memcache_key = activity_id_memcache_key(id)
1✔
1058
        leased = memcache.memcache.add(
1✔
1059
            memcache_key, 'leased', noreply=False,
1060
            expire=int(MEMCACHE_LEASE_EXPIRATION.total_seconds()))
1061

1062
        # short circuit if we've already seen this activity id.
1063
        # (don't do this for bare objects since we need to check further down
1064
        # whether they've been updated since we saw them last.)
1065
        if (obj.as1.get('objectType') == 'activity'
1✔
1066
            and 'force' not in request.values
1067
            and (not leased
1068
                 or (obj.new is False and obj.changed is False))):
1069
            error(f'Already seen this activity {id}', status=204)
1✔
1070

1071
        pruned = {k: v for k, v in obj.as1.items()
1✔
1072
                  if k not in ('contentMap', 'replies', 'signature')}
1073
        delay = ''
1✔
1074
        if (received_at and request.headers.get('X-AppEngine-TaskRetryCount') == '0'
1✔
1075
                and obj.type != 'delete'):  # we delay deletes for 2m
1076
            delay_s = int((util.now().replace(tzinfo=None)
×
1077
                           - received_at.replace(tzinfo=None)
1078
                           ).total_seconds())
1079
            delay = f'({delay_s} s behind)'
×
1080
        logger.info(f'Receiving {from_cls.LABEL} {obj.type} {id} {delay} AS1: {json_dumps(pruned, indent=2)}')
1✔
1081

1082
        # check authorization
1083
        # https://www.w3.org/wiki/ActivityPub/Primer/Authentication_Authorization
1084
        actor = as1.get_owner(obj.as1)
1✔
1085
        if not actor:
1✔
1086
            error('Activity missing actor or author')
1✔
1087
        elif from_cls.owns_id(actor) is False:
1✔
1088
            error(f"{from_cls.LABEL} doesn't own actor {actor}, this is probably a bridged activity. Skipping.", status=204)
1✔
1089

1090
        assert authed_as
1✔
1091
        assert isinstance(authed_as, str)
1✔
1092
        authed_as = normalize_user_id(id=authed_as, proto=from_cls)
1✔
1093
        actor = normalize_user_id(id=actor, proto=from_cls)
1✔
1094
        if actor != authed_as:
1✔
1095
            report_error("Auth: receive: authed_as doesn't match owner",
1✔
1096
                         user=f'{id} authed_as {authed_as} owner {actor}')
1097
            error(f"actor {actor} isn't authed user {authed_as}")
1✔
1098

1099
        # update copy ids to originals
1100
        obj.normalize_ids()
1✔
1101
        obj.resolve_ids()
1✔
1102

1103
        if (obj.type == 'follow'
1✔
1104
                and Protocol.for_bridgy_subdomain(as1.get_object(obj.as1).get('id'))):
1105
            # follows of bot user; refresh user profile first
1106
            logger.info(f'Follow of bot user, reloading {actor}')
1✔
1107
            from_user = from_cls.get_or_create(id=actor, allow_opt_out=True)
1✔
1108
            from_user.reload_profile()
1✔
1109
        else:
1110
            # load actor user
1111
            from_user = from_cls.get_or_create(id=actor, allow_opt_out=internal)
1✔
1112

1113
        if not internal and (not from_user or from_user.manual_opt_out):
1✔
1114
            error(f"Couldn't load actor {actor}", status=204)
1✔
1115

1116
        # if this is an object, ie not an activity, wrap it in a create or update
1117
        obj = from_cls.handle_bare_object(obj, authed_as=authed_as)
1✔
1118
        obj.add('users', from_user.key)
1✔
1119

1120
        inner_obj_as1 = as1.get_object(obj.as1)
1✔
1121
        inner_obj_id = inner_obj_as1.get('id')
1✔
1122
        if obj.type in as1.CRUD_VERBS | as1.VERBS_WITH_OBJECT:
1✔
1123
            if not inner_obj_id:
1✔
1124
                error(f'{obj.type} object has no id!')
1✔
1125

1126
        # check age. we support backdated posts, but if they're over 2w old, we
1127
        # don't deliver them
1128
        if obj.type == 'post':
1✔
1129
            if published := inner_obj_as1.get('published'):
1✔
1130
                try:
1✔
1131
                    published_dt = util.parse_iso8601(published)
1✔
1132
                    if not published_dt.tzinfo:
1✔
1133
                        published_dt = published_dt.replace(tzinfo=timezone.utc)
×
1134
                    age = util.now() - published_dt
1✔
1135
                    if age > CREATE_MAX_AGE and 'force' not in request.values:
1✔
1136
                        error(f'Ignoring, too old, {age} is over {CREATE_MAX_AGE}',
×
1137
                              status=204)
1138
                except ValueError:  # from parse_iso8601
×
1139
                    logger.debug(f"Couldn't parse published {published}")
×
1140

1141
        # write Object to datastore
1142
        obj.source_protocol = from_cls.LABEL
1✔
1143
        if obj.type in STORE_AS1_TYPES:
1✔
1144
            obj.put()
1✔
1145

1146
        # store inner object
1147
        # TODO: unify with big obj.type conditional below. would have to merge
1148
        # this with the DM handling block lower down.
1149
        crud_obj = None
1✔
1150
        if obj.type in ('post', 'update') and inner_obj_as1.keys() > set(['id']):
1✔
1151
            crud_obj = Object.get_or_create(inner_obj_id, our_as1=inner_obj_as1,
1✔
1152
                                            source_protocol=from_cls.LABEL,
1153
                                            authed_as=actor, users=[from_user.key],
1154
                                            deleted=False)
1155

1156
        actor = as1.get_object(obj.as1, 'actor')
1✔
1157
        actor_id = actor.get('id')
1✔
1158

1159
        # handle activity!
1160
        if obj.type == 'stop-following':
1✔
1161
            # TODO: unify with handle_follow?
1162
            # TODO: handle multiple followees
1163
            if not actor_id or not inner_obj_id:
1✔
1164
                error(f'stop-following requires actor id and object id. Got: {actor_id} {inner_obj_id} {obj.as1}')
×
1165

1166
            # deactivate Follower
1167
            from_ = from_cls.key_for(actor_id)
1✔
1168
            if not (to_cls := Protocol.for_id(inner_obj_id)):
1✔
1169
                error(f"Can't determine protocol for {inner_obj_id} , giving up")
1✔
1170
            to = to_cls.key_for(inner_obj_id)
1✔
1171
            follower = Follower.query(Follower.to == to,
1✔
1172
                                      Follower.from_ == from_,
1173
                                      Follower.status == 'active').get()
1174
            if follower:
1✔
1175
                logger.info(f'Marking {follower} inactive')
1✔
1176
                follower.status = 'inactive'
1✔
1177
                follower.put()
1✔
1178
            else:
1179
                logger.warning(f'No Follower found for {from_} => {to}')
1✔
1180

1181
            # fall through to deliver to followee
1182
            # TODO: do we convert stop-following to webmention 410 of original
1183
            # follow?
1184

1185
            # fall through to deliver to followers
1186

1187
        elif obj.type in ('delete', 'undo'):
1✔
1188
            delete_obj_id = (from_user.profile_id()
1✔
1189
                            if inner_obj_id == from_user.key.id()
1190
                            else inner_obj_id)
1191

1192
            delete_obj = Object.get_by_id(delete_obj_id, authed_as=authed_as)
1✔
1193
            if not delete_obj:
1✔
1194
                logger.info(f"Ignoring, we don't have {delete_obj_id} stored")
1✔
1195
                return 'OK', 204
1✔
1196

1197
            # TODO: just delete altogether!
1198
            logger.info(f'Marking Object {delete_obj_id} deleted')
1✔
1199
            delete_obj.deleted = True
1✔
1200
            delete_obj.put()
1✔
1201

1202
            # if this is an actor, handle deleting it later so that
1203
            # in case it's from_user, user.enabled_protocols is still populated
1204
            #
1205
            # fall through to deliver to followers and delete copy if necessary.
1206
            # should happen via protocol-specific copy target and send of
1207
            # delete activity.
1208
            # https://github.com/snarfed/bridgy-fed/issues/63
1209

1210
        elif obj.type == 'block':
1✔
1211
            if proto := Protocol.for_bridgy_subdomain(inner_obj_id):
1✔
1212
                # blocking protocol bot user disables that protocol
1213
                from_user.delete(proto)
1✔
1214
                from_user.disable_protocol(proto)
1✔
1215
                return 'OK', 200
1✔
1216

1217
        elif obj.type == 'post':
1✔
1218
            # handle DMs to bot users
1219
            if as1.is_dm(obj.as1):
1✔
1220
                return dms.receive(from_user=from_user, obj=obj)
1✔
1221

1222
        # fetch actor if necessary
1223
        if (actor and actor.keys() == set(['id'])
1✔
1224
                and obj.type not in ('delete', 'undo')):
1225
            logger.debug('Fetching actor so we have name, profile photo, etc')
1✔
1226
            actor_obj = from_cls.load(ids.profile_id(id=actor['id'], proto=from_cls),
1✔
1227
                                      raise_=False)
1228
            if actor_obj and actor_obj.as1:
1✔
1229
                obj.our_as1 = {
1✔
1230
                    **obj.as1, 'actor': {
1231
                        **actor_obj.as1,
1232
                        # override profile id with actor id
1233
                        # https://github.com/snarfed/bridgy-fed/issues/1720
1234
                        'id': actor['id'],
1235
                    }
1236
                }
1237

1238
        # fetch object if necessary
1239
        if (obj.type in ('post', 'update', 'share')
1✔
1240
                and inner_obj_as1.keys() == set(['id'])
1241
                and from_cls.owns_id(inner_obj_id) is not False):
1242
            logger.debug('Fetching inner object')
1✔
1243
            inner_obj = from_cls.load(inner_obj_id, raise_=False,
1✔
1244
                                      remote=(obj.type in ('post', 'update')))
1245
            if obj.type in ('post', 'update'):
1✔
1246
                crud_obj = inner_obj
1✔
1247
            if inner_obj and inner_obj.as1:
1✔
1248
                obj.our_as1 = {
1✔
1249
                    **obj.as1,
1250
                    'object': {
1251
                        **inner_obj_as1,
1252
                        **inner_obj.as1,
1253
                    }
1254
                }
1255
            elif obj.type in ('post', 'update'):
1✔
1256
                error(f"Need object {inner_obj_id} but couldn't fetch, giving up")
1✔
1257

1258
        if obj.type == 'follow':
1✔
1259
            if proto := Protocol.for_bridgy_subdomain(inner_obj_id):
1✔
1260
                # follow of one of our protocol bot users; enable that protocol.
1261
                # fall through so that we send an accept.
1262
                try:
1✔
1263
                    from_user.enable_protocol(proto)
1✔
1264
                except ErrorButDoNotRetryTask:
1✔
1265
                    from web import Web
1✔
1266
                    bot = Web.get_by_id(proto.bot_user_id())
1✔
1267
                    from_cls.respond_to_follow('reject', follower=from_user,
1✔
1268
                                               followee=bot, follow=obj)
1269
                    raise
1✔
1270
                proto.bot_follow(from_user)
1✔
1271

1272
            from_cls.handle_follow(obj)
1✔
1273

1274
        # deliver to targets
1275
        resp = from_cls.deliver(obj, from_user=from_user, crud_obj=crud_obj)
1✔
1276

1277
        # if this is a user, deactivate its followers/followings
1278
        # https://github.com/snarfed/bridgy-fed/issues/1304
1279
        if obj.type == 'delete':
1✔
1280
            if user_key := from_cls.key_for(id=inner_obj_id):
1✔
1281
                if user := user_key.get():
1✔
1282
                    for proto in user.enabled_protocols:
1✔
1283
                        user.disable_protocol(PROTOCOLS[proto])
1✔
1284

1285
                    logger.info(f'Deactivating Followers from or to {user_key.id()}')
1✔
1286
                    followers = Follower.query(
1✔
1287
                        OR(Follower.to == user_key, Follower.from_ == user_key)
1288
                        ).fetch()
1289
                    for f in followers:
1✔
1290
                        f.status = 'inactive'
1✔
1291
                    ndb.put_multi(followers)
1✔
1292

1293
        memcache.memcache.set(memcache_key, 'done', expire=7 * 24 * 60 * 60)  # 1w
1✔
1294
        return resp
1✔
1295

1296
    @classmethod
1✔
1297
    def handle_follow(from_cls, obj):
1✔
1298
        """Handles an incoming follow activity.
1299

1300
        Sends an ``Accept`` back, but doesn't send the ``Follow`` itself. That
1301
        happens in :meth:`deliver`.
1302

1303
        Args:
1304
          obj (models.Object): follow activity
1305
        """
1306
        logger.debug('Got follow. Loading users, storing Follow(s), sending accept(s)')
1✔
1307

1308
        # Prepare follower (from) users' data
1309
        # TODO: remove all of this and just use from_user
1310
        from_as1 = as1.get_object(obj.as1, 'actor')
1✔
1311
        from_id = from_as1.get('id')
1✔
1312
        if not from_id:
1✔
1313
            error(f'Follow activity requires actor. Got: {obj.as1}')
×
1314

1315
        from_obj = from_cls.load(from_id, raise_=False)
1✔
1316
        if not from_obj:
1✔
1317
            error(f"Couldn't load {from_id}", status=502)
×
1318

1319
        if not from_obj.as1:
1✔
1320
            from_obj.our_as1 = from_as1
1✔
1321
            from_obj.put()
1✔
1322

1323
        from_key = from_cls.key_for(from_id)
1✔
1324
        if not from_key:
1✔
1325
            error(f'Invalid {from_cls.LABEL} user key: {from_id}')
×
1326
        obj.users = [from_key]
1✔
1327
        from_user = from_cls.get_or_create(id=from_key.id(), obj=from_obj)
1✔
1328

1329
        # Prepare followee (to) users' data
1330
        to_as1s = as1.get_objects(obj.as1)
1✔
1331
        if not to_as1s:
1✔
1332
            error(f'Follow activity requires object(s). Got: {obj.as1}')
×
1333

1334
        # Store Followers
1335
        for to_as1 in to_as1s:
1✔
1336
            to_id = to_as1.get('id')
1✔
1337
            if not to_id:
1✔
1338
                error(f'Follow activity requires object(s). Got: {obj.as1}')
×
1339

1340
            logger.info(f'Follow {from_id} => {to_id}')
1✔
1341

1342
            to_cls = Protocol.for_id(to_id)
1✔
1343
            if not to_cls:
1✔
1344
                error(f"Couldn't determine protocol for {to_id}")
×
1345
            elif from_cls == to_cls:
1✔
1346
                logger.info(f'Skipping same-protocol Follower {from_id} => {to_id}')
1✔
1347
                continue
1✔
1348

1349
            to_obj = to_cls.load(to_id)
1✔
1350
            if to_obj and not to_obj.as1:
1✔
1351
                to_obj.our_as1 = to_as1
1✔
1352
                to_obj.put()
1✔
1353

1354
            to_key = to_cls.key_for(to_id)
1✔
1355
            if not to_key:
1✔
1356
                logger.info(f'Skipping invalid {from_cls.LABEL} user key: {from_id}')
×
1357
                continue
×
1358

1359
            to_user = to_cls.get_or_create(id=to_key.id(), obj=to_obj,
1✔
1360
                                           allow_opt_out=True)
1361
            follower_obj = Follower.get_or_create(to=to_user, from_=from_user,
1✔
1362
                                                  follow=obj.key, status='active')
1363
            obj.add('notify', to_key)
1✔
1364
            from_cls.respond_to_follow('accept', follower=from_user,
1✔
1365
                                       followee=to_user, follow=obj)
1366

1367
    @classmethod
1✔
1368
    def respond_to_follow(_, verb, follower, followee, follow):
1✔
1369
        """Sends an accept or reject activity for a follow.
1370

1371
        ...if the follower's protocol supports accepts/rejects. Otherwise, does
1372
        nothing.
1373

1374
        Args:
1375
          verb (str): ``accept`` or  ``reject``
1376
          follower (models.User)
1377
          followee (models.User)
1378
          follow (models.Object)
1379
        """
1380
        assert verb in ('accept', 'reject')
1✔
1381
        if verb not in follower.SUPPORTED_AS1_TYPES:
1✔
1382
            return
1✔
1383

1384
        target = follower.target_for(follower.obj)
1✔
1385
        if not target:
1✔
1386
            error(f"Couldn't find delivery target for follower {follower.key.id()}")
×
1387

1388
        # send. note that this is one response for the whole follow, even if it
1389
        # has multiple followees!
1390
        id = f'{followee.key.id()}/followers#{verb}-{follow.key.id()}'
1✔
1391
        accept = {
1✔
1392
            'id': id,
1393
            'objectType': 'activity',
1394
            'verb': verb,
1395
            'actor': followee.key.id(),
1396
            'object': follow.as1,
1397
        }
1398
        common.create_task(queue='send', id=id, our_as1=accept, url=target,
1✔
1399
                           protocol=follower.LABEL, user=followee.key.urlsafe())
1400

1401
    @classmethod
1✔
1402
    def bot_follow(bot_cls, user):
1✔
1403
        """Follow a user from a protocol bot user.
1404

1405
        ...so that the protocol starts sending us their activities, if it needs
1406
        a follow for that (eg ActivityPub).
1407

1408
        Args:
1409
          user (User)
1410
        """
1411
        from web import Web
1✔
1412
        bot = Web.get_by_id(bot_cls.bot_user_id())
1✔
1413
        now = util.now().isoformat()
1✔
1414
        logger.info(f'Following {user.key.id()} back from bot user {bot.key.id()}')
1✔
1415

1416
        if not user.obj:
1✔
1417
            logger.info("  can't follow, user has no profile obj")
1✔
1418
            return
1✔
1419

1420
        target = user.target_for(user.obj)
1✔
1421
        follow_back_id = f'https://{bot.key.id()}/#follow-back-{user.key.id()}-{now}'
1✔
1422
        follow_back_as1 = {
1✔
1423
            'objectType': 'activity',
1424
            'verb': 'follow',
1425
            'id': follow_back_id,
1426
            'actor': bot.key.id(),
1427
            'object': user.key.id(),
1428
        }
1429
        common.create_task(queue='send', id=follow_back_id,
1✔
1430
                           our_as1=follow_back_as1, url=target,
1431
                           source_protocol='web', protocol=user.LABEL,
1432
                           user=bot.key.urlsafe())
1433

1434
    @classmethod
1✔
1435
    def handle_bare_object(cls, obj, authed_as=None):
1✔
1436
        """If obj is a bare object, wraps it in a create or update activity.
1437

1438
        Checks if we've seen it before.
1439

1440
        Args:
1441
          obj (models.Object)
1442
          authed_as (str): authenticated actor id who sent this activity
1443

1444
        Returns:
1445
          models.Object: ``obj`` if it's an activity, otherwise a new object
1446
        """
1447
        is_actor = obj.type in as1.ACTOR_TYPES
1✔
1448
        if not is_actor and obj.type not in ('note', 'article', 'comment'):
1✔
1449
            return obj
1✔
1450

1451
        obj_actor = ids.normalize_user_id(id=as1.get_owner(obj.as1), proto=cls)
1✔
1452
        now = util.now().isoformat()
1✔
1453

1454
        # occasionally we override the object, eg if this is a profile object
1455
        # coming in via a user with use_instead set
1456
        obj_as1 = obj.as1
1✔
1457
        if obj_id := obj.key.id():
1✔
1458
            if obj_as1_id := obj_as1.get('id'):
1✔
1459
                if obj_id != obj_as1_id:
1✔
1460
                    logger.info(f'Overriding AS1 object id {obj_as1_id} with Object id {obj_id}')
1✔
1461
                    obj_as1['id'] = obj_id
1✔
1462

1463
        # this is a raw post; wrap it in a create or update activity
1464
        if obj.changed or is_actor:
1✔
1465
            if obj.changed:
1✔
1466
                logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes')
1✔
1467
            else:
1468
                logger.info(f'Got actor profile object, wrapping in update')
1✔
1469
            id = f'{obj.key.id()}#bridgy-fed-update-{now}'
1✔
1470
            update_as1 = {
1✔
1471
                'objectType': 'activity',
1472
                'verb': 'update',
1473
                'id': id,
1474
                'actor': obj_actor,
1475
                'object': {
1476
                    # Mastodon requires the updated field for Updates, so
1477
                    # add a default value.
1478
                    # https://docs.joinmastodon.org/spec/activitypub/#supported-activities-for-statuses
1479
                    # https://socialhub.activitypub.rocks/t/what-could-be-the-reason-that-my-update-activity-does-not-work/2893/4
1480
                    # https://github.com/mastodon/documentation/pull/1150
1481
                    'updated': now,
1482
                    **obj_as1,
1483
                },
1484
            }
1485
            logger.debug(f'  AS1: {json_dumps(update_as1, indent=2)}')
1✔
1486
            return Object(id=id, our_as1=update_as1,
1✔
1487
                          source_protocol=obj.source_protocol)
1488

1489
        if (obj.new
1✔
1490
                # HACK: force query param here is specific to webmention
1491
                or 'force' in request.form):
1492
            create_id = f'{obj.key.id()}#bridgy-fed-create'
1✔
1493
            create_as1 = {
1✔
1494
                'objectType': 'activity',
1495
                'verb': 'post',
1496
                'id': create_id,
1497
                'actor': obj_actor,
1498
                'object': obj_as1,
1499
                'published': now,
1500
            }
1501
            logger.info(f'Wrapping in post')
1✔
1502
            logger.debug(f'  AS1: {json_dumps(create_as1, indent=2)}')
1✔
1503
            return Object(id=create_id, our_as1=create_as1,
1✔
1504
                          source_protocol=obj.source_protocol)
1505

1506
        error(f'{obj.key.id()} is unchanged, nothing to do', status=204)
1✔
1507

1508
    @classmethod
1✔
1509
    def deliver(from_cls, obj, from_user, crud_obj=None, to_proto=None):
1✔
1510
        """Delivers an activity to its external recipients.
1511

1512
        Args:
1513
          obj (models.Object): activity to deliver
1514
          from_user (models.User): user (actor) this activity is from
1515
          crud_obj (models.Object): if this is a create, update, or delete/undo
1516
            activity, the inner object that's being written, otherwise None.
1517
            (This object's ``notify`` and ``feed`` properties may be updated.)
1518
          to_proto (protocol.Protocol): optional; if provided, only deliver to
1519
            targets on this protocol
1520

1521
        Returns:
1522
          (str, int) tuple: Flask response
1523
        """
1524
        if to_proto:
1✔
1525
            logger.info(f'Only delivering to {to_proto.LABEL}')
1✔
1526

1527
        # find delivery targets. maps Target to Object or None
1528
        #
1529
        # ...then write the relevant object, since targets() has a side effect of
1530
        # setting the notify and feed properties (and dirty attribute)
1531
        targets = from_cls.targets(obj, from_user=from_user, crud_obj=crud_obj)
1✔
1532
        if to_proto:
1✔
1533
            targets = {t: obj for t, obj in targets.items()
1✔
1534
                       if t.protocol == to_proto.LABEL}
1535
        if not targets:
1✔
1536
            return r'No targets, nothing to do ¯\_(ツ)_/¯', 204
1✔
1537

1538
        # store object that targets() updated
1539
        if crud_obj and crud_obj.dirty:
1✔
1540
            crud_obj.put()
1✔
1541
        elif obj.type in STORE_AS1_TYPES and obj.dirty:
1✔
1542
            obj.put()
1✔
1543

1544
        obj_params = ({'obj_id': obj.key.id()} if obj.type in STORE_AS1_TYPES
1✔
1545
                      else obj.to_request())
1546

1547
        # sort targets so order is deterministic for tests, debugging, etc
1548
        sorted_targets = sorted(targets.items(), key=lambda t: t[0].uri)
1✔
1549

1550
        # enqueue send task for each targets
1551
        logger.info(f'Delivering to: {[t for t, _ in sorted_targets]}')
1✔
1552
        user = from_user.key.urlsafe()
1✔
1553
        for i, (target, orig_obj) in enumerate(sorted_targets):
1✔
1554
            orig_obj_id = orig_obj.key.id() if orig_obj else None
1✔
1555
            common.create_task(queue='send', url=target.uri, protocol=target.protocol,
1✔
1556
                               orig_obj_id=orig_obj_id, user=user, **obj_params)
1557

1558
        return 'OK', 202
1✔
1559

1560
    @classmethod
1✔
1561
    def targets(from_cls, obj, from_user, crud_obj=None, internal=False):
1✔
1562
        """Collects the targets to send a :class:`models.Object` to.
1563

1564
        Targets are both objects - original posts, events, etc - and actors.
1565

1566
        Args:
1567
          obj (models.Object)
1568
          from_user (User)
1569
          crud_obj (models.Object): if this is a create, update, or delete/undo
1570
            activity, the inner object that's being written, otherwise None.
1571
            (This object's ``notify`` and ``feed`` properties may be updated.)
1572
          internal (bool): whether this is a recursive internal call
1573

1574
        Returns:
1575
          dict: maps :class:`models.Target` to original (in response to)
1576
          :class:`models.Object`, if any, otherwise None
1577
        """
1578
        logger.debug('Finding recipients and their targets')
1✔
1579

1580
        # we should only have crud_obj iff this is a create or update
1581
        assert (crud_obj is not None) == (obj.type in ('post', 'update')), obj.type
1✔
1582
        write_obj = crud_obj or obj
1✔
1583
        write_obj.dirty = False
1✔
1584

1585
        target_uris = as1.targets(obj.as1)
1✔
1586
        orig_obj = None
1✔
1587
        targets = {}  # maps Target to Object or None
1✔
1588
        owner = as1.get_owner(obj.as1)
1✔
1589
        allow_opt_out = (obj.type == 'delete')
1✔
1590
        inner_obj_as1 = as1.get_object(obj.as1)
1✔
1591
        inner_obj_id = inner_obj_as1.get('id')
1✔
1592
        in_reply_tos = as1.get_ids(inner_obj_as1, 'inReplyTo')
1✔
1593
        quoted_posts = as1.quoted_posts(inner_obj_as1)
1✔
1594
        mentioned_urls = as1.mentions(inner_obj_as1)
1✔
1595
        is_reply = obj.type == 'comment' or in_reply_tos
1✔
1596
        is_self_reply = False
1✔
1597

1598
        original_ids = []
1✔
1599
        if is_reply:
1✔
1600
            original_ids = in_reply_tos
1✔
1601
        elif inner_obj_id:
1✔
1602
            if inner_obj_id == from_user.key.id():
1✔
1603
                inner_obj_id = from_user.profile_id()
1✔
1604
            original_ids = [inner_obj_id]
1✔
1605

1606
        original_objs = {}
1✔
1607
        for id in original_ids:
1✔
1608
            if proto := Protocol.for_id(id):
1✔
1609
                original_objs[id] = proto.load(id, raise_=False)
1✔
1610

1611
        # for AP, add in-reply-tos' mentions
1612
        # https://github.com/snarfed/bridgy-fed/issues/1608
1613
        # https://github.com/snarfed/bridgy-fed/issues/1218
1614
        orig_post_mentions = {}  # maps mentioned id to original post Object
1✔
1615
        for id in in_reply_tos:
1✔
1616
            if ((in_reply_to_obj := original_objs.get(id))
1✔
1617
                    and (proto := PROTOCOLS.get(in_reply_to_obj.source_protocol))
1618
                    and proto.SEND_REPLIES_TO_ORIG_POSTS_MENTIONS
1619
                    and (mentions := as1.mentions(in_reply_to_obj.as1))):
1620
                logger.info(f"Adding in-reply-to {id} 's mentions to targets: {mentions}")
1✔
1621
                target_uris.extend(mentions)
1✔
1622
                for mention in mentions:
1✔
1623
                    orig_post_mentions[mention] = in_reply_to_obj
1✔
1624

1625
        target_uris = sorted(set(target_uris))
1✔
1626
        logger.info(f'Raw targets: {target_uris}')
1✔
1627

1628
        # which protocols should we allow delivering to?
1629
        to_protocols = []
1✔
1630
        for label in (list(from_user.DEFAULT_ENABLED_PROTOCOLS)
1✔
1631
                      + from_user.enabled_protocols):
1632
            if not (proto := PROTOCOLS.get(label)):
1✔
1633
                report_error(f'unknown enabled protocol {label} for {from_user.key.id()}')
1✔
1634
                continue
1✔
1635

1636
            if proto.HAS_COPIES and (obj.type in ('update', 'delete', 'share', 'undo')
1✔
1637
                                     or is_reply):
1638
                origs_could_bridge = None
1✔
1639

1640
                for id in original_ids:
1✔
1641
                    if not (orig := original_objs.get(id)):
1✔
1642
                        continue
1✔
1643
                    elif isinstance(orig, proto):
1✔
1644
                        logger.info(f'Allowing {label} for original post {id}')
×
1645
                        break
×
1646
                    elif orig.get_copy(proto):
1✔
1647
                        logger.info(f'Allowing {label}, original post {id} was bridged there')
1✔
1648
                        break
1✔
1649

1650
                    if (origs_could_bridge is not False
1✔
1651
                            and (orig_author_id := as1.get_owner(orig.as1))
1652
                            and (orig_proto := PROTOCOLS.get(orig.source_protocol))
1653
                            and (orig_author := orig_proto.get_by_id(orig_author_id))):
1654
                        origs_could_bridge = orig_author.is_enabled(proto)
1✔
1655

1656
                else:
1657
                    msg = f"original object(s) {original_ids} weren't bridged to {label}"
1✔
1658
                    if (proto.LABEL not in from_user.DEFAULT_ENABLED_PROTOCOLS
1✔
1659
                            and origs_could_bridge):
1660
                        # retry later; original obj may still be bridging
1661
                        # TODO: limit to brief window, eg no older than 2h? 1d?
1662
                        error(msg, status=304)
1✔
1663

1664
                    logger.info(msg)
1✔
1665
                    continue
1✔
1666

1667

1668
            util.add(to_protocols, proto)
1✔
1669

1670
        # process direct targets
1671
        for target_id in target_uris:
1✔
1672
            target_proto = Protocol.for_id(target_id)
1✔
1673
            if not target_proto:
1✔
1674
                logger.info(f"Can't determine protocol for {target_id}")
1✔
1675
                continue
1✔
1676
            elif target_proto.is_blocklisted(target_id):
1✔
1677
                logger.debug(f'{target_id} is blocklisted')
1✔
1678
                continue
1✔
1679

1680
            orig_obj = target_proto.load(target_id, raise_=False)
1✔
1681
            if not orig_obj or not orig_obj.as1:
1✔
1682
                logger.info(f"Couldn't load {target_id}")
1✔
1683
                continue
1✔
1684

1685
            target_author_key = (target_proto(id=target_id).key
1✔
1686
                                 if target_id in mentioned_urls
1687
                                 else target_proto.actor_key(orig_obj))
1688
            if not from_user.is_enabled(target_proto):
1✔
1689
                # if author isn't bridged and target user is, DM a prompt and
1690
                # add a notif for the target user
1691
                if (target_id in (in_reply_tos + quoted_posts + mentioned_urls)
1✔
1692
                        and target_author_key):
1693
                    if target_author := target_author_key.get():
1✔
1694
                        if target_author.is_enabled(from_cls):
1✔
1695
                            notifications.add_notification(target_author, write_obj)
1✔
1696
                            verb, noun = (
1✔
1697
                                ('replied to', 'replies') if target_id in in_reply_tos
1698
                                else ('quoted', 'quotes') if target_id in quoted_posts
1699
                                else ('mentioned', 'mentions'))
1700
                            dms.maybe_send(from_=target_proto, to_user=from_user,
1✔
1701
                                           type='replied_to_bridged_user', text=f"""\
1702
Hi! You <a href="{inner_obj_as1.get('url') or inner_obj_id}">recently {verb}</a> {target_author.user_link()}, who's bridged here from {target_proto.PHRASE}. If you want them to see your {noun}, you can bridge your account into {target_proto.PHRASE} by following this account. <a href="https://fed.brid.gy/docs">See the docs</a> for more information.""")
1703

1704
                continue
1✔
1705

1706
            # deliver self-replies to followers
1707
            # https://github.com/snarfed/bridgy-fed/issues/639
1708
            if target_id in in_reply_tos and owner == as1.get_owner(orig_obj.as1):
1✔
1709
                is_self_reply = True
1✔
1710
                logger.info(f'self reply!')
1✔
1711

1712
            # also add copies' targets
1713
            for copy in orig_obj.copies:
1✔
1714
                proto = PROTOCOLS[copy.protocol]
1✔
1715
                if proto in to_protocols:
1✔
1716
                    # copies generally won't have their own Objects
1717
                    if target := proto.target_for(Object(id=copy.uri)):
1✔
1718
                        logger.debug(f'Adding target {target} for copy {copy.uri} of original {target_id}')
1✔
1719
                        targets[Target(protocol=copy.protocol, uri=target)] = orig_obj
1✔
1720

1721
            if target_proto == from_cls:
1✔
1722
                logger.debug(f'Skipping same-protocol target {target_id}')
1✔
1723
                continue
1✔
1724

1725
            target = target_proto.target_for(orig_obj)
1✔
1726
            if not target:
1✔
1727
                # TODO: surface errors like this somehow?
1728
                logger.error(f"Can't find delivery target for {target_id}")
×
1729
                continue
×
1730

1731
            logger.debug(f'Target for {target_id} is {target}')
1✔
1732
            # only use orig_obj for inReplyTos, like/repost objects, reply's original
1733
            # post's mentions, etc
1734
            # https://github.com/snarfed/bridgy-fed/issues/1237
1735
            target_obj = None
1✔
1736
            if target_id in in_reply_tos + as1.get_ids(obj.as1, 'object'):
1✔
1737
                target_obj = orig_obj
1✔
1738
            elif target_id in orig_post_mentions:
1✔
1739
                target_obj = orig_post_mentions[target_id]
1✔
1740
            targets[Target(protocol=target_proto.LABEL, uri=target)] = target_obj
1✔
1741

1742
            if target_author_key:
1✔
1743
                logger.debug(f'Recipient is {target_author_key}')
1✔
1744
                if write_obj.add('notify', target_author_key):
1✔
1745
                    write_obj.dirty = True
1✔
1746

1747
        if obj.type == 'undo':
1✔
1748
            logger.debug('Object is an undo; adding targets for inner object')
1✔
1749
            if set(inner_obj_as1.keys()) == {'id'}:
1✔
1750
                inner_obj = from_cls.load(inner_obj_id, raise_=False)
1✔
1751
            else:
1752
                inner_obj = Object(id=inner_obj_id, our_as1=inner_obj_as1)
1✔
1753
            if inner_obj:
1✔
1754
                targets.update(from_cls.targets(inner_obj, from_user=from_user,
1✔
1755
                                                internal=True))
1756

1757
        logger.info(f'Direct targets: {[t.uri for t in targets.keys()]}')
1✔
1758

1759
        # deliver to followers, if appropriate
1760
        user_key = from_cls.actor_key(obj, allow_opt_out=allow_opt_out)
1✔
1761
        if not user_key:
1✔
1762
            logger.info("Can't tell who this is from! Skipping followers.")
1✔
1763
            return targets
1✔
1764

1765
        followers = []
1✔
1766
        if (obj.type in ('post', 'update', 'delete', 'move', 'share', 'undo')
1✔
1767
                and (not is_reply or is_self_reply)):
1768
            logger.info(f'Delivering to followers of {user_key}')
1✔
1769
            followers = []
1✔
1770
            for f in Follower.query(Follower.to == user_key,
1✔
1771
                                    Follower.status == 'active'):
1772
                proto = PROTOCOLS_BY_KIND[f.from_.kind()]
1✔
1773
                # skip protocol bot users
1774
                if (not Protocol.for_bridgy_subdomain(f.from_.id())
1✔
1775
                        # skip protocols this user hasn't enabled, or where the base
1776
                        # object of this activity hasn't been bridged
1777
                        and proto in to_protocols
1778
                        # we deliver to HAS_COPIES protocols separately, below. we
1779
                        # assume they have follower-independent targets.
1780
                        and not (proto.HAS_COPIES and proto.DEFAULT_TARGET)):
1781
                    followers.append(f)
1✔
1782

1783
            user_keys = [f.from_ for f in followers]
1✔
1784
            users = [u for u in ndb.get_multi(user_keys) if u]
1✔
1785
            User.load_multi(users)
1✔
1786

1787
            if (not followers and
1✔
1788
                (util.domain_or_parent_in(from_user.key.id(), LIMITED_DOMAINS)
1789
                 or util.domain_or_parent_in(obj.key.id(), LIMITED_DOMAINS))):
1790
                logger.info(f'skipping, {from_user.key.id()} is on a limited domain and has no followers')
1✔
1791
                return {}
1✔
1792

1793
            # add to followers' feeds, if any
1794
            if not internal and obj.type in ('post', 'update', 'share'):
1✔
1795
                if write_obj.type not in as1.ACTOR_TYPES:
1✔
1796
                    write_obj.feed = [u.key for u in users if u.USES_OBJECT_FEED]
1✔
1797
                    if write_obj.feed:
1✔
1798
                        write_obj.dirty = True
1✔
1799

1800
            # collect targets for followers
1801
            for user in users:
1✔
1802
                # TODO: should we pass remote=False through here to Protocol.load?
1803
                target = user.target_for(user.obj, shared=True) if user.obj else None
1✔
1804
                if not target:
1✔
1805
                    # logger.error(f'Follower {user.key} has no delivery target')
1806
                    continue
1✔
1807

1808
                # normalize URL (lower case hostname, etc)
1809
                # ...but preserve our PDS URL without trailing slash in path
1810
                # https://atproto.com/specs/did#did-documents
1811
                target = util.dedupe_urls([target], trailing_slash=False)[0]
1✔
1812

1813
                targets[Target(protocol=user.LABEL, uri=target)] = \
1✔
1814
                    Object.get_by_id(inner_obj_id) if obj.type == 'share' else None
1815

1816
        # deliver to enabled HAS_COPIES protocols proactively
1817
        if obj.type in ('post', 'update', 'delete', 'share'):
1✔
1818
            for proto in to_protocols:
1✔
1819
                if proto.HAS_COPIES and proto.DEFAULT_TARGET:
1✔
1820
                    logger.info(f'user has {proto.LABEL} enabled, adding {proto.DEFAULT_TARGET}')
1✔
1821
                    targets.setdefault(
1✔
1822
                        Target(protocol=proto.LABEL, uri=proto.DEFAULT_TARGET), None)
1823

1824
        # de-dupe targets, discard same-domain
1825
        # maps string target URL to (Target, Object) tuple
1826
        candidates = {t.uri: (t, obj) for t, obj in targets.items()}
1✔
1827
        # maps Target to Object or None
1828
        targets = {}
1✔
1829
        source_domains = [
1✔
1830
            util.domain_from_link(url) for url in
1831
            (obj.as1.get('id'), obj.as1.get('url'), as1.get_owner(obj.as1))
1832
            if util.is_web(url)
1833
        ]
1834
        for url in sorted(util.dedupe_urls(
1✔
1835
                candidates.keys(),
1836
                # preserve our PDS URL without trailing slash in path
1837
                # https://atproto.com/specs/did#did-documents
1838
                trailing_slash=False)):
1839
            if util.is_web(url) and util.domain_from_link(url) in source_domains:
1✔
1840
                logger.info(f'Skipping same-domain target {url}')
×
1841
                continue
×
1842
            target, obj = candidates[url]
1✔
1843
            targets[target] = obj
1✔
1844

1845
        return targets
1✔
1846

1847
    @classmethod
1✔
1848
    def load(cls, id, remote=None, local=True, raise_=True, **kwargs):
1✔
1849
        """Loads and returns an Object from datastore or HTTP fetch.
1850

1851
        Sets the :attr:`new` and :attr:`changed` attributes if we know either
1852
        one for the loaded object, ie local is True and remote is True or None.
1853

1854
        Args:
1855
          id (str)
1856
          remote (bool): whether to fetch the object over the network. If True,
1857
            fetches even if we already have the object stored, and updates our
1858
            stored copy. If False and we don't have the object stored, returns
1859
            None. Default (None) means to fetch over the network only if we
1860
            don't already have it stored.
1861
          local (bool): whether to load from the datastore before
1862
            fetching over the network. If False, still stores back to the
1863
            datastore after a successful remote fetch.
1864
          raise_ (bool): if False, catches any :class:`request.RequestException`
1865
            or :class:`HTTPException` raised by :meth:`fetch()` and returns
1866
            ``None`` instead
1867
          kwargs: passed through to :meth:`fetch()`
1868

1869
        Returns:
1870
          models.Object: loaded object, or None if it isn't fetchable, eg a
1871
          non-URL string for Web, or ``remote`` is False and it isn't in the
1872
          datastore
1873

1874
        Raises:
1875
          requests.HTTPError: anything that :meth:`fetch` raises, if ``raise_``
1876
            is True
1877
        """
1878
        assert id
1✔
1879
        assert local or remote is not False
1✔
1880
        # logger.debug(f'Loading Object {id} local={local} remote={remote}')
1881

1882
        obj = orig_as1 = None
1✔
1883
        if local:
1✔
1884
            obj = Object.get_by_id(id)
1✔
1885
            if not obj:
1✔
1886
                # logger.debug(f' {id} not in datastore')
1887
                pass
1✔
1888
            elif obj.as1 or obj.raw or obj.deleted:
1✔
1889
                # logger.debug(f'  {id} got from datastore')
1890
                obj.new = False
1✔
1891

1892
        if remote is False:
1✔
1893
            return obj
1✔
1894
        elif remote is None and obj:
1✔
1895
            if obj.updated < util.as_utc(util.now() - OBJECT_REFRESH_AGE):
1✔
1896
                # logger.debug(f'  last updated {obj.updated}, refreshing')
1897
                pass
1✔
1898
            else:
1899
                return obj
1✔
1900

1901
        if obj:
1✔
1902
            orig_as1 = obj.as1
1✔
1903
            obj.our_as1 = None
1✔
1904
            obj.new = False
1✔
1905
        else:
1906
            obj = Object(id=id)
1✔
1907
            if local:
1✔
1908
                # logger.debug(f'  {id} not in datastore')
1909
                obj.new = True
1✔
1910
                obj.changed = False
1✔
1911

1912
        try:
1✔
1913
            fetched = cls.fetch(obj, **kwargs)
1✔
1914
        except (RequestException, HTTPException) as e:
1✔
1915
            if raise_:
1✔
1916
                raise
1✔
1917
            util.interpret_http_exception(e)
1✔
1918
            return None
1✔
1919

1920
        if not fetched:
1✔
1921
            return None
1✔
1922

1923
        # https://stackoverflow.com/a/3042250/186123
1924
        size = len(_entity_to_protobuf(obj)._pb.SerializeToString())
1✔
1925
        if size > models.MAX_ENTITY_SIZE:
1✔
1926
            logger.warning(f'Object is too big! {size} bytes is over {models.MAX_ENTITY_SIZE}')
1✔
1927
            return None
1✔
1928

1929
        obj.resolve_ids()
1✔
1930
        obj.normalize_ids()
1✔
1931

1932
        if obj.new is False:
1✔
1933
            obj.changed = obj.activity_changed(orig_as1)
1✔
1934

1935
        if obj.source_protocol not in (cls.LABEL, cls.ABBREV):
1✔
1936
            if obj.source_protocol:
1✔
1937
                logger.warning(f'Object {obj.key.id()} changed protocol from {obj.source_protocol} to {cls.LABEL} ?!')
×
1938
            obj.source_protocol = cls.LABEL
1✔
1939

1940
        obj.put()
1✔
1941
        return obj
1✔
1942

1943
    @classmethod
1✔
1944
    def check_supported(cls, obj, direction):
1✔
1945
        """If this protocol doesn't support this activity, raises HTTP 204.
1946

1947
        Also reports an error.
1948

1949
        (This logic is duplicated in some protocols, eg ActivityPub, so that
1950
        they can short circuit out early. It generally uses their native formats
1951
        instead of AS1, before an :class:`models.Object` is created.)
1952

1953
        Args:
1954
          obj (Object)
1955
          direction (str): ``'receive'`` or  ``'send'``
1956

1957
        Raises:
1958
          werkzeug.HTTPException: if this protocol doesn't support this object
1959
        """
1960
        assert direction in ('receive', 'send')
1✔
1961
        if not obj.type:
1✔
1962
            return
×
1963

1964
        inner = as1.get_object(obj.as1)
1✔
1965
        inner_type = as1.object_type(inner) or ''
1✔
1966
        if (obj.type not in cls.SUPPORTED_AS1_TYPES
1✔
1967
            or (obj.type in as1.CRUD_VERBS
1968
                and inner_type
1969
                and inner_type not in cls.SUPPORTED_AS1_TYPES)):
1970
            error(f"Bridgy Fed for {cls.LABEL} doesn't support {obj.type} {inner_type} yet", status=204)
1✔
1971

1972
        # don't allow posts with blank content and no image/video/audio
1973
        crud_obj = (as1.get_object(obj.as1) if obj.type in ('post', 'update')
1✔
1974
                    else obj.as1)
1975
        if (crud_obj.get('objectType') in as1.POST_TYPES
1✔
1976
                and not util.get_url(crud_obj, key='image')
1977
                and not any(util.get_urls(crud_obj, 'attachments', inner_key='stream'))
1978
                # TODO: handle articles with displayName but not content
1979
                and not source.html_to_text(crud_obj.get('content')).strip()):
1980
            error('Blank content and no image or video or audio', status=204)
1✔
1981

1982
        # receiving DMs is only allowed to protocol bot accounts
1983
        if direction == 'receive':
1✔
1984
            if recip := as1.recipient_if_dm(obj.as1):
1✔
1985
                owner = as1.get_owner(obj.as1)
1✔
1986
                if (not cls.SUPPORTS_DMS or (recip not in common.bot_user_ids()
1✔
1987
                                             and owner not in common.bot_user_ids())):
1988
                    # reply and say DMs aren't supported
1989
                    from_proto = PROTOCOLS.get(obj.source_protocol)
1✔
1990
                    to_proto = Protocol.for_id(recip)
1✔
1991
                    if owner and from_proto and to_proto:
1✔
1992
                        if ((from_user := from_proto.get_or_create(id=owner))
1✔
1993
                                and (to_user := to_proto.get_or_create(id=recip))):
1994
                            in_reply_to = (inner.get('id') if obj.type == 'post'
1✔
1995
                                           else obj.as1.get('id'))
1996
                            text = f"Hi! Sorry, this account is bridged from {to_user.PHRASE}, so it doesn't support DMs. Try getting in touch another way!"
1✔
1997
                            type = f'dms_not_supported-{to_user.key.id()}'
1✔
1998
                            dms.maybe_send(from_=to_user, to_user=from_user,
1✔
1999
                                           text=text, type=type,
2000
                                           in_reply_to=in_reply_to)
2001

2002
                    error("Bridgy Fed doesn't support DMs", status=204)
1✔
2003

2004
            # check that this activity is public. only do this for some activities,
2005
            # not eg likes or follows, since Mastodon doesn't currently mark those
2006
            # as explicitly public.
2007
            elif (obj.type in set(('post', 'update')) | as1.POST_TYPES | as1.ACTOR_TYPES
1✔
2008
                      and not as1.is_public(obj.as1, unlisted=False)):
2009
                  error('Bridgy Fed only supports public activities', status=204)
1✔
2010

2011

2012
@cloud_tasks_only(log=None)
1✔
2013
def receive_task():
1✔
2014
    """Task handler for a newly received :class:`models.Object`.
2015

2016
    Calls :meth:`Protocol.receive` with the form parameters.
2017

2018
    Parameters:
2019
      authed_as (str): passed to :meth:`Protocol.receive`
2020
      obj_id (str): key id of :class:`models.Object` to handle
2021
      received_at (str, ISO 8601 timestamp): when we first saw (received)
2022
        this activity
2023
      *: If ``obj_id`` is unset, all other parameters are properties for a new
2024
        :class:`models.Object` to handle
2025

2026
    TODO: migrate incoming webmentions to this. See how we did it for AP. The
2027
    difficulty is that parts of :meth:`protocol.Protocol.receive` depend on
2028
    setup in :func:`web.webmention`, eg :class:`models.Object` with ``new`` and
2029
    ``changed``, HTTP request details, etc. See stash for attempt at this for
2030
    :class:`web.Web`.
2031
    """
2032
    common.log_request()
1✔
2033
    form = request.form.to_dict()
1✔
2034

2035
    authed_as = form.pop('authed_as', None)
1✔
2036
    internal = (authed_as == common.PRIMARY_DOMAIN
1✔
2037
                or authed_as in common.PROTOCOL_DOMAINS)
2038

2039
    obj = Object.from_request()
1✔
2040
    assert obj
1✔
2041
    assert obj.source_protocol
1✔
2042
    obj.new = True
1✔
2043

2044
    if received_at := form.pop('received_at', None):
1✔
2045
        received_at = datetime.fromisoformat(received_at)
1✔
2046

2047
    try:
1✔
2048
        return PROTOCOLS[obj.source_protocol].receive(
1✔
2049
            obj=obj, authed_as=authed_as, internal=internal, received_at=received_at)
2050
    except RequestException as e:
1✔
2051
        util.interpret_http_exception(e)
1✔
2052
        error(e, status=304)
1✔
2053
    except ValueError as e:
1✔
UNCOV
2054
        logger.warning(e, exc_info=True)
×
2055
        error(e, status=304)
×
2056

2057

2058
@cloud_tasks_only(log=None)
1✔
2059
def send_task():
1✔
2060
    """Task handler for sending an activity to a single specific destination.
2061

2062
    Calls :meth:`Protocol.send` with the form parameters.
2063

2064
    Parameters:
2065
      protocol (str): :class:`Protocol` to send to
2066
      url (str): destination URL to send to
2067
      obj_id (str): key id of :class:`models.Object` to send
2068
      orig_obj_id (str): optional, :class:`models.Object` key id of the
2069
        "original object" that this object refers to, eg replies to or reposts
2070
        or likes
2071
      user (url-safe google.cloud.ndb.key.Key): :class:`models.User` (actor)
2072
        this activity is from
2073
      *: If ``obj_id`` is unset, all other parameters are properties for a new
2074
        :class:`models.Object` to handle
2075
    """
2076
    common.log_request()
1✔
2077

2078
    # prepare
2079
    form = request.form.to_dict()
1✔
2080
    url = form.get('url')
1✔
2081
    protocol = form.get('protocol')
1✔
2082
    if not url or not protocol:
1✔
2083
        logger.warning(f'Missing protocol or url; got {protocol} {url}')
1✔
2084
        return '', 204
1✔
2085

2086
    target = Target(uri=url, protocol=protocol)
1✔
2087
    obj = Object.from_request()
1✔
2088
    assert obj and obj.key and obj.key.id()
1✔
2089

2090
    PROTOCOLS[protocol].check_supported(obj, 'send')
1✔
2091
    allow_opt_out = (obj.type == 'delete')
1✔
2092

2093
    user = None
1✔
2094
    if user_key := form.get('user'):
1✔
2095
        key = ndb.Key(urlsafe=user_key)
1✔
2096
        # use get_by_id so that we follow use_instead
2097
        user = PROTOCOLS_BY_KIND[key.kind()].get_by_id(
1✔
2098
            key.id(), allow_opt_out=allow_opt_out)
2099

2100
    # send
2101
    delay = ''
1✔
2102
    if request.headers.get('X-AppEngine-TaskRetryCount') == '0' and obj.created:
1✔
2103
        delay_s = int((util.now().replace(tzinfo=None) - obj.created).total_seconds())
1✔
2104
        delay = f'({delay_s} s behind)'
1✔
2105
    logger.info(f'Sending {obj.source_protocol} {obj.type} {obj.key.id()} to {protocol} {url} {delay}')
1✔
2106
    logger.debug(f'  AS1: {json_dumps(obj.as1, indent=2)}')
1✔
2107
    sent = None
1✔
2108
    try:
1✔
2109
        sent = PROTOCOLS[protocol].send(obj, url, from_user=user,
1✔
2110
                                        orig_obj_id=form.get('orig_obj_id'))
2111
    except BaseException as e:
1✔
2112
        code, body = util.interpret_http_exception(e)
1✔
2113
        if not code and not body:
1✔
2114
            raise
1✔
2115

2116
    if sent is False:
1✔
2117
        logger.info(f'Failed sending!')
1✔
2118

2119
    return '', 200 if sent else 204 if sent is False else 304
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc