• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

snarfed / bridgy-fed / 39608619-b71e-4ac9-922f-ba319d83563d

29 Oct 2025 07:21PM UTC coverage: 92.878% (-0.001%) from 92.879%
39608619-b71e-4ac9-922f-ba319d83563d

push

circleci

snarfed
move use_instead profile update handling from Protocol.handle_bare_object to receive

5 of 5 new or added lines in 1 file covered. (100.0%)

18 existing lines in 1 file now uncovered.

5999 of 6459 relevant lines covered (92.88%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.57
/protocol.py
1
"""Base protocol class and common code."""
2
import copy
1✔
3
from datetime import datetime, timedelta, timezone
1✔
4
import logging
1✔
5
import os
1✔
6
import re
1✔
7
from threading import Lock
1✔
8
from urllib.parse import urljoin, urlparse
1✔
9

10
from cachetools import cached, LRUCache
1✔
11
from flask import request
1✔
12
from google.cloud import ndb
1✔
13
from google.cloud.ndb import OR
1✔
14
from google.cloud.ndb.model import _entity_to_protobuf
1✔
15
from granary import as1, as2, source
1✔
16
from granary.source import html_to_text
1✔
17
from oauth_dropins.webutil.appengine_info import DEBUG
1✔
18
from oauth_dropins.webutil.flask_util import cloud_tasks_only
1✔
19
from oauth_dropins.webutil import models
1✔
20
from oauth_dropins.webutil import util
1✔
21
from oauth_dropins.webutil.util import json_dumps, json_loads
1✔
22
from requests import RequestException
1✔
23
import werkzeug.exceptions
1✔
24
from werkzeug.exceptions import BadGateway, HTTPException
1✔
25

26
import common
1✔
27
from common import (
1✔
28
    DOMAIN_BLOCKLIST,
29
    DOMAIN_RE,
30
    DOMAINS,
31
    ErrorButDoNotRetryTask,
32
    PRIMARY_DOMAIN,
33
    PROTOCOL_DOMAINS,
34
    report_error,
35
    subdomain_wrap,
36
)
37
import dms
1✔
38
import ids
1✔
39
import memcache
1✔
40
from models import (
1✔
41
    DM,
42
    Follower,
43
    Object,
44
    PROTOCOLS,
45
    PROTOCOLS_BY_KIND,
46
    Target,
47
    User,
48
)
49
import notifications
1✔
50

51
OBJECT_REFRESH_AGE = timedelta(days=30)
1✔
52
DELETE_TASK_DELAY = timedelta(minutes=1)
1✔
53
CREATE_MAX_AGE = timedelta(weeks=2)
1✔
54
# WARNING: keep this below the receive queue's min_backoff_seconds in queue.yaml!
55
MEMCACHE_LEASE_EXPIRATION = timedelta(seconds=25)
1✔
56

57
# require a follow for users on these domains before we deliver anything from
58
# them other than their profile
59
LIMITED_DOMAINS = (os.getenv('LIMITED_DOMAINS', '').split()
1✔
60
                   or util.load_file_lines('limited_domains'))
61

62
# domains to allow non-public activities from
63
NON_PUBLIC_DOMAINS = (
1✔
64
    # bridged from twitter (X). bird.makeup federates tweets as followers-only, but
65
    # they're public on twitter itself
66
    'bird.makeup',
67
)
68

69
DONT_STORE_AS1_TYPES = as1.CRUD_VERBS | set((
1✔
70
    'accept',
71
    'reject',
72
    'stop-following',
73
    'undo',
74
))
75
STORE_AS1_TYPES = (as1.ACTOR_TYPES | as1.POST_TYPES | as1.VERBS_WITH_OBJECT
1✔
76
                   - DONT_STORE_AS1_TYPES)
77

78
logger = logging.getLogger(__name__)
1✔
79

80

81
def error(*args, status=299, **kwargs):
1✔
82
    """Default HTTP status code to 299 to prevent retrying task."""
83
    return common.error(*args, status=status, **kwargs)
1✔
84

85

86
def activity_id_memcache_key(id):
1✔
87
    return memcache.key(f'receive-{id}')
1✔
88

89

90
class Protocol:
1✔
91
    """Base protocol class. Not to be instantiated; classmethods only."""
92
    ABBREV = None
1✔
93
    """str: lower case abbreviation, used in URL paths"""
1✔
94
    PHRASE = None
1✔
95
    """str: human-readable name or phrase. Used in phrases like ``Follow this person on {PHRASE}``"""
1✔
96
    OTHER_LABELS = ()
1✔
97
    """sequence of str: label aliases"""
1✔
98
    LOGO_EMOJI = ''
1✔
99
    """str: logo emoji, if any"""
1✔
100
    LOGO_HTML = ''
1✔
101
    """str: logo ``<img>`` tag, if any"""
1✔
102
    CONTENT_TYPE = None
1✔
103
    """str: MIME type of this protocol's native data format, appropriate for the ``Content-Type`` HTTP header."""
1✔
104
    HAS_COPIES = False
1✔
105
    """bool: whether this protocol is push and needs us to proactively create "copy" users and objects, as opposed to pulling converted objects on demand"""
1✔
106
    DEFAULT_TARGET = None
1✔
107
    """str: optional, the default target URI to send this protocol's activities to. May be used as the "shared" target. Often only set if ``HAS_COPIES`` is true."""
1✔
108
    REQUIRES_AVATAR = False
1✔
109
    """bool: whether accounts on this protocol are required to have a profile picture. If they don't, their ``User.status`` will be ``blocked``."""
1✔
110
    REQUIRES_NAME = False
1✔
111
    """bool: whether accounts on this protocol are required to have a profile name that's different than their handle or id. If they don't, their ``User.status`` will be ``blocked``."""
1✔
112
    REQUIRES_OLD_ACCOUNT = False
1✔
113
    """bool: whether accounts on this protocol are required to be at least :const:`common.OLD_ACCOUNT_AGE` old. If their profile includes creation date and it's not old enough, their ``User.status`` will be ``blocked``."""
1✔
114
    DEFAULT_ENABLED_PROTOCOLS = ()
1✔
115
    """sequence of str: labels of other protocols that are automatically enabled for this protocol to bridge into"""
1✔
116
    DEFAULT_SERVE_USER_PAGES = False
1✔
117
    """bool: whether to serve user pages for all of this protocol's users on the fed.brid.gy. If ``False``, user pages will only be served for users who have explictly opted in."""
1✔
118
    SUPPORTED_AS1_TYPES = ()
1✔
119
    """sequence of str: AS1 objectTypes and verbs that this protocol supports receiving and sending"""
1✔
120
    SUPPORTS_DMS = False
1✔
121
    """bool: whether this protocol can receive DMs (chat messages)"""
1✔
122
    USES_OBJECT_FEED = False
1✔
123
    """bool: whether to store followers on this protocol in :attr:`Object.feed`."""
1✔
124
    HTML_PROFILES = False
1✔
125
    """bool: whether this protocol supports HTML in profile descriptions. If False, profile descriptions should be plain text."""
1✔
126
    SEND_REPLIES_TO_ORIG_POSTS_MENTIONS = False
1✔
127
    """bool: whether replies to this protocol should include the original post's mentions as delivery targets"""
1✔
128
    BOTS_FOLLOW_BACK = False
1✔
129
    """bool: when a user on this protocol follows a bot user to enable bridging, does the bot follow them back?"""
1✔
130

131
    @classmethod
1✔
132
    @property
1✔
133
    def LABEL(cls):
1✔
134
        """str: human-readable lower case name of this protocol, eg ``'activitypub``"""
135
        return cls.__name__.lower()
1✔
136

137
    @staticmethod
1✔
138
    def for_request(fed=None):
1✔
139
        """Returns the protocol for the current request.
140

141
        ...based on the request's hostname.
142

143
        Args:
144
          fed (str or protocol.Protocol): protocol to return if the current
145
            request is on ``fed.brid.gy``
146

147
        Returns:
148
          Protocol: protocol, or None if the provided domain or request hostname
149
          domain is not a subdomain of ``brid.gy`` or isn't a known protocol
150
        """
151
        return Protocol.for_bridgy_subdomain(request.host, fed=fed)
1✔
152

153
    @staticmethod
1✔
154
    def for_bridgy_subdomain(domain_or_url, fed=None):
1✔
155
        """Returns the protocol for a brid.gy subdomain.
156

157
        Args:
158
          domain_or_url (str)
159
          fed (str or protocol.Protocol): protocol to return if the current
160
            request is on ``fed.brid.gy``
161

162
        Returns:
163
          class: :class:`Protocol` subclass, or None if the provided domain or request
164
          hostname domain is not a subdomain of ``brid.gy`` or isn't a known
165
          protocol
166
        """
167
        domain = (util.domain_from_link(domain_or_url, minimize=False)
1✔
168
                  if util.is_web(domain_or_url)
169
                  else domain_or_url)
170

171
        if domain == common.PRIMARY_DOMAIN or domain in common.LOCAL_DOMAINS:
1✔
172
            return PROTOCOLS[fed] if isinstance(fed, str) else fed
1✔
173
        elif domain and domain.endswith(common.SUPERDOMAIN):
1✔
174
            label = domain.removesuffix(common.SUPERDOMAIN)
1✔
175
            return PROTOCOLS.get(label)
1✔
176

177
    @classmethod
1✔
178
    def owns_id(cls, id):
1✔
179
        """Returns whether this protocol owns the id, or None if it's unclear.
180

181
        To be implemented by subclasses.
182

183
        IDs are string identities that uniquely identify users, and are intended
184
        primarily to be machine readable and usable. Compare to handles, which
185
        are human-chosen, human-meaningful, and often but not always unique.
186

187
        Some protocols' ids are more or less deterministic based on the id
188
        format, eg AT Protocol owns ``at://`` URIs. Others, like http(s) URLs,
189
        could be owned by eg Web or ActivityPub.
190

191
        This should be a quick guess without expensive side effects, eg no
192
        external HTTP fetches to fetch the id itself or otherwise perform
193
        discovery.
194

195
        Returns False if the id's domain is in :const:`common.DOMAIN_BLOCKLIST`.
196

197
        Args:
198
          id (str)
199

200
        Returns:
201
          bool or None:
202
        """
203
        return False
1✔
204

205
    @classmethod
1✔
206
    def owns_handle(cls, handle, allow_internal=False):
1✔
207
        """Returns whether this protocol owns the handle, or None if it's unclear.
208

209
        To be implemented by subclasses.
210

211
        Handles are string identities that are human-chosen, human-meaningful,
212
        and often but not always unique. Compare to IDs, which uniquely identify
213
        users, and are intended primarily to be machine readable and usable.
214

215
        Some protocols' handles are more or less deterministic based on the id
216
        format, eg ActivityPub (technically WebFinger) handles are
217
        ``@user@instance.com``. Others, like domains, could be owned by eg Web,
218
        ActivityPub, AT Protocol, or others.
219

220
        This should be a quick guess without expensive side effects, eg no
221
        external HTTP fetches to fetch the id itself or otherwise perform
222
        discovery.
223

224
        Args:
225
          handle (str)
226
          allow_internal (bool): whether to return False for internal domains
227
            like ``fed.brid.gy``, ``bsky.brid.gy``, etc
228

229
        Returns:
230
          bool or None
231
        """
232
        return False
1✔
233

234
    @classmethod
1✔
235
    def handle_to_id(cls, handle):
1✔
236
        """Converts a handle to an id.
237

238
        To be implemented by subclasses.
239

240
        May incur network requests, eg DNS queries or HTTP requests. Avoids
241
        blocked or opted out users.
242

243
        Args:
244
          handle (str)
245

246
        Returns:
247
          str: corresponding id, or None if the handle can't be found
248
        """
249
        raise NotImplementedError()
×
250

251
    @classmethod
1✔
252
    def key_for(cls, id, allow_opt_out=False):
1✔
253
        """Returns the :class:`google.cloud.ndb.Key` for a given id's :class:`models.User`.
254

255
        To be implemented by subclasses. Canonicalizes the id if necessary.
256

257
        If called via `Protocol.key_for`, infers the appropriate protocol with
258
        :meth:`for_id`. If called with a concrete subclass, uses that subclass
259
        as is.
260

261
        Args:
262
          id (str):
263
          allow_opt_out (bool): whether to allow users who are currently opted out
264

265
        Returns:
266
          google.cloud.ndb.Key: matching key, or None if the given id is not a
267
          valid :class:`User` id for this protocol.
268
        """
269
        if cls == Protocol:
1✔
270
            proto = Protocol.for_id(id)
1✔
271
            return proto.key_for(id, allow_opt_out=allow_opt_out) if proto else None
1✔
272

273
        # load user so that we follow use_instead
274
        existing = cls.get_by_id(id, allow_opt_out=True)
1✔
275
        if existing:
1✔
276
            if existing.status and not allow_opt_out:
1✔
277
                return None
1✔
278
            return existing.key
1✔
279

280
        return cls(id=id).key
1✔
281

282
    @staticmethod
1✔
283
    def _for_id_memcache_key(id, remote=None):
1✔
284
        """If id is a URL, uses its domain, otherwise returns None.
285

286
        Args:
287
          id (str)
288

289
        Returns:
290
          (str domain, bool remote) or None
291
        """
292
        domain = util.domain_from_link(id)
1✔
293
        if domain in PROTOCOL_DOMAINS:
1✔
294
            return id
1✔
295
        elif remote and util.is_web(id):
1✔
296
            return domain
1✔
297

298
    @cached(LRUCache(20000), lock=Lock())
1✔
299
    @memcache.memoize(key=_for_id_memcache_key, write=lambda id, remote=True: remote,
1✔
300
                      version=3)
301
    @staticmethod
1✔
302
    def for_id(id, remote=True):
1✔
303
        """Returns the protocol for a given id.
304

305
        Args:
306
          id (str)
307
          remote (bool): whether to perform expensive side effects like fetching
308
            the id itself over the network, or other discovery.
309

310
        Returns:
311
          Protocol subclass: matching protocol, or None if no single known
312
          protocol definitively owns this id
313
        """
314
        logger.debug(f'Determining protocol for id {id}')
1✔
315
        if not id:
1✔
316
            return None
1✔
317

318
        # remove our synthetic id fragment, if any
319
        #
320
        # will this eventually cause false positives for other services that
321
        # include our full ids inside their own ids, non-URL-encoded? guess
322
        # we'll figure that out if/when it happens.
323
        id = id.partition('#bridgy-fed-')[0]
1✔
324
        if not id:
1✔
325
            return None
1✔
326

327
        if util.is_web(id):
1✔
328
            # step 1: check for our per-protocol subdomains
329
            try:
1✔
330
                parsed = urlparse(id)
1✔
331
            except ValueError as e:
1✔
332
                logger.info(f'urlparse ValueError: {e}')
1✔
333
                return None
1✔
334

335
            is_homepage = parsed.path.strip('/') == ''
1✔
336
            is_internal = parsed.path.startswith(ids.INTERNAL_PATH_PREFIX)
1✔
337
            by_subdomain = Protocol.for_bridgy_subdomain(id)
1✔
338
            if by_subdomain and not (is_homepage or is_internal
1✔
339
                                     or id in ids.BOT_ACTOR_AP_IDS):
340
                logger.debug(f'  {by_subdomain.LABEL} owns id {id}')
1✔
341
                return by_subdomain
1✔
342

343
        # step 2: check if any Protocols say conclusively that they own it
344
        # sort to be deterministic
345
        protocols = sorted(set(p for p in PROTOCOLS.values() if p),
1✔
346
                           key=lambda p: p.LABEL)
347
        candidates = []
1✔
348
        for protocol in protocols:
1✔
349
            owns = protocol.owns_id(id)
1✔
350
            if owns:
1✔
351
                logger.debug(f'  {protocol.LABEL} owns id {id}')
1✔
352
                return protocol
1✔
353
            elif owns is not False:
1✔
354
                candidates.append(protocol)
1✔
355

356
        if len(candidates) == 1:
1✔
357
            logger.debug(f'  {candidates[0].LABEL} owns id {id}')
1✔
358
            return candidates[0]
1✔
359

360
        # step 3: look for existing Objects in the datastore
361
        #
362
        # note that we don't currently see if this is a copy id because I have FUD
363
        # over which Protocol for_id should return in that case...and also because a
364
        # protocol may already say definitively above that it owns the id, eg ATProto
365
        # with DIDs and at:// URIs.
366
        obj = Protocol.load(id, remote=False)
1✔
367
        if obj and obj.source_protocol:
1✔
368
            logger.debug(f'  {obj.key.id()} owned by source_protocol {obj.source_protocol}')
1✔
369
            return PROTOCOLS[obj.source_protocol]
1✔
370

371
        # step 4: fetch over the network, if necessary
372
        if not remote:
1✔
373
            return None
1✔
374

375
        for protocol in candidates:
1✔
376
            logger.debug(f'Trying {protocol.LABEL}')
1✔
377
            try:
1✔
378
                obj = protocol.load(id, local=False, remote=True)
1✔
379

380
                if protocol.ABBREV == 'web':
1✔
381
                    # for web, if we fetch and get HTML without microformats,
382
                    # load returns False but the object will be stored in the
383
                    # datastore with source_protocol web, and in cache. load it
384
                    # again manually to check for that.
385
                    obj = Object.get_by_id(id)
1✔
386
                    if obj and obj.source_protocol != 'web':
1✔
387
                        obj = None
×
388

389
                if obj:
1✔
390
                    logger.debug(f'  {protocol.LABEL} owns id {id}')
1✔
391
                    return protocol
1✔
392
            except BadGateway:
1✔
393
                # we tried and failed fetching the id over the network.
394
                # this depends on ActivityPub.fetch raising this!
395
                return None
1✔
396
            except HTTPException as e:
×
397
                # internal error we generated ourselves; try next protocol
398
                pass
×
399
            except Exception as e:
×
400
                code, _ = util.interpret_http_exception(e)
×
401
                if code:
×
402
                    # we tried and failed fetching the id over the network
403
                    return None
×
404
                raise
×
405

406
        logger.info(f'No matching protocol found for {id} !')
1✔
407
        return None
1✔
408

409
    @cached(LRUCache(20000), lock=Lock())
1✔
410
    @staticmethod
1✔
411
    def for_handle(handle):
1✔
412
        """Returns the protocol for a given handle.
413

414
        May incur expensive side effects like resolving the handle itself over
415
        the network or other discovery.
416

417
        Args:
418
          handle (str)
419

420
        Returns:
421
          (Protocol subclass, str) tuple: matching protocol and optional id (if
422
          resolved), or ``(None, None)`` if no known protocol owns this handle
423
        """
424
        # TODO: normalize, eg convert domains to lower case
425
        logger.debug(f'Determining protocol for handle {handle}')
1✔
426
        if not handle:
1✔
427
            return (None, None)
1✔
428

429
        # step 1: check if any Protocols say conclusively that they own it.
430
        # sort to be deterministic.
431
        protocols = sorted(set(p for p in PROTOCOLS.values() if p),
1✔
432
                           key=lambda p: p.LABEL)
433
        candidates = []
1✔
434
        for proto in protocols:
1✔
435
            owns = proto.owns_handle(handle)
1✔
436
            if owns:
1✔
437
                logger.debug(f'  {proto.LABEL} owns handle {handle}')
1✔
438
                return (proto, None)
1✔
439
            elif owns is not False:
1✔
440
                candidates.append(proto)
1✔
441

442
        if len(candidates) == 1:
1✔
443
            logger.debug(f'  {candidates[0].LABEL} owns handle {handle}')
×
444
            return (candidates[0], None)
×
445

446
        # step 2: look for matching User in the datastore
447
        for proto in candidates:
1✔
448
            user = proto.query(proto.handle == handle).get()
1✔
449
            if user:
1✔
450
                if user.status:
1✔
451
                    return (None, None)
1✔
452
                logger.debug(f'  user {user.key} handle {handle}')
1✔
453
                return (proto, user.key.id())
1✔
454

455
        # step 3: resolve handle to id
456
        for proto in candidates:
1✔
457
            id = proto.handle_to_id(handle)
1✔
458
            if id:
1✔
459
                logger.debug(f'  {proto.LABEL} resolved handle {handle} to id {id}')
1✔
460
                return (proto, id)
1✔
461

462
        logger.info(f'No matching protocol found for handle {handle} !')
1✔
463
        return (None, None)
1✔
464

465
    @classmethod
1✔
466
    def is_user_at_domain(cls, handle, allow_internal=False):
1✔
467
        """Returns True if handle is formatted ``user@domain.tld``, False otherwise.
468

469
        Example: ``@user@instance.com``
470

471
        Args:
472
          handle (str)
473
          allow_internal (bool): whether the domain can be a Bridgy Fed domain
474
        """
475
        parts = handle.split('@')
1✔
476
        if len(parts) != 2:
1✔
477
            return False
1✔
478

479
        user, domain = parts
1✔
480
        return bool(user and domain
1✔
481
                    and not cls.is_blocklisted(domain, allow_internal=allow_internal))
482

483
    @classmethod
1✔
484
    def bridged_web_url_for(cls, user, fallback=False):
1✔
485
        """Returns the web URL for a user's bridged profile in this protocol.
486

487
        For example, for Web user ``alice.com``, :meth:`ATProto.bridged_web_url_for`
488
        returns ``https://bsky.app/profile/alice.com.web.brid.gy``
489

490
        Args:
491
          user (models.User)
492
          fallback (bool): if True, and bridged users have no canonical user
493
            profile URL in this protocol, return the native protocol's profile URL
494

495
        Returns:
496
          str, or None if there isn't a canonical URL
497
        """
498
        if fallback:
1✔
499
            return user.web_url()
1✔
500

501
    @classmethod
1✔
502
    def actor_key(cls, obj, allow_opt_out=False):
1✔
503
        """Returns the :class:`User`: key for a given object's author or actor.
504

505
        Args:
506
          obj (models.Object)
507
          allow_opt_out (bool): whether to return a user key if they're opted out
508

509
        Returns:
510
          google.cloud.ndb.key.Key or None:
511
        """
512
        owner = as1.get_owner(obj.as1)
1✔
513
        if owner:
1✔
514
            return cls.key_for(owner, allow_opt_out=allow_opt_out)
1✔
515

516
    @classmethod
1✔
517
    def bot_user_id(cls):
1✔
518
        """Returns the Web user id for the bot user for this protocol.
519

520
        For example, ``'bsky.brid.gy'`` for ATProto.
521

522
        Returns:
523
          str:
524
        """
525
        return f'{cls.ABBREV}{common.SUPERDOMAIN}'
1✔
526

527
    @classmethod
1✔
528
    def create_for(cls, user):
1✔
529
        """Creates or re-activate a copy user in this protocol.
530

531
        Should add the copy user to :attr:`copies`.
532

533
        If the copy user already exists and active, should do nothing.
534

535
        Args:
536
          user (models.User): original source user. Shouldn't already have a
537
            copy user for this protocol in :attr:`copies`.
538

539
        Raises:
540
          ValueError: if we can't create a copy of the given user in this protocol
541
        """
542
        raise NotImplementedError()
×
543

544
    @classmethod
1✔
545
    def send(to_cls, obj, target, from_user=None, orig_obj_id=None):
1✔
546
        """Sends an outgoing activity.
547

548
        To be implemented by subclasses. Should call
549
        ``to_cls.translate_ids(obj.as1)`` before converting it to this Protocol's
550
        format.
551

552
        NOTE: if this protocol's ``HAS_COPIES`` is True, and this method creates
553
        a copy and sends it, it *must* add that copy to the *object*'s (not
554
        activity's) :attr:`copies`, and store it back in the datastore!
555

556
        Args:
557
          obj (models.Object): with activity to send
558
          target (str): destination URL to send to
559
          from_user (models.User): user (actor) this activity is from
560
          orig_obj_id (str): :class:`models.Object` key id of the "original object"
561
            that this object refers to, eg replies to or reposts or likes
562

563
        Returns:
564
          bool: True if the activity is sent successfully, False if it is
565
          ignored or otherwise unsent due to protocol logic, eg no webmention
566
          endpoint, protocol doesn't support the activity type. (Failures are
567
          raised as exceptions.)
568

569
        Raises:
570
          werkzeug.HTTPException if the request fails
571
        """
572
        raise NotImplementedError()
×
573

574
    @classmethod
1✔
575
    def fetch(cls, obj, **kwargs):
1✔
576
        """Fetches a protocol-specific object and populates it in an :class:`Object`.
577

578
        Errors are raised as exceptions. If this method returns False, the fetch
579
        didn't fail but didn't succeed either, eg the id isn't valid for this
580
        protocol, or the fetch didn't return valid data for this protocol.
581

582
        To be implemented by subclasses.
583

584
        Args:
585
          obj (models.Object): with the id to fetch. Data is filled into one of
586
            the protocol-specific properties, eg ``as2``, ``mf2``, ``bsky``.
587
          kwargs: subclass-specific
588

589
        Returns:
590
          bool: True if the object was fetched and populated successfully,
591
          False otherwise
592

593
        Raises:
594
          requests.RequestException, werkzeug.HTTPException,
595
          websockets.WebSocketException, etc: if the fetch fails
596
        """
597
        raise NotImplementedError()
×
598

599
    @classmethod
1✔
600
    def convert(cls, obj, from_user=None, **kwargs):
1✔
601
        """Converts an :class:`Object` to this protocol's data format.
602

603
        For example, an HTML string for :class:`Web`, or a dict with AS2 JSON
604
        and ``application/activity+json`` for :class:`ActivityPub`.
605

606
        Just passes through to :meth:`_convert`, then does minor
607
        protocol-independent postprocessing.
608

609
        Args:
610
          obj (models.Object):
611
          from_user (models.User): user (actor) this activity/object is from
612
          kwargs: protocol-specific, passed through to :meth:`_convert`
613

614
        Returns:
615
          converted object in the protocol's native format, often a dict
616
        """
617
        if not obj or not obj.as1:
1✔
618
            return {}
1✔
619

620
        id = obj.key.id() if obj.key else obj.as1.get('id')
1✔
621
        is_crud = obj.as1.get('verb') in as1.CRUD_VERBS
1✔
622
        base_obj = as1.get_object(obj.as1) if is_crud else obj.as1
1✔
623
        orig_our_as1 = obj.our_as1
1✔
624

625
        # mark bridged actors as bots and add "bridged by Bridgy Fed" to their bios
626
        if (from_user and base_obj
1✔
627
            and base_obj.get('objectType') in as1.ACTOR_TYPES
628
            and PROTOCOLS.get(obj.source_protocol) != cls
629
            and Protocol.for_bridgy_subdomain(id) not in DOMAINS
630
            # Web users are special cased, they don't get the label if they've
631
            # explicitly enabled Bridgy Fed with redirects or webmentions
632
            and not (from_user.LABEL == 'web'
633
                     and (from_user.last_webmention_in or from_user.has_redirects))):
634
            cls.add_source_links(obj=obj, from_user=from_user)
1✔
635

636
        converted = cls._convert(obj, from_user=from_user, **kwargs)
1✔
637
        obj.our_as1 = orig_our_as1
1✔
638
        return converted
1✔
639

640
    @classmethod
1✔
641
    def _convert(cls, obj, from_user=None, **kwargs):
1✔
642
        """Converts an :class:`Object` to this protocol's data format.
643

644
        To be implemented by subclasses. Implementations should generally call
645
        :meth:`Protocol.translate_ids` (as their own class) before converting to
646
        their format.
647

648
        Args:
649
          obj (models.Object):
650
          from_user (models.User): user (actor) this activity/object is from
651
          kwargs: protocol-specific
652

653
        Returns:
654
          converted object in the protocol's native format, often a dict. May
655
            return the ``{}`` empty dict if the object can't be converted.
656
        """
657
        raise NotImplementedError()
×
658

659
    @classmethod
1✔
660
    def add_source_links(cls, obj, from_user):
1✔
661
        """Adds "bridged from ... by Bridgy Fed" to the user's actor's ``summary``.
662

663
        Uses HTML for protocols that support it, plain text otherwise.
664

665
        Args:
666
          cls (Protocol subclass): protocol that the user is bridging into
667
          obj (models.Object): user's actor/profile object
668
          from_user (models.User): user (actor) this activity/object is from
669
        """
670
        assert obj and obj.as1
1✔
671
        assert from_user
1✔
672

673
        obj.our_as1 = copy.deepcopy(obj.as1)
1✔
674
        actor = (as1.get_object(obj.as1) if obj.type in as1.CRUD_VERBS
1✔
675
                 else obj.as1)
676
        actor['objectType'] = 'person'
1✔
677

678
        orig_summary = actor.setdefault('summary', '')
1✔
679
        summary_text = html_to_text(orig_summary, ignore_links=True)
1✔
680

681
        # Check if we've already added source links
682
        if '🌉 bridged' in summary_text:
1✔
683
            return
1✔
684

685
        actor_id = actor.get('id')
1✔
686

687
        url = (as1.get_url(actor)
1✔
688
               or (from_user.web_url() if from_user.profile_id() == actor_id
689
                   else actor_id))
690

691
        from web import Web
1✔
692
        bot_user = Web.get_by_id(from_user.bot_user_id())
1✔
693

694
        if cls.HTML_PROFILES:
1✔
695
            if bot_user and from_user.LABEL not in cls.DEFAULT_ENABLED_PROTOCOLS:
1✔
696
                mention = bot_user.user_link(proto=cls, name=False, handle='short')
1✔
697
                suffix = f', follow {mention} to interact'
1✔
698
            else:
699
                suffix = f' by <a href="https://{PRIMARY_DOMAIN}/">Bridgy Fed</a>'
1✔
700

701
            separator = '<br><br>'
1✔
702

703
            is_user = from_user.key and actor_id in (from_user.key.id(),
1✔
704
                                                     from_user.profile_id())
705
            if is_user:
1✔
706
                bridged = f'🌉 <a href="https://{PRIMARY_DOMAIN}{from_user.user_page_path()}">bridged</a>'
1✔
707
                from_ = f'<a href="{from_user.web_url()}">{from_user.handle}</a>'
1✔
708
            else:
709
                bridged = '🌉 bridged'
1✔
710
                from_ = util.pretty_link(url) if url else '?'
1✔
711

712
        else:  # plain text
713
            # TODO: unify with above. which is right?
714
            id = obj.key.id() if obj.key else obj.our_as1.get('id')
1✔
715
            is_user = from_user.key and id in (from_user.key.id(),
1✔
716
                                               from_user.profile_id())
717
            from_ = (from_user.web_url() if is_user else url) or '?'
1✔
718

719
            bridged = '🌉 bridged'
1✔
720
            suffix = (
1✔
721
                f': https://{PRIMARY_DOMAIN}{from_user.user_page_path()}'
722
                # link web users to their user pages
723
                if from_user.LABEL == 'web'
724
                else f', follow @{bot_user.handle_as(cls)} to interact'
725
                if bot_user and from_user.LABEL not in cls.DEFAULT_ENABLED_PROTOCOLS
726
                else f' by https://{PRIMARY_DOMAIN}/')
727
            separator = '\n\n'
1✔
728
            orig_summary = summary_text
1✔
729

730
        logo = f'{from_user.LOGO_EMOJI} ' if from_user.LOGO_EMOJI else ''
1✔
731
        source_links = f'{separator if orig_summary else ""}{bridged} from {logo}{from_}{suffix}'
1✔
732
        actor['summary'] = orig_summary + source_links
1✔
733

734
    @classmethod
1✔
735
    def set_username(to_cls, user, username):
1✔
736
        """Sets a custom username for a user's bridged account in this protocol.
737

738
        Args:
739
          user (models.User)
740
          username (str)
741

742
        Raises:
743
          ValueError: if the username is invalid
744
          RuntimeError: if the username could not be set
745
        """
746
        raise NotImplementedError()
1✔
747

748
    @classmethod
1✔
749
    def migrate_out(cls, user, to_user_id):
1✔
750
        """Migrates a bridged account out to be a native account.
751

752
        Args:
753
          user (models.User)
754
          to_user_id (str)
755

756
        Raises:
757
          ValueError: eg if this protocol doesn't own ``to_user_id``, or if
758
            ``user`` is on this protocol or not bridged to this protocol
759
        """
760
        raise NotImplementedError()
×
761

762
    @classmethod
1✔
763
    def check_can_migrate_out(cls, user, to_user_id):
1✔
764
        """Raises an exception if a user can't yet migrate to a native account.
765

766
        For example, if ``to_user_id`` isn't on this protocol, or if ``user`` is on
767
        this protocol, or isn't bridged to this protocol.
768

769
        If the user is ready to migrate, returns ``None``.
770

771
        Subclasses may override this to add more criteria, but they should call this
772
        implementation first.
773

774
        Args:
775
          user (models.User)
776
          to_user_id (str)
777

778
        Raises:
779
          ValueError: if ``user`` isn't ready to migrate to this protocol yet
780
        """
781
        def _error(msg):
1✔
782
            logger.warning(msg)
1✔
783
            raise ValueError(msg)
1✔
784

785
        if cls.owns_id(to_user_id) is False:
1✔
786
            _error(f"{to_user_id} doesn't look like an {cls.LABEL} id")
1✔
787
        elif isinstance(user, cls):
1✔
788
            _error(f"{user.handle_or_id()} is on {cls.PHRASE}")
1✔
789
        elif not user.is_enabled(cls):
1✔
790
            _error(f"{user.handle_or_id()} isn't currently bridged to {cls.PHRASE}")
1✔
791

792
    @classmethod
1✔
793
    def migrate_in(cls, user, from_user_id, **kwargs):
1✔
794
        """Migrates a native account in to be a bridged account.
795

796
        The protocol independent parts are done here; protocol-specific parts are
797
        done in :meth:`_migrate_in`, which this wraps.
798

799
        Reloads the user's profile before calling :meth:`_migrate_in`.
800

801
        Args:
802
          user (models.User): native user on another protocol to attach the
803
            newly imported bridged account to
804
          from_user_id (str)
805
          kwargs: additional protocol-specific parameters
806

807
        Raises:
808
          ValueError: eg if this protocol doesn't own ``from_user_id``, or if
809
            ``user`` is on this protocol or already bridged to this protocol
810
        """
811
        def _error(msg):
1✔
812
            logger.warning(msg)
1✔
813
            raise ValueError(msg)
1✔
814

815
        logger.info(f"Migrating in {from_user_id} for {user.key.id()}")
1✔
816

817
        # check req'ts
818
        if cls.owns_id(from_user_id) is False:
1✔
819
            _error(f"{from_user_id} doesn't look like an {cls.LABEL} id")
1✔
820
        elif isinstance(user, cls):
1✔
821
            _error(f"{user.handle_or_id()} is on {cls.PHRASE}")
1✔
822
        elif cls.HAS_COPIES and cls.LABEL in user.enabled_protocols:
1✔
823
            _error(f"{user.handle_or_id()} is already bridged to {cls.PHRASE}")
1✔
824

825
        # reload profile
826
        try:
1✔
827
            user.reload_profile()
1✔
828
        except (RequestException, HTTPException) as e:
×
829
            _, msg = util.interpret_http_exception(e)
×
830

831
        # migrate!
832
        cls._migrate_in(user, from_user_id, **kwargs)
1✔
833
        user.add('enabled_protocols', cls.LABEL)
1✔
834
        user.put()
1✔
835

836
        # attach profile object
837
        if user.obj:
1✔
838
            if cls.HAS_COPIES:
1✔
839
                profile_id = ids.profile_id(id=from_user_id, proto=cls)
1✔
840
                user.obj.remove_copies_on(cls)
1✔
841
                user.obj.add('copies', Target(uri=profile_id, protocol=cls.LABEL))
1✔
842
                user.obj.put()
1✔
843

844
            common.create_task(queue='receive', obj_id=user.obj_key.id(),
1✔
845
                               authed_as=user.key.id())
846

847
    @classmethod
1✔
848
    def _migrate_in(cls, user, from_user_id, **kwargs):
1✔
849
        """Protocol-specific parts of migrating in external account.
850

851
        Called by :meth:`migrate_in`, which does most of the work, including calling
852
        :meth:`reload_profile` before this.
853

854
        Args:
855
          user (models.User): native user on another protocol to attach the
856
            newly imported account to. Unused.
857
          from_user_id (str): DID of the account to be migrated in
858
          kwargs: protocol dependent
859
        """
860
        raise NotImplementedError()
×
861

862
    @classmethod
1✔
863
    def target_for(cls, obj, shared=False):
1✔
864
        """Returns an :class:`Object`'s delivery target (endpoint).
865

866
        To be implemented by subclasses.
867

868
        Examples:
869

870
        * If obj has ``source_protocol`` ``web``, returns its URL, as a
871
          webmention target.
872
        * If obj is an ``activitypub`` actor, returns its inbox.
873
        * If obj is an ``activitypub`` object, returns it's author's or actor's
874
          inbox.
875

876
        Args:
877
          obj (models.Object):
878
          shared (bool): optional. If True, returns a common/shared
879
            endpoint, eg ActivityPub's ``sharedInbox``, that can be reused for
880
            multiple recipients for efficiency
881

882
        Returns:
883
          str: target endpoint, or None if not available.
884
        """
885
        raise NotImplementedError()
×
886

887
    @classmethod
1✔
888
    def is_blocklisted(cls, url, allow_internal=False):
1✔
889
        """Returns True if we block the given URL and shouldn't deliver to it.
890

891
        Default implementation here, subclasses may override.
892

893
        Args:
894
          url (str):
895
          allow_internal (bool): whether to return False for internal domains
896
            like ``fed.brid.gy``, ``bsky.brid.gy``, etc
897
        """
898
        blocklist = DOMAIN_BLOCKLIST
1✔
899
        if not DEBUG:
1✔
900
            blocklist += tuple(util.RESERVED_TLDS | util.LOCAL_TLDS)
×
901
        if not allow_internal:
1✔
902
            blocklist += DOMAINS
1✔
903
        return util.domain_or_parent_in(url, blocklist)
1✔
904

905
    @classmethod
1✔
906
    def translate_ids(to_cls, obj):
1✔
907
        """Translates all ids in an AS1 object to a specific protocol.
908

909
        Infers source protocol for each id value separately.
910

911
        For example, if ``proto`` is :class:`ActivityPub`, the ATProto URI
912
        ``at://did:plc:abc/coll/123`` will be converted to
913
        ``https://bsky.brid.gy/ap/at://did:plc:abc/coll/123``.
914

915
        Wraps these AS1 fields:
916

917
        * ``id``
918
        * ``actor``
919
        * ``author``
920
        * ``bcc``
921
        * ``bto``
922
        * ``cc``
923
        * ``featured[].items``, ``featured[].orderedItems``
924
        * ``object``
925
        * ``object.actor``
926
        * ``object.author``
927
        * ``object.id``
928
        * ``object.inReplyTo``
929
        * ``object.object``
930
        * ``attachments[].id``
931
        * ``tags[objectType=mention].url``
932
        * ``to``
933

934
        This is the inverse of :meth:`models.Object.resolve_ids`. Much of the
935
        same logic is duplicated there!
936

937
        TODO: unify with :meth:`Object.resolve_ids`,
938
        :meth:`models.Object.normalize_ids`.
939

940
        Args:
941
          to_proto (Protocol subclass)
942
          obj (dict): AS1 object or activity (not :class:`models.Object`!)
943

944
        Returns:
945
          dict: wrapped AS1 version of ``obj``
946
        """
947
        assert to_cls != Protocol
1✔
948
        if not obj:
1✔
949
            return obj
1✔
950

951
        outer_obj = copy.deepcopy(obj)
1✔
952
        inner_objs = outer_obj['object'] = as1.get_objects(outer_obj)
1✔
953

954
        def translate(elem, field, fn, uri=False):
1✔
955
            elem[field] = as1.get_objects(elem, field)
1✔
956
            for obj in elem[field]:
1✔
957
                if id := obj.get('id'):
1✔
958
                    if field in ('to', 'cc', 'bcc', 'bto') and as1.is_audience(id):
1✔
959
                        continue
1✔
960
                    from_cls = Protocol.for_id(id)
1✔
961
                    # TODO: what if from_cls is None? relax translate_object_id,
962
                    # make it a noop if we don't know enough about from/to?
963
                    if from_cls and from_cls != to_cls:
1✔
964
                        obj['id'] = fn(id=id, from_=from_cls, to=to_cls)
1✔
965
                    if obj['id'] and uri:
1✔
966
                        obj['id'] = to_cls(id=obj['id']).id_uri()
1✔
967

968
            elem[field] = [o['id'] if o.keys() == {'id'} else o
1✔
969
                           for o in elem[field]]
970

971
            if len(elem[field]) == 1 and field not in ('items', 'orderedItems'):
1✔
972
                elem[field] = elem[field][0]
1✔
973

974
        type = as1.object_type(outer_obj)
1✔
975
        translate(outer_obj, 'id',
1✔
976
                  ids.translate_user_id if type in as1.ACTOR_TYPES
977
                  else ids.translate_object_id)
978

979
        for o in inner_objs:
1✔
980
            is_actor = (as1.object_type(o) in as1.ACTOR_TYPES
1✔
981
                        or as1.get_owner(outer_obj) == o.get('id')
982
                        or type in ('follow', 'stop-following'))
983
            translate(o, 'id', (ids.translate_user_id if is_actor
1✔
984
                                else ids.translate_object_id))
985
            obj_is_actor = o.get('verb') in as1.VERBS_WITH_ACTOR_OBJECT
1✔
986
            translate(o, 'object', (ids.translate_user_id if obj_is_actor
1✔
987
                                    else ids.translate_object_id))
988

989
        for o in [outer_obj] + inner_objs:
1✔
990
            translate(o, 'inReplyTo', ids.translate_object_id)
1✔
991
            for field in 'actor', 'author', 'to', 'cc', 'bto', 'bcc':
1✔
992
                translate(o, field, ids.translate_user_id)
1✔
993
            for tag in as1.get_objects(o, 'tags'):
1✔
994
                if tag.get('objectType') == 'mention':
1✔
995
                    translate(tag, 'url', ids.translate_user_id, uri=True)
1✔
996
            for att in as1.get_objects(o, 'attachments'):
1✔
997
                translate(att, 'id', ids.translate_object_id)
1✔
998
                url = att.get('url')
1✔
999
                if url and not att.get('id'):
1✔
1000
                    if from_cls := Protocol.for_id(url):
1✔
1001
                        att['id'] = ids.translate_object_id(from_=from_cls, to=to_cls,
1✔
1002
                                                            id=url)
1003
            if feat := as1.get_object(o, 'featured'):
1✔
1004
                translate(feat, 'orderedItems', ids.translate_object_id)
1✔
1005
                translate(feat, 'items', ids.translate_object_id)
1✔
1006

1007
        outer_obj = util.trim_nulls(outer_obj)
1✔
1008

1009
        if objs := util.get_list(outer_obj ,'object'):
1✔
1010
            outer_obj['object'] = [o['id'] if o.keys() == {'id'} else o for o in objs]
1✔
1011
            if len(outer_obj['object']) == 1:
1✔
1012
                outer_obj['object'] = outer_obj['object'][0]
1✔
1013

1014
        return outer_obj
1✔
1015

1016
    @classmethod
1✔
1017
    def receive(from_cls, obj, authed_as=None, internal=False, received_at=None):
1✔
1018
        """Handles an incoming activity.
1019

1020
        If ``obj``'s key is unset, ``obj.as1``'s id field is used. If both are
1021
        unset, returns HTTP 299.
1022

1023
        Args:
1024
          obj (models.Object)
1025
          authed_as (str): authenticated actor id who sent this activity
1026
          internal (bool): whether to allow activity ids on internal domains,
1027
            from opted out/blocked users, etc.
1028
          received_at (datetime): when we first saw (received) this activity.
1029
            Right now only used for monitoring.
1030

1031
        Returns:
1032
          (str, int) tuple: (response body, HTTP status code) Flask response
1033

1034
        Raises:
1035
          werkzeug.HTTPException: if the request is invalid
1036
        """
1037
        # check some invariants
1038
        assert from_cls != Protocol
1✔
1039
        assert isinstance(obj, Object), obj
1✔
1040

1041
        if not obj.as1:
1✔
1042
            error('No object data provided')
1✔
1043

1044
        orig_obj = obj
1✔
1045
        id = None
1✔
1046
        if obj.key and obj.key.id():
1✔
1047
            id = obj.key.id()
1✔
1048

1049
        if not id:
1✔
1050
            id = obj.as1.get('id')
1✔
1051
            obj.key = ndb.Key(Object, id)
1✔
1052

1053
        if not id:
1✔
1054
            error('No id provided')
×
1055
        elif from_cls.owns_id(id) is False:
1✔
1056
            error(f'Protocol {from_cls.LABEL} does not own id {id}')
1✔
1057
        elif from_cls.is_blocklisted(id, allow_internal=internal):
1✔
1058
            error(f'Activity {id} is blocklisted')
1✔
1059

1060
        # does this protocol support this activity/object type?
1061
        from_cls.check_supported(obj, 'receive')
1✔
1062

1063
        # lease this object, atomically
1064
        memcache_key = activity_id_memcache_key(id)
1✔
1065
        leased = memcache.memcache.add(
1✔
1066
            memcache_key, 'leased', noreply=False,
1067
            expire=int(MEMCACHE_LEASE_EXPIRATION.total_seconds()))
1068

1069
        # short circuit if we've already seen this activity id.
1070
        # (don't do this for bare objects since we need to check further down
1071
        # whether they've been updated since we saw them last.)
1072
        if (obj.as1.get('objectType') == 'activity'
1✔
1073
            and 'force' not in request.values
1074
            and (not leased
1075
                 or (obj.new is False and obj.changed is False))):
1076
            error(f'Already seen this activity {id}', status=204)
1✔
1077

1078
        pruned = {k: v for k, v in obj.as1.items()
1✔
1079
                  if k not in ('contentMap', 'replies', 'signature')}
1080
        delay = ''
1✔
1081
        retry = request.headers.get('X-AppEngine-TaskRetryCount')
1✔
1082
        if (received_at and retry in (None, '0')
1✔
1083
                and obj.type not in ('delete', 'undo')):  # we delay deletes/undos
1084
            delay_s = int((util.now().replace(tzinfo=None)
1✔
1085
                           - received_at.replace(tzinfo=None)
1086
                           ).total_seconds())
1087
            delay = f'({delay_s} s behind)'
1✔
1088
        logger.info(f'Receiving {from_cls.LABEL} {obj.type} {id} {delay} AS1: {json_dumps(pruned, indent=2)}')
1✔
1089

1090
        # check authorization
1091
        # https://www.w3.org/wiki/ActivityPub/Primer/Authentication_Authorization
1092
        actor = as1.get_owner(obj.as1)
1✔
1093
        if not actor:
1✔
1094
            error('Activity missing actor or author')
1✔
1095
        elif from_cls.owns_id(actor) is False:
1✔
1096
            error(f"{from_cls.LABEL} doesn't own actor {actor}, this is probably a bridged activity. Skipping.", status=204)
1✔
1097

1098
        assert authed_as
1✔
1099
        assert isinstance(authed_as, str)
1✔
1100
        authed_as = ids.normalize_user_id(id=authed_as, proto=from_cls)
1✔
1101
        actor = ids.normalize_user_id(id=actor, proto=from_cls)
1✔
1102
        if actor != authed_as:
1✔
1103
            report_error("Auth: receive: authed_as doesn't match owner",
1✔
1104
                         user=f'{id} authed_as {authed_as} owner {actor}')
1105
            error(f"actor {actor} isn't authed user {authed_as}")
1✔
1106

1107
        # update copy ids to originals
1108
        obj.normalize_ids()
1✔
1109
        obj.resolve_ids()
1✔
1110

1111
        if (obj.type == 'follow'
1✔
1112
                and Protocol.for_bridgy_subdomain(as1.get_object(obj.as1).get('id'))):
1113
            # follows of bot user; refresh user profile first
1114
            logger.info(f'Follow of bot user, reloading {actor}')
1✔
1115
            from_user = from_cls.get_or_create(id=actor, allow_opt_out=True)
1✔
1116
            from_user.reload_profile()
1✔
1117
        else:
1118
            # load actor user
1119
            #
1120
            # TODO: we should maybe eventually allow non-None status users here if
1121
            # this is a profile update, so that we store the user again below and
1122
            # re-calculate its status. right now, if a bridged user updates their
1123
            # profile and invalidates themselves, eg by removing their profile
1124
            # picture, and then updates again to make themselves valid again, we'll
1125
            # ignore the second update. they'll have to un-bridge and re-bridge
1126
            # themselves to get back working again.
1127
            from_user = from_cls.get_or_create(id=actor, allow_opt_out=internal)
1✔
1128

1129
        if not internal and (not from_user or from_user.manual_opt_out):
1✔
1130
            error(f"Couldn't load actor {actor}", status=204)
1✔
1131

1132
        # check if this is a profile object coming in via a user with use_instead
1133
        # set. if so, override the object's id to be the final user id (from_user's),
1134
        # after following use_instead.
1135
        if obj.type in as1.ACTOR_TYPES and from_user.key.id() != actor:
1✔
1136
            as1_id = obj.as1.get('id')
1✔
1137
            if ids.normalize_user_id(id=as1_id, proto=from_user) == actor:
1✔
1138
                logger.info(f'Overriding AS1 object id {as1_id} with Object id {from_user.profile_id()}')
1✔
1139
                obj.our_as1 = {**obj.as1, 'id': from_user.profile_id()}
1✔
1140

1141
        # if this is an object, ie not an activity, wrap it in a create or update
1142
        obj = from_cls.handle_bare_object(obj, authed_as=authed_as,
1✔
1143
                                          from_user=from_user)
1144
        obj.add('users', from_user.key)
1✔
1145

1146
        inner_obj_as1 = as1.get_object(obj.as1)
1✔
1147
        inner_obj_id = inner_obj_as1.get('id')
1✔
1148
        if obj.type in as1.CRUD_VERBS | as1.VERBS_WITH_OBJECT:
1✔
1149
            if not inner_obj_id:
1✔
1150
                error(f'{obj.type} object has no id!')
1✔
1151

1152
        # check age. we support backdated posts, but if they're over 2w old, we
1153
        # don't deliver them
1154
        if obj.type == 'post':
1✔
1155
            if published := inner_obj_as1.get('published'):
1✔
1156
                try:
1✔
1157
                    published_dt = util.parse_iso8601(published)
1✔
1158
                    if not published_dt.tzinfo:
1✔
UNCOV
1159
                        published_dt = published_dt.replace(tzinfo=timezone.utc)
×
1160
                    age = util.now() - published_dt
1✔
1161
                    if age > CREATE_MAX_AGE and 'force' not in request.values:
1✔
UNCOV
1162
                        error(f'Ignoring, too old, {age} is over {CREATE_MAX_AGE}',
×
1163
                              status=204)
1164
                except ValueError:  # from parse_iso8601
×
UNCOV
1165
                    logger.debug(f"Couldn't parse published {published}")
×
1166

1167
        # write Object to datastore
1168
        obj.source_protocol = from_cls.LABEL
1✔
1169
        if obj.type in STORE_AS1_TYPES:
1✔
1170
            obj.put()
1✔
1171

1172
        # store inner object
1173
        # TODO: unify with big obj.type conditional below. would have to merge
1174
        # this with the DM handling block lower down.
1175
        crud_obj = None
1✔
1176
        if obj.type in ('post', 'update') and inner_obj_as1.keys() > set(['id']):
1✔
1177
            crud_obj = Object.get_or_create(inner_obj_id, our_as1=inner_obj_as1,
1✔
1178
                                            source_protocol=from_cls.LABEL,
1179
                                            authed_as=actor, users=[from_user.key],
1180
                                            deleted=False)
1181

1182
        actor = as1.get_object(obj.as1, 'actor')
1✔
1183
        actor_id = actor.get('id')
1✔
1184

1185
        # handle activity!
1186
        if obj.type == 'stop-following':
1✔
1187
            # TODO: unify with handle_follow?
1188
            # TODO: handle multiple followees
1189
            if not actor_id or not inner_obj_id:
1✔
UNCOV
1190
                error(f'stop-following requires actor id and object id. Got: {actor_id} {inner_obj_id} {obj.as1}')
×
1191

1192
            # deactivate Follower
1193
            from_ = from_cls.key_for(actor_id)
1✔
1194
            if not (to_cls := Protocol.for_id(inner_obj_id)):
1✔
1195
                error(f"Can't determine protocol for {inner_obj_id} , giving up")
1✔
1196
            to = to_cls.key_for(inner_obj_id)
1✔
1197
            follower = Follower.query(Follower.to == to,
1✔
1198
                                      Follower.from_ == from_,
1199
                                      Follower.status == 'active').get()
1200
            if follower:
1✔
1201
                logger.info(f'Marking {follower} inactive')
1✔
1202
                follower.status = 'inactive'
1✔
1203
                follower.put()
1✔
1204
            else:
1205
                logger.warning(f'No Follower found for {from_} => {to}')
1✔
1206

1207
            # fall through to deliver to followee
1208
            # TODO: do we convert stop-following to webmention 410 of original
1209
            # follow?
1210

1211
            # fall through to deliver to followers
1212

1213
        elif obj.type in ('delete', 'undo'):
1✔
1214
            delete_obj_id = (from_user.profile_id()
1✔
1215
                            if inner_obj_id == from_user.key.id()
1216
                            else inner_obj_id)
1217

1218
            delete_obj = Object.get_by_id(delete_obj_id, authed_as=authed_as)
1✔
1219
            if not delete_obj:
1✔
1220
                logger.info(f"Ignoring, we don't have {delete_obj_id} stored")
1✔
1221
                return 'OK', 204
1✔
1222

1223
            # TODO: just delete altogether!
1224
            logger.info(f'Marking Object {delete_obj_id} deleted')
1✔
1225
            delete_obj.deleted = True
1✔
1226
            delete_obj.put()
1✔
1227

1228
            # if this is an actor, handle deleting it later so that
1229
            # in case it's from_user, user.enabled_protocols is still populated
1230
            #
1231
            # fall through to deliver to followers and delete copy if necessary.
1232
            # should happen via protocol-specific copy target and send of
1233
            # delete activity.
1234
            # https://github.com/snarfed/bridgy-fed/issues/63
1235

1236
        elif obj.type == 'block':
1✔
1237
            if proto := Protocol.for_bridgy_subdomain(inner_obj_id):
1✔
1238
                # blocking protocol bot user disables that protocol
1239
                from_user.delete(proto)
1✔
1240
                from_user.disable_protocol(proto)
1✔
1241
                return 'OK', 200
1✔
1242

1243
        elif obj.type == 'post':
1✔
1244
            # handle DMs to bot users
1245
            if as1.is_dm(obj.as1):
1✔
1246
                return dms.receive(from_user=from_user, obj=obj)
1✔
1247

1248
        # fetch actor if necessary
1249
        is_user = (inner_obj_id in (from_user.key.id(), from_user.profile_id())
1✔
1250
                   or from_user.is_profile(orig_obj))
1251
        if (actor and actor.keys() == set(['id'])
1✔
1252
                and not is_user and obj.type not in ('delete', 'undo')):
1253
            logger.debug('Fetching actor so we have name, profile photo, etc')
1✔
1254
            actor_obj = from_cls.load(ids.profile_id(id=actor['id'], proto=from_cls),
1✔
1255
                                      raise_=False)
1256
            if actor_obj and actor_obj.as1:
1✔
1257
                obj.our_as1 = {
1✔
1258
                    **obj.as1, 'actor': {
1259
                        **actor_obj.as1,
1260
                        # override profile id with actor id
1261
                        # https://github.com/snarfed/bridgy-fed/issues/1720
1262
                        'id': actor['id'],
1263
                    }
1264
                }
1265

1266
        # fetch object if necessary
1267
        if (obj.type in ('post', 'update', 'share')
1✔
1268
                and inner_obj_as1.keys() == set(['id'])
1269
                and from_cls.owns_id(inner_obj_id) is not False):
1270
            logger.debug('Fetching inner object')
1✔
1271
            inner_obj = from_cls.load(inner_obj_id, raise_=False,
1✔
1272
                                      remote=(obj.type in ('post', 'update')))
1273
            if obj.type in ('post', 'update'):
1✔
1274
                crud_obj = inner_obj
1✔
1275
            if inner_obj and inner_obj.as1:
1✔
1276
                obj.our_as1 = {
1✔
1277
                    **obj.as1,
1278
                    'object': {
1279
                        **inner_obj_as1,
1280
                        **inner_obj.as1,
1281
                    }
1282
                }
1283
            elif obj.type in ('post', 'update'):
1✔
1284
                error(f"Need object {inner_obj_id} but couldn't fetch, giving up")
1✔
1285

1286
        if obj.type == 'follow':
1✔
1287
            if proto := Protocol.for_bridgy_subdomain(inner_obj_id):
1✔
1288
                # follow of one of our protocol bot users; enable that protocol.
1289
                # fall through so that we send an accept.
1290
                try:
1✔
1291
                    from_user.enable_protocol(proto)
1✔
1292
                except ErrorButDoNotRetryTask:
1✔
1293
                    from web import Web
1✔
1294
                    bot = Web.get_by_id(proto.bot_user_id())
1✔
1295
                    from_cls.respond_to_follow('reject', follower=from_user,
1✔
1296
                                               followee=bot, follow=obj)
1297
                    raise
1✔
1298
                proto.bot_maybe_follow_back(from_user)
1✔
1299

1300
            from_cls.handle_follow(obj, from_user=from_user)
1✔
1301

1302
        # on update of the user's own actor/profile, set user.obj and store user back
1303
        # to datastore so that we recalculate computed properties like status etc
1304
        if is_user:
1✔
1305
            if obj.type == 'update' and crud_obj:
1✔
1306
                logger.info("update of the user's profile, re-storing user")
1✔
1307
                from_user.obj = crud_obj
1✔
1308
                from_user.put()
1✔
1309

1310
        # deliver to targets
1311
        resp = from_cls.deliver(obj, from_user=from_user, crud_obj=crud_obj)
1✔
1312

1313
        # on user deleting themselves, deactivate their followers/followings.
1314
        # https://github.com/snarfed/bridgy-fed/issues/1304
1315
        #
1316
        # do this *after* delivering because delivery finds targets based on
1317
        # stored Followers
1318
        if is_user and obj.type == 'delete':
1✔
1319
            for proto in from_user.enabled_protocols:
1✔
1320
                from_user.disable_protocol(PROTOCOLS[proto])
1✔
1321

1322
            logger.info(f'Deactivating Followers from or to {from_user.key.id()}')
1✔
1323
            followers = Follower.query(
1✔
1324
                OR(Follower.to == from_user.key, Follower.from_ == from_user.key)
1325
            ).fetch()
1326
            for f in followers:
1✔
1327
                f.status = 'inactive'
1✔
1328
            ndb.put_multi(followers)
1✔
1329

1330
        memcache.memcache.set(memcache_key, 'done', expire=7 * 24 * 60 * 60)  # 1w
1✔
1331
        return resp
1✔
1332

1333
    @classmethod
1✔
1334
    def handle_follow(from_cls, obj, from_user):
1✔
1335
        """Handles an incoming follow activity.
1336

1337
        Sends an ``Accept`` back, but doesn't send the ``Follow`` itself. That
1338
        happens in :meth:`deliver`.
1339

1340
        Args:
1341
          obj (models.Object): follow activity
1342
        """
1343
        logger.debug('Got follow. storing Follow(s), sending accept(s)')
1✔
1344
        from_id = from_user.key.id()
1✔
1345

1346
        # Prepare followee (to) users' data
1347
        to_as1s = as1.get_objects(obj.as1)
1✔
1348
        if not to_as1s:
1✔
UNCOV
1349
            error(f'Follow activity requires object(s). Got: {obj.as1}')
×
1350

1351
        # Store Followers
1352
        for to_as1 in to_as1s:
1✔
1353
            to_id = to_as1.get('id')
1✔
1354
            if not to_id:
1✔
UNCOV
1355
                error(f'Follow activity requires object(s). Got: {obj.as1}')
×
1356

1357
            logger.info(f'Follow {from_id} => {to_id}')
1✔
1358

1359
            to_cls = Protocol.for_id(to_id)
1✔
1360
            if not to_cls:
1✔
UNCOV
1361
                error(f"Couldn't determine protocol for {to_id}")
×
1362
            elif from_cls == to_cls:
1✔
1363
                logger.info(f'Skipping same-protocol Follower {from_id} => {to_id}')
1✔
1364
                continue
1✔
1365

1366
            to_key = to_cls.key_for(to_id)
1✔
1367
            if not to_key:
1✔
1368
                logger.info(f'Skipping invalid {from_cls.LABEL} user key: {from_id}')
×
UNCOV
1369
                continue
×
1370

1371
            to_user = to_cls.get_or_create(id=to_key.id())
1✔
1372
            if not to_user or not to_user.is_enabled(from_user):
1✔
1373
                error(f'{to_id} not found')
1✔
1374

1375
            follower_obj = Follower.get_or_create(to=to_user, from_=from_user,
1✔
1376
                                                  follow=obj.key, status='active')
1377
            obj.add('notify', to_key)
1✔
1378
            from_cls.respond_to_follow('accept', follower=from_user,
1✔
1379
                                       followee=to_user, follow=obj)
1380

1381
    @classmethod
1✔
1382
    def respond_to_follow(_, verb, follower, followee, follow):
1✔
1383
        """Sends an accept or reject activity for a follow.
1384

1385
        ...if the follower's protocol supports accepts/rejects. Otherwise, does
1386
        nothing.
1387

1388
        Args:
1389
          verb (str): ``accept`` or  ``reject``
1390
          follower (models.User)
1391
          followee (models.User)
1392
          follow (models.Object)
1393
        """
1394
        assert verb in ('accept', 'reject')
1✔
1395
        if verb not in follower.SUPPORTED_AS1_TYPES:
1✔
1396
            return
1✔
1397

1398
        if not follower.obj or not (target := follower.target_for(follower.obj)):
1✔
1399
            error(f"Couldn't find delivery target for follower {follower.key.id()}")
1✔
1400

1401
        # send. note that this is one response for the whole follow, even if it
1402
        # has multiple followees!
1403
        id = f'{followee.key.id()}/followers#{verb}-{follow.key.id()}'
1✔
1404
        accept = {
1✔
1405
            'id': id,
1406
            'objectType': 'activity',
1407
            'verb': verb,
1408
            'actor': followee.key.id(),
1409
            'object': follow.as1,
1410
        }
1411
        common.create_task(queue='send', id=id, our_as1=accept, url=target,
1✔
1412
                           protocol=follower.LABEL, user=followee.key.urlsafe())
1413

1414
    @classmethod
1✔
1415
    def bot_maybe_follow_back(bot_cls, user):
1✔
1416
        """Follow a user from a protocol bot user, if their protocol needs that.
1417

1418
        ...so that the protocol starts sending us their activities, if it needs
1419
        a follow for that (eg ActivityPub).
1420

1421
        Args:
1422
          user (User)
1423
        """
1424
        if not user.BOTS_FOLLOW_BACK:
1✔
1425
            return
1✔
1426

1427
        from web import Web
1✔
1428
        bot = Web.get_by_id(bot_cls.bot_user_id())
1✔
1429
        now = util.now().isoformat()
1✔
1430
        logger.info(f'Following {user.key.id()} back from bot user {bot.key.id()}')
1✔
1431

1432
        if not user.obj:
1✔
1433
            logger.info("  can't follow, user has no profile obj")
1✔
1434
            return
1✔
1435

1436
        target = user.target_for(user.obj)
1✔
1437
        follow_back_id = f'https://{bot.key.id()}/#follow-back-{user.key.id()}-{now}'
1✔
1438
        follow_back_as1 = {
1✔
1439
            'objectType': 'activity',
1440
            'verb': 'follow',
1441
            'id': follow_back_id,
1442
            'actor': bot.key.id(),
1443
            'object': user.key.id(),
1444
        }
1445
        common.create_task(queue='send', id=follow_back_id,
1✔
1446
                           our_as1=follow_back_as1, url=target,
1447
                           source_protocol='web', protocol=user.LABEL,
1448
                           user=bot.key.urlsafe())
1449

1450
    @classmethod
1✔
1451
    def handle_bare_object(cls, obj, *, authed_as, from_user):
1✔
1452
        """If obj is a bare object, wraps it in a create or update activity.
1453

1454
        Checks if we've seen it before.
1455

1456
        Args:
1457
          obj (models.Object)
1458
          authed_as (str): authenticated actor id who sent this activity
1459
          from_user (models.User): user (actor) this activity/object is from
1460

1461
        Returns:
1462
          models.Object: ``obj`` if it's an activity, otherwise a new object
1463
        """
1464
        is_actor = obj.type in as1.ACTOR_TYPES
1✔
1465
        if not is_actor and obj.type not in ('note', 'article', 'comment'):
1✔
1466
            return obj
1✔
1467

1468
        obj_actor = ids.normalize_user_id(id=as1.get_owner(obj.as1), proto=cls)
1✔
1469
        now = util.now().isoformat()
1✔
1470

1471
        # this is a raw post; wrap it in a create or update activity
1472
        if obj.changed or is_actor:
1✔
1473
            if obj.changed:
1✔
1474
                logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes')
1✔
1475
            else:
1476
                logger.info(f'Got actor profile object, wrapping in update')
1✔
1477
            id = f'{obj.key.id()}#bridgy-fed-update-{now}'
1✔
1478
            update_as1 = {
1✔
1479
                'objectType': 'activity',
1480
                'verb': 'update',
1481
                'id': id,
1482
                'actor': obj_actor,
1483
                'object': {
1484
                    # Mastodon requires the updated field for Updates, so
1485
                    # add a default value.
1486
                    # https://docs.joinmastodon.org/spec/activitypub/#supported-activities-for-statuses
1487
                    # https://socialhub.activitypub.rocks/t/what-could-be-the-reason-that-my-update-activity-does-not-work/2893/4
1488
                    # https://github.com/mastodon/documentation/pull/1150
1489
                    'updated': now,
1490
                    **obj.as1,
1491
                },
1492
            }
1493
            logger.debug(f'  AS1: {json_dumps(update_as1, indent=2)}')
1✔
1494
            return Object(id=id, our_as1=update_as1,
1✔
1495
                          source_protocol=obj.source_protocol)
1496

1497
        if (obj.new
1✔
1498
                # HACK: force query param here is specific to webmention
1499
                or 'force' in request.form):
1500
            create_id = f'{obj.key.id()}#bridgy-fed-create'
1✔
1501
            create_as1 = {
1✔
1502
                'objectType': 'activity',
1503
                'verb': 'post',
1504
                'id': create_id,
1505
                'actor': obj_actor,
1506
                'object': obj.as1,
1507
                'published': now,
1508
            }
1509
            logger.info(f'Wrapping in post')
1✔
1510
            logger.debug(f'  AS1: {json_dumps(create_as1, indent=2)}')
1✔
1511
            return Object(id=create_id, our_as1=create_as1,
1✔
1512
                          source_protocol=obj.source_protocol)
1513

1514
        error(f'{obj.key.id()} is unchanged, nothing to do', status=204)
1✔
1515

1516
    @classmethod
1✔
1517
    def deliver(from_cls, obj, from_user, crud_obj=None, to_proto=None):
1✔
1518
        """Delivers an activity to its external recipients.
1519

1520
        Args:
1521
          obj (models.Object): activity to deliver
1522
          from_user (models.User): user (actor) this activity is from
1523
          crud_obj (models.Object): if this is a create, update, or delete/undo
1524
            activity, the inner object that's being written, otherwise None.
1525
            (This object's ``notify`` and ``feed`` properties may be updated.)
1526
          to_proto (protocol.Protocol): optional; if provided, only deliver to
1527
            targets on this protocol
1528

1529
        Returns:
1530
          (str, int) tuple: Flask response
1531
        """
1532
        if to_proto:
1✔
1533
            logger.info(f'Only delivering to {to_proto.LABEL}')
1✔
1534

1535
        # find delivery targets. maps Target to Object or None
1536
        #
1537
        # ...then write the relevant object, since targets() has a side effect of
1538
        # setting the notify and feed properties (and dirty attribute)
1539
        targets = from_cls.targets(obj, from_user=from_user, crud_obj=crud_obj)
1✔
1540
        if to_proto:
1✔
1541
            targets = {t: obj for t, obj in targets.items()
1✔
1542
                       if t.protocol == to_proto.LABEL}
1543
        if not targets:
1✔
1544
            return r'No targets, nothing to do ¯\_(ツ)_/¯', 204
1✔
1545

1546
        # store object that targets() updated
1547
        if crud_obj and crud_obj.dirty:
1✔
1548
            crud_obj.put()
1✔
1549
        elif obj.type in STORE_AS1_TYPES and obj.dirty:
1✔
1550
            obj.put()
1✔
1551

1552
        obj_params = ({'obj_id': obj.key.id()} if obj.type in STORE_AS1_TYPES
1✔
1553
                      else obj.to_request())
1554

1555
        # sort targets so order is deterministic for tests, debugging, etc
1556
        sorted_targets = sorted(targets.items(), key=lambda t: t[0].uri)
1✔
1557

1558
        # enqueue send task for each targets
1559
        logger.info(f'Delivering to: {[t for t, _ in sorted_targets]}')
1✔
1560
        user = from_user.key.urlsafe()
1✔
1561
        for i, (target, orig_obj) in enumerate(sorted_targets):
1✔
1562
            orig_obj_id = orig_obj.key.id() if orig_obj else None
1✔
1563
            common.create_task(queue='send', url=target.uri, protocol=target.protocol,
1✔
1564
                               orig_obj_id=orig_obj_id, user=user, **obj_params)
1565

1566
        return 'OK', 202
1✔
1567

1568
    @classmethod
1✔
1569
    def targets(from_cls, obj, from_user, crud_obj=None, internal=False):
1✔
1570
        """Collects the targets to send a :class:`models.Object` to.
1571

1572
        Targets are both objects - original posts, events, etc - and actors.
1573

1574
        Args:
1575
          obj (models.Object)
1576
          from_user (User)
1577
          crud_obj (models.Object): if this is a create, update, or delete/undo
1578
            activity, the inner object that's being written, otherwise None.
1579
            (This object's ``notify`` and ``feed`` properties may be updated.)
1580
          internal (bool): whether this is a recursive internal call
1581

1582
        Returns:
1583
          dict: maps :class:`models.Target` to original (in response to)
1584
          :class:`models.Object`, if any, otherwise None
1585
        """
1586
        logger.debug('Finding recipients and their targets')
1✔
1587

1588
        # we should only have crud_obj iff this is a create or update
1589
        assert (crud_obj is not None) == (obj.type in ('post', 'update')), obj.type
1✔
1590
        write_obj = crud_obj or obj
1✔
1591
        write_obj.dirty = False
1✔
1592

1593
        target_uris = as1.targets(obj.as1)
1✔
1594
        orig_obj = None
1✔
1595
        targets = {}  # maps Target to Object or None
1✔
1596
        owner = as1.get_owner(obj.as1)
1✔
1597
        allow_opt_out = (obj.type == 'delete')
1✔
1598
        inner_obj_as1 = as1.get_object(obj.as1)
1✔
1599
        inner_obj_id = inner_obj_as1.get('id')
1✔
1600
        in_reply_tos = as1.get_ids(inner_obj_as1, 'inReplyTo')
1✔
1601
        quoted_posts = as1.quoted_posts(inner_obj_as1)
1✔
1602
        mentioned_urls = as1.mentions(inner_obj_as1)
1✔
1603
        is_reply = obj.type == 'comment' or in_reply_tos
1✔
1604
        is_self_reply = False
1✔
1605

1606
        original_ids = []
1✔
1607
        if is_reply:
1✔
1608
            original_ids = in_reply_tos
1✔
1609
        elif inner_obj_id:
1✔
1610
            if inner_obj_id == from_user.key.id():
1✔
1611
                inner_obj_id = from_user.profile_id()
1✔
1612
            original_ids = [inner_obj_id]
1✔
1613

1614
        original_objs = {}
1✔
1615
        for id in original_ids:
1✔
1616
            if proto := Protocol.for_id(id):
1✔
1617
                original_objs[id] = proto.load(id, raise_=False)
1✔
1618

1619
        # for AP, add in-reply-tos' mentions
1620
        # https://github.com/snarfed/bridgy-fed/issues/1608
1621
        # https://github.com/snarfed/bridgy-fed/issues/1218
1622
        orig_post_mentions = {}  # maps mentioned id to original post Object
1✔
1623
        for id in in_reply_tos:
1✔
1624
            if ((in_reply_to_obj := original_objs.get(id))
1✔
1625
                    and (proto := PROTOCOLS.get(in_reply_to_obj.source_protocol))
1626
                    and proto.SEND_REPLIES_TO_ORIG_POSTS_MENTIONS
1627
                    and (mentions := as1.mentions(in_reply_to_obj.as1))):
1628
                logger.info(f"Adding in-reply-to {id} 's mentions to targets: {mentions}")
1✔
1629
                target_uris.extend(mentions)
1✔
1630
                for mention in mentions:
1✔
1631
                    orig_post_mentions[mention] = in_reply_to_obj
1✔
1632

1633
        target_uris = sorted(set(target_uris))
1✔
1634
        logger.info(f'Raw targets: {target_uris}')
1✔
1635

1636
        # which protocols should we allow delivering to?
1637
        to_protocols = []
1✔
1638
        for label in (list(from_user.DEFAULT_ENABLED_PROTOCOLS)
1✔
1639
                      + from_user.enabled_protocols):
1640
            if not (proto := PROTOCOLS.get(label)):
1✔
1641
                report_error(f'unknown enabled protocol {label} for {from_user.key.id()}')
1✔
1642
                continue
1✔
1643

1644
            if proto.HAS_COPIES and (obj.type in ('update', 'delete', 'share', 'undo')
1✔
1645
                                     or is_reply):
1646
                origs_could_bridge = None
1✔
1647

1648
                for id in original_ids:
1✔
1649
                    if not (orig := original_objs.get(id)):
1✔
1650
                        continue
1✔
1651
                    elif isinstance(orig, proto):
1✔
UNCOV
1652
                        logger.info(f'Allowing {label} for original {id}')
×
UNCOV
1653
                        break
×
1654
                    elif orig.get_copy(proto):
1✔
1655
                        logger.info(f'Allowing {label}, original {id} was bridged there')
1✔
1656
                        break
1✔
1657

1658
                    if (origs_could_bridge is not False
1✔
1659
                            and (orig_author_id := as1.get_owner(orig.as1))
1660
                            and (orig_proto := PROTOCOLS.get(orig.source_protocol))
1661
                            and (orig_author := orig_proto.get_by_id(orig_author_id))):
1662
                        origs_could_bridge = orig_author.is_enabled(proto)
1✔
1663

1664
                else:
1665
                    msg = f"original object(s) {original_ids} weren't bridged to {label}"
1✔
1666
                    if (proto.LABEL not in from_user.DEFAULT_ENABLED_PROTOCOLS
1✔
1667
                            and origs_could_bridge):
1668
                        # retry later; original obj may still be bridging
1669
                        # TODO: limit to brief window, eg no older than 2h? 1d?
1670
                        error(msg, status=304)
1✔
1671

1672
                    logger.info(msg)
1✔
1673
                    continue
1✔
1674

1675
            util.add(to_protocols, proto)
1✔
1676

1677
        # process direct targets
1678
        for target_id in target_uris:
1✔
1679
            target_proto = Protocol.for_id(target_id)
1✔
1680
            if not target_proto:
1✔
1681
                logger.info(f"Can't determine protocol for {target_id}")
1✔
1682
                continue
1✔
1683
            elif target_proto.is_blocklisted(target_id):
1✔
1684
                logger.debug(f'{target_id} is blocklisted')
1✔
1685
                continue
1✔
1686

1687
            orig_obj = target_proto.load(target_id, raise_=False)
1✔
1688
            if not orig_obj or not orig_obj.as1:
1✔
1689
                logger.info(f"Couldn't load {target_id}")
1✔
1690
                continue
1✔
1691

1692
            target_author_key = (target_proto(id=target_id).key
1✔
1693
                                 if target_id in mentioned_urls
1694
                                 else target_proto.actor_key(orig_obj))
1695
            if not from_user.is_enabled(target_proto):
1✔
1696
                # if author isn't bridged and target user is, DM a prompt and
1697
                # add a notif for the target user
1698
                if (target_id in (in_reply_tos + quoted_posts + mentioned_urls)
1✔
1699
                        and target_author_key):
1700
                    if target_author := target_author_key.get():
1✔
1701
                        if target_author.is_enabled(from_cls):
1✔
1702
                            notifications.add_notification(target_author, write_obj)
1✔
1703
                            verb, noun = (
1✔
1704
                                ('replied to', 'replies') if target_id in in_reply_tos
1705
                                else ('quoted', 'quotes') if target_id in quoted_posts
1706
                                else ('mentioned', 'mentions'))
1707
                            dms.maybe_send(from_=target_proto, to_user=from_user,
1✔
1708
                                           type='replied_to_bridged_user', text=f"""\
1709
Hi! You <a href="{inner_obj_as1.get('url') or inner_obj_id}">recently {verb}</a> {target_author.user_link()}, who's bridged here from {target_proto.PHRASE}. If you want them to see your {noun}, you can bridge your account into {target_proto.PHRASE} by following this account. <a href="https://fed.brid.gy/docs">See the docs</a> for more information.""")
1710

1711
                continue
1✔
1712

1713
            # deliver self-replies to followers
1714
            # https://github.com/snarfed/bridgy-fed/issues/639
1715
            if target_id in in_reply_tos and owner == as1.get_owner(orig_obj.as1):
1✔
1716
                is_self_reply = True
1✔
1717
                logger.info(f'self reply!')
1✔
1718

1719
            # also add copies' targets
1720
            for copy in orig_obj.copies:
1✔
1721
                proto = PROTOCOLS[copy.protocol]
1✔
1722
                if proto in to_protocols:
1✔
1723
                    # copies generally won't have their own Objects
1724
                    if target := proto.target_for(Object(id=copy.uri)):
1✔
1725
                        logger.debug(f'Adding target {target} for copy {copy.uri} of original {target_id}')
1✔
1726
                        targets[Target(protocol=copy.protocol, uri=target)] = orig_obj
1✔
1727

1728
            if target_proto == from_cls:
1✔
1729
                logger.debug(f'Skipping same-protocol target {target_id}')
1✔
1730
                continue
1✔
1731

1732
            target = target_proto.target_for(orig_obj)
1✔
1733
            if not target:
1✔
1734
                # TODO: surface errors like this somehow?
UNCOV
1735
                logger.error(f"Can't find delivery target for {target_id}")
×
UNCOV
1736
                continue
×
1737

1738
            logger.debug(f'Target for {target_id} is {target}')
1✔
1739
            # only use orig_obj for inReplyTos, like/repost objects, reply's original
1740
            # post's mentions, etc
1741
            # https://github.com/snarfed/bridgy-fed/issues/1237
1742
            target_obj = None
1✔
1743
            if target_id in in_reply_tos + as1.get_ids(obj.as1, 'object'):
1✔
1744
                target_obj = orig_obj
1✔
1745
            elif target_id in orig_post_mentions:
1✔
1746
                target_obj = orig_post_mentions[target_id]
1✔
1747
            targets[Target(protocol=target_proto.LABEL, uri=target)] = target_obj
1✔
1748

1749
            if target_author_key:
1✔
1750
                logger.debug(f'Recipient is {target_author_key}')
1✔
1751
                if write_obj.add('notify', target_author_key):
1✔
1752
                    write_obj.dirty = True
1✔
1753

1754
        if obj.type == 'undo':
1✔
1755
            logger.debug('Object is an undo; adding targets for inner object')
1✔
1756
            if set(inner_obj_as1.keys()) == {'id'}:
1✔
1757
                inner_obj = from_cls.load(inner_obj_id, raise_=False)
1✔
1758
            else:
1759
                inner_obj = Object(id=inner_obj_id, our_as1=inner_obj_as1)
1✔
1760
            if inner_obj:
1✔
1761
                targets.update(from_cls.targets(inner_obj, from_user=from_user,
1✔
1762
                                                internal=True))
1763

1764
        logger.info(f'Direct targets: {[t.uri for t in targets.keys()]}')
1✔
1765

1766
        # deliver to followers, if appropriate
1767
        user_key = from_cls.actor_key(obj, allow_opt_out=allow_opt_out)
1✔
1768
        if not user_key:
1✔
1769
            logger.info("Can't tell who this is from! Skipping followers.")
1✔
1770
            return targets
1✔
1771

1772
        followers = []
1✔
1773
        if (obj.type in ('post', 'update', 'delete', 'move', 'share', 'undo')
1✔
1774
                and (not is_reply or is_self_reply)):
1775
            logger.info(f'Delivering to followers of {user_key}')
1✔
1776
            followers = []
1✔
1777
            for f in Follower.query(Follower.to == user_key,
1✔
1778
                                    Follower.status == 'active'):
1779
                proto = PROTOCOLS_BY_KIND[f.from_.kind()]
1✔
1780
                # skip protocol bot users
1781
                if (not Protocol.for_bridgy_subdomain(f.from_.id())
1✔
1782
                        # skip protocols this user hasn't enabled, or where the base
1783
                        # object of this activity hasn't been bridged
1784
                        and proto in to_protocols
1785
                        # we deliver to HAS_COPIES protocols separately, below. we
1786
                        # assume they have follower-independent targets.
1787
                        and not (proto.HAS_COPIES and proto.DEFAULT_TARGET)):
1788
                    followers.append(f)
1✔
1789

1790
            user_keys = [f.from_ for f in followers]
1✔
1791
            users = [u for u in ndb.get_multi(user_keys) if u]
1✔
1792
            User.load_multi(users)
1✔
1793

1794
            if (not followers and
1✔
1795
                (util.domain_or_parent_in(from_user.key.id(), LIMITED_DOMAINS)
1796
                 or util.domain_or_parent_in(obj.key.id(), LIMITED_DOMAINS))):
1797
                logger.info(f'skipping, {from_user.key.id()} is on a limited domain and has no followers')
1✔
1798
                return {}
1✔
1799

1800
            # add to followers' feeds, if any
1801
            if not internal and obj.type in ('post', 'update', 'share'):
1✔
1802
                if write_obj.type not in as1.ACTOR_TYPES:
1✔
1803
                    write_obj.feed = [u.key for u in users if u.USES_OBJECT_FEED]
1✔
1804
                    if write_obj.feed:
1✔
1805
                        write_obj.dirty = True
1✔
1806

1807
            # collect targets for followers
1808
            for user in users:
1✔
1809
                # TODO: should we pass remote=False through here to Protocol.load?
1810
                target = user.target_for(user.obj, shared=True) if user.obj else None
1✔
1811
                if not target:
1✔
1812
                    # logger.error(f'Follower {user.key} has no delivery target')
1813
                    continue
1✔
1814

1815
                # normalize URL (lower case hostname, etc)
1816
                # ...but preserve our PDS URL without trailing slash in path
1817
                # https://atproto.com/specs/did#did-documents
1818
                target = util.dedupe_urls([target], trailing_slash=False)[0]
1✔
1819

1820
                targets[Target(protocol=user.LABEL, uri=target)] = \
1✔
1821
                    Object.get_by_id(inner_obj_id) if obj.type == 'share' else None
1822

1823
        # deliver to enabled HAS_COPIES protocols proactively
1824
        if obj.type in ('post', 'update', 'delete', 'share'):
1✔
1825
            for proto in to_protocols:
1✔
1826
                if proto.HAS_COPIES and proto.DEFAULT_TARGET:
1✔
1827
                    logger.info(f'user has {proto.LABEL} enabled, adding {proto.DEFAULT_TARGET}')
1✔
1828
                    targets.setdefault(
1✔
1829
                        Target(protocol=proto.LABEL, uri=proto.DEFAULT_TARGET), None)
1830

1831
        # de-dupe targets, discard same-domain
1832
        # maps string target URL to (Target, Object) tuple
1833
        candidates = {t.uri: (t, obj) for t, obj in targets.items()}
1✔
1834
        # maps Target to Object or None
1835
        targets = {}
1✔
1836
        source_domains = [
1✔
1837
            util.domain_from_link(url) for url in
1838
            (obj.as1.get('id'), obj.as1.get('url'), as1.get_owner(obj.as1))
1839
            if util.is_web(url)
1840
        ]
1841
        for url in sorted(util.dedupe_urls(
1✔
1842
                candidates.keys(),
1843
                # preserve our PDS URL without trailing slash in path
1844
                # https://atproto.com/specs/did#did-documents
1845
                trailing_slash=False)):
1846
            if util.is_web(url) and util.domain_from_link(url) in source_domains:
1✔
UNCOV
1847
                logger.info(f'Skipping same-domain target {url}')
×
UNCOV
1848
                continue
×
1849
            target, obj = candidates[url]
1✔
1850
            targets[target] = obj
1✔
1851

1852
        return targets
1✔
1853

1854
    @classmethod
1✔
1855
    def load(cls, id, remote=None, local=True, raise_=True, **kwargs):
1✔
1856
        """Loads and returns an Object from datastore or HTTP fetch.
1857

1858
        Sets the :attr:`new` and :attr:`changed` attributes if we know either
1859
        one for the loaded object, ie local is True and remote is True or None.
1860

1861
        Args:
1862
          id (str)
1863
          remote (bool): whether to fetch the object over the network. If True,
1864
            fetches even if we already have the object stored, and updates our
1865
            stored copy. If False and we don't have the object stored, returns
1866
            None. Default (None) means to fetch over the network only if we
1867
            don't already have it stored.
1868
          local (bool): whether to load from the datastore before
1869
            fetching over the network. If False, still stores back to the
1870
            datastore after a successful remote fetch.
1871
          raise_ (bool): if False, catches any :class:`request.RequestException`
1872
            or :class:`HTTPException` raised by :meth:`fetch()` and returns
1873
            ``None`` instead
1874
          kwargs: passed through to :meth:`fetch()`
1875

1876
        Returns:
1877
          models.Object: loaded object, or None if it isn't fetchable, eg a
1878
          non-URL string for Web, or ``remote`` is False and it isn't in the
1879
          datastore
1880

1881
        Raises:
1882
          requests.HTTPError: anything that :meth:`fetch` raises, if ``raise_``
1883
            is True
1884
        """
1885
        assert id
1✔
1886
        assert local or remote is not False
1✔
1887
        # logger.debug(f'Loading Object {id} local={local} remote={remote}')
1888

1889
        id = ids.normalize_object_id(id=id, proto=cls)
1✔
1890

1891
        obj = orig_as1 = None
1✔
1892
        if local:
1✔
1893
            obj = Object.get_by_id(id)
1✔
1894
            if not obj:
1✔
1895
                # logger.debug(f' {id} not in datastore')
1896
                pass
1✔
1897
            elif obj.as1 or obj.raw or obj.deleted:
1✔
1898
                # logger.debug(f'  {id} got from datastore')
1899
                obj.new = False
1✔
1900

1901
        if remote is False:
1✔
1902
            return obj
1✔
1903
        elif remote is None and obj:
1✔
1904
            if obj.updated < util.as_utc(util.now() - OBJECT_REFRESH_AGE):
1✔
1905
                # logger.debug(f'  last updated {obj.updated}, refreshing')
1906
                pass
1✔
1907
            else:
1908
                return obj
1✔
1909

1910
        if obj:
1✔
1911
            orig_as1 = obj.as1
1✔
1912
            obj.our_as1 = None
1✔
1913
            obj.new = False
1✔
1914
        else:
1915
            obj = Object(id=id)
1✔
1916
            if local:
1✔
1917
                # logger.debug(f'  {id} not in datastore')
1918
                obj.new = True
1✔
1919
                obj.changed = False
1✔
1920

1921
        try:
1✔
1922
            fetched = cls.fetch(obj, **kwargs)
1✔
1923
        except (RequestException, HTTPException) as e:
1✔
1924
            if raise_:
1✔
1925
                raise
1✔
1926
            util.interpret_http_exception(e)
1✔
1927
            return None
1✔
1928

1929
        if not fetched:
1✔
1930
            return None
1✔
1931

1932
        # https://stackoverflow.com/a/3042250/186123
1933
        size = len(_entity_to_protobuf(obj)._pb.SerializeToString())
1✔
1934
        if size > models.MAX_ENTITY_SIZE:
1✔
1935
            logger.warning(f'Object is too big! {size} bytes is over {models.MAX_ENTITY_SIZE}')
1✔
1936
            return None
1✔
1937

1938
        obj.resolve_ids()
1✔
1939
        obj.normalize_ids()
1✔
1940

1941
        if obj.new is False:
1✔
1942
            obj.changed = obj.activity_changed(orig_as1)
1✔
1943

1944
        if obj.source_protocol not in (cls.LABEL, cls.ABBREV):
1✔
1945
            if obj.source_protocol:
1✔
UNCOV
1946
                logger.warning(f'Object {obj.key.id()} changed protocol from {obj.source_protocol} to {cls.LABEL} ?!')
×
1947
            obj.source_protocol = cls.LABEL
1✔
1948

1949
        obj.put()
1✔
1950
        return obj
1✔
1951

1952
    @classmethod
1✔
1953
    def check_supported(cls, obj, direction):
1✔
1954
        """If this protocol doesn't support this activity, raises HTTP 204.
1955

1956
        Also reports an error.
1957

1958
        (This logic is duplicated in some protocols, eg ActivityPub, so that
1959
        they can short circuit out early. It generally uses their native formats
1960
        instead of AS1, before an :class:`models.Object` is created.)
1961

1962
        Args:
1963
          obj (Object)
1964
          direction (str): ``'receive'`` or  ``'send'``
1965

1966
        Raises:
1967
          werkzeug.HTTPException: if this protocol doesn't support this object
1968
        """
1969
        assert direction in ('receive', 'send')
1✔
1970
        if not obj.type:
1✔
UNCOV
1971
            return
×
1972

1973
        inner = as1.get_object(obj.as1)
1✔
1974
        inner_type = as1.object_type(inner) or ''
1✔
1975
        if (obj.type not in cls.SUPPORTED_AS1_TYPES
1✔
1976
            or (obj.type in as1.CRUD_VERBS
1977
                and inner_type
1978
                and inner_type not in cls.SUPPORTED_AS1_TYPES)):
1979
            error(f"Bridgy Fed for {cls.LABEL} doesn't support {obj.type} {inner_type} yet", status=204)
1✔
1980

1981
        # don't allow posts with blank content and no image/video/audio
1982
        crud_obj = (as1.get_object(obj.as1) if obj.type in ('post', 'update')
1✔
1983
                    else obj.as1)
1984
        if (crud_obj.get('objectType') in as1.POST_TYPES
1✔
1985
                and not util.get_url(crud_obj, key='image')
1986
                and not any(util.get_urls(crud_obj, 'attachments', inner_key='stream'))
1987
                # TODO: handle articles with displayName but not content
1988
                and not source.html_to_text(crud_obj.get('content')).strip()):
1989
            error('Blank content and no image or video or audio', status=204)
1✔
1990

1991
        # receiving DMs is only allowed to protocol bot accounts
1992
        if direction == 'receive':
1✔
1993
            if recip := as1.recipient_if_dm(obj.as1):
1✔
1994
                owner = as1.get_owner(obj.as1)
1✔
1995
                if (not cls.SUPPORTS_DMS or (recip not in common.bot_user_ids()
1✔
1996
                                             and owner not in common.bot_user_ids())):
1997
                    # reply and say DMs aren't supported
1998
                    from_proto = PROTOCOLS.get(obj.source_protocol)
1✔
1999
                    to_proto = Protocol.for_id(recip)
1✔
2000
                    if owner and from_proto and to_proto:
1✔
2001
                        if ((from_user := from_proto.get_or_create(id=owner))
1✔
2002
                                and (to_user := to_proto.get_or_create(id=recip))):
2003
                            in_reply_to = (inner.get('id') if obj.type == 'post'
1✔
2004
                                           else obj.as1.get('id'))
2005
                            text = f"Hi! Sorry, this account is bridged from {to_user.PHRASE}, so it doesn't support DMs. Try getting in touch another way!"
1✔
2006
                            type = f'dms_not_supported-{to_user.key.id()}'
1✔
2007
                            dms.maybe_send(from_=to_user, to_user=from_user,
1✔
2008
                                           text=text, type=type,
2009
                                           in_reply_to=in_reply_to)
2010

2011
                    error("Bridgy Fed doesn't support DMs", status=204)
1✔
2012

2013
            # check that this activity is public. only do this for some activities,
2014
            # not eg likes or follows, since Mastodon doesn't currently mark those
2015
            # as explicitly public.
2016
            elif (obj.type in set(('post', 'update')) | as1.POST_TYPES | as1.ACTOR_TYPES
1✔
2017
                  and not util.domain_or_parent_in(crud_obj.get('id'), NON_PUBLIC_DOMAINS)
2018
                  and not as1.is_public(obj.as1, unlisted=False)):
2019
                error('Bridgy Fed only supports public activities', status=204)
1✔
2020

2021

2022
@cloud_tasks_only(log=None)
1✔
2023
def receive_task():
1✔
2024
    """Task handler for a newly received :class:`models.Object`.
2025

2026
    Calls :meth:`Protocol.receive` with the form parameters.
2027

2028
    Parameters:
2029
      authed_as (str): passed to :meth:`Protocol.receive`
2030
      obj_id (str): key id of :class:`models.Object` to handle
2031
      received_at (str, ISO 8601 timestamp): when we first saw (received)
2032
        this activity
2033
      *: If ``obj_id`` is unset, all other parameters are properties for a new
2034
        :class:`models.Object` to handle
2035

2036
    TODO: migrate incoming webmentions to this. See how we did it for AP. The
2037
    difficulty is that parts of :meth:`protocol.Protocol.receive` depend on
2038
    setup in :func:`web.webmention`, eg :class:`models.Object` with ``new`` and
2039
    ``changed``, HTTP request details, etc. See stash for attempt at this for
2040
    :class:`web.Web`.
2041
    """
2042
    common.log_request()
1✔
2043
    form = request.form.to_dict()
1✔
2044

2045
    authed_as = form.pop('authed_as', None)
1✔
2046
    internal = (authed_as == common.PRIMARY_DOMAIN
1✔
2047
                or authed_as in common.PROTOCOL_DOMAINS)
2048

2049
    obj = Object.from_request()
1✔
2050
    assert obj
1✔
2051
    assert obj.source_protocol
1✔
2052
    obj.new = True
1✔
2053

2054
    if received_at := form.pop('received_at', None):
1✔
2055
        received_at = datetime.fromisoformat(received_at)
1✔
2056

2057
    try:
1✔
2058
        return PROTOCOLS[obj.source_protocol].receive(
1✔
2059
            obj=obj, authed_as=authed_as, internal=internal, received_at=received_at)
2060
    except RequestException as e:
1✔
2061
        util.interpret_http_exception(e)
1✔
2062
        error(e, status=304)
1✔
2063
    except ValueError as e:
1✔
UNCOV
2064
        logger.warning(e, exc_info=True)
×
UNCOV
2065
        error(e, status=304)
×
2066

2067

2068
@cloud_tasks_only(log=None)
1✔
2069
def send_task():
1✔
2070
    """Task handler for sending an activity to a single specific destination.
2071

2072
    Calls :meth:`Protocol.send` with the form parameters.
2073

2074
    Parameters:
2075
      protocol (str): :class:`Protocol` to send to
2076
      url (str): destination URL to send to
2077
      obj_id (str): key id of :class:`models.Object` to send
2078
      orig_obj_id (str): optional, :class:`models.Object` key id of the
2079
        "original object" that this object refers to, eg replies to or reposts
2080
        or likes
2081
      user (url-safe google.cloud.ndb.key.Key): :class:`models.User` (actor)
2082
        this activity is from
2083
      *: If ``obj_id`` is unset, all other parameters are properties for a new
2084
        :class:`models.Object` to handle
2085
    """
2086
    common.log_request()
1✔
2087

2088
    # prepare
2089
    form = request.form.to_dict()
1✔
2090
    url = form.get('url')
1✔
2091
    protocol = form.get('protocol')
1✔
2092
    if not url or not protocol:
1✔
2093
        logger.warning(f'Missing protocol or url; got {protocol} {url}')
1✔
2094
        return '', 204
1✔
2095

2096
    target = Target(uri=url, protocol=protocol)
1✔
2097
    obj = Object.from_request()
1✔
2098
    assert obj and obj.key and obj.key.id()
1✔
2099

2100
    PROTOCOLS[protocol].check_supported(obj, 'send')
1✔
2101
    allow_opt_out = (obj.type == 'delete')
1✔
2102

2103
    user = None
1✔
2104
    if user_key := form.get('user'):
1✔
2105
        key = ndb.Key(urlsafe=user_key)
1✔
2106
        # use get_by_id so that we follow use_instead
2107
        user = PROTOCOLS_BY_KIND[key.kind()].get_by_id(
1✔
2108
            key.id(), allow_opt_out=allow_opt_out)
2109

2110
    # send
2111
    delay = ''
1✔
2112
    if request.headers.get('X-AppEngine-TaskRetryCount') == '0' and obj.created:
1✔
2113
        delay_s = int((util.now().replace(tzinfo=None) - obj.created).total_seconds())
1✔
2114
        delay = f'({delay_s} s behind)'
1✔
2115
    logger.info(f'Sending {obj.source_protocol} {obj.type} {obj.key.id()} to {protocol} {url} {delay}')
1✔
2116
    logger.debug(f'  AS1: {json_dumps(obj.as1, indent=2)}')
1✔
2117
    sent = None
1✔
2118
    try:
1✔
2119
        sent = PROTOCOLS[protocol].send(obj, url, from_user=user,
1✔
2120
                                        orig_obj_id=form.get('orig_obj_id'))
2121
    except BaseException as e:
1✔
2122
        code, body = util.interpret_http_exception(e)
1✔
2123
        if not code and not body:
1✔
2124
            raise
1✔
2125

2126
    if sent is False:
1✔
2127
        logger.info(f'Failed sending!')
1✔
2128

2129
    return '', 200 if sent else 204 if sent is False else 304
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc