• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

snarfed / bridgy-fed / 878ced7f-3b32-4aaf-9280-63d2666620db

17 Oct 2025 03:56AM UTC coverage: 92.855% (-0.002%) from 92.857%
878ced7f-3b32-4aaf-9280-63d2666620db

push

circleci

snarfed
fix integration test test_nostr_follow_activitypub_bot_user_invalid_nip05

5939 of 6396 relevant lines covered (92.85%)

0.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.75
/protocol.py
1
"""Base protocol class and common code."""
2
import copy
1✔
3
from datetime import datetime, timedelta, timezone
1✔
4
import logging
1✔
5
import os
1✔
6
import re
1✔
7
from threading import Lock
1✔
8
from urllib.parse import urljoin, urlparse
1✔
9

10
from cachetools import cached, LRUCache
1✔
11
from flask import request
1✔
12
from google.cloud import ndb
1✔
13
from google.cloud.ndb import OR
1✔
14
from google.cloud.ndb.model import _entity_to_protobuf
1✔
15
from granary import as1, as2, source
1✔
16
from granary.source import html_to_text
1✔
17
from oauth_dropins.webutil.appengine_info import DEBUG
1✔
18
from oauth_dropins.webutil.flask_util import cloud_tasks_only
1✔
19
from oauth_dropins.webutil import models
1✔
20
from oauth_dropins.webutil import util
1✔
21
from oauth_dropins.webutil.util import json_dumps, json_loads
1✔
22
from requests import RequestException
1✔
23
import werkzeug.exceptions
1✔
24
from werkzeug.exceptions import BadGateway, HTTPException
1✔
25

26
import common
1✔
27
from common import (
1✔
28
    DOMAIN_BLOCKLIST,
29
    DOMAIN_RE,
30
    DOMAINS,
31
    ErrorButDoNotRetryTask,
32
    PRIMARY_DOMAIN,
33
    PROTOCOL_DOMAINS,
34
    report_error,
35
    subdomain_wrap,
36
)
37
import dms
1✔
38
import ids
1✔
39
import memcache
1✔
40
from models import (
1✔
41
    DM,
42
    Follower,
43
    Object,
44
    PROTOCOLS,
45
    PROTOCOLS_BY_KIND,
46
    Target,
47
    User,
48
)
49
import notifications
1✔
50

51
OBJECT_REFRESH_AGE = timedelta(days=30)
1✔
52
DELETE_TASK_DELAY = timedelta(minutes=1)
1✔
53
CREATE_MAX_AGE = timedelta(weeks=2)
1✔
54
# WARNING: keep this below the receive queue's min_backoff_seconds in queue.yaml!
55
MEMCACHE_LEASE_EXPIRATION = timedelta(seconds=25)
1✔
56

57
# require a follow for users on these domains before we deliver anything from
58
# them other than their profile
59
LIMITED_DOMAINS = (os.getenv('LIMITED_DOMAINS', '').split()
1✔
60
                   or util.load_file_lines('limited_domains'))
61

62
DONT_STORE_AS1_TYPES = as1.CRUD_VERBS | set((
1✔
63
    'accept',
64
    'reject',
65
    'stop-following',
66
    'undo',
67
))
68
STORE_AS1_TYPES = (as1.ACTOR_TYPES | as1.POST_TYPES | as1.VERBS_WITH_OBJECT
1✔
69
                   - DONT_STORE_AS1_TYPES)
70

71
logger = logging.getLogger(__name__)
1✔
72

73

74
def error(*args, status=299, **kwargs):
1✔
75
    """Default HTTP status code to 299 to prevent retrying task."""
76
    return common.error(*args, status=status, **kwargs)
1✔
77

78

79
def activity_id_memcache_key(id):
1✔
80
    return memcache.key(f'receive-{id}')
1✔
81

82

83
class Protocol:
1✔
84
    """Base protocol class. Not to be instantiated; classmethods only."""
85
    ABBREV = None
1✔
86
    """str: lower case abbreviation, used in URL paths"""
1✔
87
    PHRASE = None
1✔
88
    """str: human-readable name or phrase. Used in phrases like ``Follow this person on {PHRASE}``"""
1✔
89
    OTHER_LABELS = ()
1✔
90
    """sequence of str: label aliases"""
1✔
91
    LOGO_EMOJI = ''
1✔
92
    """str: logo emoji, if any"""
1✔
93
    LOGO_HTML = ''
1✔
94
    """str: logo ``<img>`` tag, if any"""
1✔
95
    CONTENT_TYPE = None
1✔
96
    """str: MIME type of this protocol's native data format, appropriate for the ``Content-Type`` HTTP header."""
1✔
97
    HAS_COPIES = False
1✔
98
    """bool: whether this protocol is push and needs us to proactively create "copy" users and objects, as opposed to pulling converted objects on demand"""
1✔
99
    DEFAULT_TARGET = None
1✔
100
    """str: optional, the default target URI to send this protocol's activities to. May be used as the "shared" target. Often only set if ``HAS_COPIES`` is true."""
1✔
101
    REQUIRES_AVATAR = False
1✔
102
    """bool: whether accounts on this protocol are required to have a profile picture. If they don't, their ``User.status`` will be ``blocked``."""
1✔
103
    REQUIRES_NAME = False
1✔
104
    """bool: whether accounts on this protocol are required to have a profile name that's different than their handle or id. If they don't, their ``User.status`` will be ``blocked``."""
1✔
105
    REQUIRES_OLD_ACCOUNT = False
1✔
106
    """bool: whether accounts on this protocol are required to be at least :const:`common.OLD_ACCOUNT_AGE` old. If their profile includes creation date and it's not old enough, their ``User.status`` will be ``blocked``."""
1✔
107
    DEFAULT_ENABLED_PROTOCOLS = ()
1✔
108
    """sequence of str: labels of other protocols that are automatically enabled for this protocol to bridge into"""
1✔
109
    DEFAULT_SERVE_USER_PAGES = False
1✔
110
    """bool: whether to serve user pages for all of this protocol's users on the fed.brid.gy. If ``False``, user pages will only be served for users who have explictly opted in."""
1✔
111
    SUPPORTED_AS1_TYPES = ()
1✔
112
    """sequence of str: AS1 objectTypes and verbs that this protocol supports receiving and sending"""
1✔
113
    SUPPORTS_DMS = False
1✔
114
    """bool: whether this protocol can receive DMs (chat messages)"""
1✔
115
    USES_OBJECT_FEED = False
1✔
116
    """bool: whether to store followers on this protocol in :attr:`Object.feed`."""
1✔
117
    HTML_PROFILES = False
1✔
118
    """bool: whether this protocol supports HTML in profile descriptions. If False, profile descriptions should be plain text."""
1✔
119
    SEND_REPLIES_TO_ORIG_POSTS_MENTIONS = False
1✔
120
    """bool: whether replies to this protocol should include the original post's mentions as delivery targets"""
1✔
121
    BOTS_FOLLOW_BACK = False
1✔
122
    """bool: when a user on this protocol follows a bot user to enable bridging, does the bot follow them back?"""
1✔
123

124
    @classmethod
1✔
125
    @property
1✔
126
    def LABEL(cls):
1✔
127
        """str: human-readable lower case name of this protocol, eg ``'activitypub``"""
128
        return cls.__name__.lower()
1✔
129

130
    @staticmethod
1✔
131
    def for_request(fed=None):
1✔
132
        """Returns the protocol for the current request.
133

134
        ...based on the request's hostname.
135

136
        Args:
137
          fed (str or protocol.Protocol): protocol to return if the current
138
            request is on ``fed.brid.gy``
139

140
        Returns:
141
          Protocol: protocol, or None if the provided domain or request hostname
142
          domain is not a subdomain of ``brid.gy`` or isn't a known protocol
143
        """
144
        return Protocol.for_bridgy_subdomain(request.host, fed=fed)
1✔
145

146
    @staticmethod
1✔
147
    def for_bridgy_subdomain(domain_or_url, fed=None):
1✔
148
        """Returns the protocol for a brid.gy subdomain.
149

150
        Args:
151
          domain_or_url (str)
152
          fed (str or protocol.Protocol): protocol to return if the current
153
            request is on ``fed.brid.gy``
154

155
        Returns:
156
          class: :class:`Protocol` subclass, or None if the provided domain or request
157
          hostname domain is not a subdomain of ``brid.gy`` or isn't a known
158
          protocol
159
        """
160
        domain = (util.domain_from_link(domain_or_url, minimize=False)
1✔
161
                  if util.is_web(domain_or_url)
162
                  else domain_or_url)
163

164
        if domain == common.PRIMARY_DOMAIN or domain in common.LOCAL_DOMAINS:
1✔
165
            return PROTOCOLS[fed] if isinstance(fed, str) else fed
1✔
166
        elif domain and domain.endswith(common.SUPERDOMAIN):
1✔
167
            label = domain.removesuffix(common.SUPERDOMAIN)
1✔
168
            return PROTOCOLS.get(label)
1✔
169

170
    @classmethod
1✔
171
    def owns_id(cls, id):
1✔
172
        """Returns whether this protocol owns the id, or None if it's unclear.
173

174
        To be implemented by subclasses.
175

176
        IDs are string identities that uniquely identify users, and are intended
177
        primarily to be machine readable and usable. Compare to handles, which
178
        are human-chosen, human-meaningful, and often but not always unique.
179

180
        Some protocols' ids are more or less deterministic based on the id
181
        format, eg AT Protocol owns ``at://`` URIs. Others, like http(s) URLs,
182
        could be owned by eg Web or ActivityPub.
183

184
        This should be a quick guess without expensive side effects, eg no
185
        external HTTP fetches to fetch the id itself or otherwise perform
186
        discovery.
187

188
        Returns False if the id's domain is in :const:`common.DOMAIN_BLOCKLIST`.
189

190
        Args:
191
          id (str)
192

193
        Returns:
194
          bool or None:
195
        """
196
        return False
1✔
197

198
    @classmethod
1✔
199
    def owns_handle(cls, handle, allow_internal=False):
1✔
200
        """Returns whether this protocol owns the handle, or None if it's unclear.
201

202
        To be implemented by subclasses.
203

204
        Handles are string identities that are human-chosen, human-meaningful,
205
        and often but not always unique. Compare to IDs, which uniquely identify
206
        users, and are intended primarily to be machine readable and usable.
207

208
        Some protocols' handles are more or less deterministic based on the id
209
        format, eg ActivityPub (technically WebFinger) handles are
210
        ``@user@instance.com``. Others, like domains, could be owned by eg Web,
211
        ActivityPub, AT Protocol, or others.
212

213
        This should be a quick guess without expensive side effects, eg no
214
        external HTTP fetches to fetch the id itself or otherwise perform
215
        discovery.
216

217
        Args:
218
          handle (str)
219
          allow_internal (bool): whether to return False for internal domains
220
            like ``fed.brid.gy``, ``bsky.brid.gy``, etc
221

222
        Returns:
223
          bool or None
224
        """
225
        return False
1✔
226

227
    @classmethod
1✔
228
    def handle_to_id(cls, handle):
1✔
229
        """Converts a handle to an id.
230

231
        To be implemented by subclasses.
232

233
        May incur network requests, eg DNS queries or HTTP requests. Avoids
234
        blocked or opted out users.
235

236
        Args:
237
          handle (str)
238

239
        Returns:
240
          str: corresponding id, or None if the handle can't be found
241
        """
242
        raise NotImplementedError()
×
243

244
    @classmethod
1✔
245
    def key_for(cls, id, allow_opt_out=False):
1✔
246
        """Returns the :class:`google.cloud.ndb.Key` for a given id's :class:`models.User`.
247

248
        To be implemented by subclasses. Canonicalizes the id if necessary.
249

250
        If called via `Protocol.key_for`, infers the appropriate protocol with
251
        :meth:`for_id`. If called with a concrete subclass, uses that subclass
252
        as is.
253

254
        Args:
255
          id (str):
256
          allow_opt_out (bool): whether to allow users who are currently opted out
257

258
        Returns:
259
          google.cloud.ndb.Key: matching key, or None if the given id is not a
260
          valid :class:`User` id for this protocol.
261
        """
262
        if cls == Protocol:
1✔
263
            proto = Protocol.for_id(id)
1✔
264
            return proto.key_for(id, allow_opt_out=allow_opt_out) if proto else None
1✔
265

266
        # load user so that we follow use_instead
267
        existing = cls.get_by_id(id, allow_opt_out=True)
1✔
268
        if existing:
1✔
269
            if existing.status and not allow_opt_out:
1✔
270
                return None
1✔
271
            return existing.key
1✔
272

273
        return cls(id=id).key
1✔
274

275
    @staticmethod
1✔
276
    def _for_id_memcache_key(id, remote=None):
1✔
277
        """If id is a URL, uses its domain, otherwise returns None.
278

279
        Args:
280
          id (str)
281

282
        Returns:
283
          (str domain, bool remote) or None
284
        """
285
        domain = util.domain_from_link(id)
1✔
286
        if domain in PROTOCOL_DOMAINS:
1✔
287
            return id
1✔
288
        elif remote and util.is_web(id):
1✔
289
            return domain
1✔
290

291
    @cached(LRUCache(20000), lock=Lock())
1✔
292
    @memcache.memoize(key=_for_id_memcache_key, write=lambda id, remote=True: remote,
1✔
293
                      version=3)
294
    @staticmethod
1✔
295
    def for_id(id, remote=True):
1✔
296
        """Returns the protocol for a given id.
297

298
        Args:
299
          id (str)
300
          remote (bool): whether to perform expensive side effects like fetching
301
            the id itself over the network, or other discovery.
302

303
        Returns:
304
          Protocol subclass: matching protocol, or None if no single known
305
          protocol definitively owns this id
306
        """
307
        logger.debug(f'Determining protocol for id {id}')
1✔
308
        if not id:
1✔
309
            return None
1✔
310

311
        # remove our synthetic id fragment, if any
312
        #
313
        # will this eventually cause false positives for other services that
314
        # include our full ids inside their own ids, non-URL-encoded? guess
315
        # we'll figure that out if/when it happens.
316
        id = id.partition('#bridgy-fed-')[0]
1✔
317
        if not id:
1✔
318
            return None
1✔
319

320
        if util.is_web(id):
1✔
321
            # step 1: check for our per-protocol subdomains
322
            try:
1✔
323
                parsed = urlparse(id)
1✔
324
            except ValueError as e:
1✔
325
                logger.info(f'urlparse ValueError: {e}')
1✔
326
                return None
1✔
327

328
            is_homepage = parsed.path.strip('/') == ''
1✔
329
            is_internal = parsed.path.startswith(ids.INTERNAL_PATH_PREFIX)
1✔
330
            by_subdomain = Protocol.for_bridgy_subdomain(id)
1✔
331
            if by_subdomain and not (is_homepage or is_internal
1✔
332
                                     or id in ids.BOT_ACTOR_AP_IDS):
333
                logger.debug(f'  {by_subdomain.LABEL} owns id {id}')
1✔
334
                return by_subdomain
1✔
335

336
        # step 2: check if any Protocols say conclusively that they own it
337
        # sort to be deterministic
338
        protocols = sorted(set(p for p in PROTOCOLS.values() if p),
1✔
339
                           key=lambda p: p.LABEL)
340
        candidates = []
1✔
341
        for protocol in protocols:
1✔
342
            owns = protocol.owns_id(id)
1✔
343
            if owns:
1✔
344
                logger.debug(f'  {protocol.LABEL} owns id {id}')
1✔
345
                return protocol
1✔
346
            elif owns is not False:
1✔
347
                candidates.append(protocol)
1✔
348

349
        if len(candidates) == 1:
1✔
350
            logger.debug(f'  {candidates[0].LABEL} owns id {id}')
1✔
351
            return candidates[0]
1✔
352

353
        # step 3: look for existing Objects in the datastore
354
        #
355
        # note that we don't currently see if this is a copy id because I have FUD
356
        # over which Protocol for_id should return in that case...and also because a
357
        # protocol may already say definitively above that it owns the id, eg ATProto
358
        # with DIDs and at:// URIs.
359
        obj = Protocol.load(id, remote=False)
1✔
360
        if obj and obj.source_protocol:
1✔
361
            logger.debug(f'  {obj.key.id()} owned by source_protocol {obj.source_protocol}')
1✔
362
            return PROTOCOLS[obj.source_protocol]
1✔
363

364
        # step 4: fetch over the network, if necessary
365
        if not remote:
1✔
366
            return None
1✔
367

368
        for protocol in candidates:
1✔
369
            logger.debug(f'Trying {protocol.LABEL}')
1✔
370
            try:
1✔
371
                obj = protocol.load(id, local=False, remote=True)
1✔
372

373
                if protocol.ABBREV == 'web':
1✔
374
                    # for web, if we fetch and get HTML without microformats,
375
                    # load returns False but the object will be stored in the
376
                    # datastore with source_protocol web, and in cache. load it
377
                    # again manually to check for that.
378
                    obj = Object.get_by_id(id)
1✔
379
                    if obj and obj.source_protocol != 'web':
1✔
380
                        obj = None
×
381

382
                if obj:
1✔
383
                    logger.debug(f'  {protocol.LABEL} owns id {id}')
1✔
384
                    return protocol
1✔
385
            except BadGateway:
1✔
386
                # we tried and failed fetching the id over the network.
387
                # this depends on ActivityPub.fetch raising this!
388
                return None
1✔
389
            except HTTPException as e:
×
390
                # internal error we generated ourselves; try next protocol
391
                pass
×
392
            except Exception as e:
×
393
                code, _ = util.interpret_http_exception(e)
×
394
                if code:
×
395
                    # we tried and failed fetching the id over the network
396
                    return None
×
397
                raise
×
398

399
        logger.info(f'No matching protocol found for {id} !')
1✔
400
        return None
1✔
401

402
    @cached(LRUCache(20000), lock=Lock())
1✔
403
    @staticmethod
1✔
404
    def for_handle(handle):
1✔
405
        """Returns the protocol for a given handle.
406

407
        May incur expensive side effects like resolving the handle itself over
408
        the network or other discovery.
409

410
        Args:
411
          handle (str)
412

413
        Returns:
414
          (Protocol subclass, str) tuple: matching protocol and optional id (if
415
          resolved), or ``(None, None)`` if no known protocol owns this handle
416
        """
417
        # TODO: normalize, eg convert domains to lower case
418
        logger.debug(f'Determining protocol for handle {handle}')
1✔
419
        if not handle:
1✔
420
            return (None, None)
1✔
421

422
        # step 1: check if any Protocols say conclusively that they own it.
423
        # sort to be deterministic.
424
        protocols = sorted(set(p for p in PROTOCOLS.values() if p),
1✔
425
                           key=lambda p: p.LABEL)
426
        candidates = []
1✔
427
        for proto in protocols:
1✔
428
            owns = proto.owns_handle(handle)
1✔
429
            if owns:
1✔
430
                logger.debug(f'  {proto.LABEL} owns handle {handle}')
1✔
431
                return (proto, None)
1✔
432
            elif owns is not False:
1✔
433
                candidates.append(proto)
1✔
434

435
        if len(candidates) == 1:
1✔
436
            logger.debug(f'  {candidates[0].LABEL} owns handle {handle}')
×
437
            return (candidates[0], None)
×
438

439
        # step 2: look for matching User in the datastore
440
        for proto in candidates:
1✔
441
            user = proto.query(proto.handle == handle).get()
1✔
442
            if user:
1✔
443
                if user.status:
1✔
444
                    return (None, None)
1✔
445
                logger.debug(f'  user {user.key} handle {handle}')
1✔
446
                return (proto, user.key.id())
1✔
447

448
        # step 3: resolve handle to id
449
        for proto in candidates:
1✔
450
            id = proto.handle_to_id(handle)
1✔
451
            if id:
1✔
452
                logger.debug(f'  {proto.LABEL} resolved handle {handle} to id {id}')
1✔
453
                return (proto, id)
1✔
454

455
        logger.info(f'No matching protocol found for handle {handle} !')
1✔
456
        return (None, None)
1✔
457

458
    @classmethod
1✔
459
    def is_user_at_domain(cls, handle, allow_internal=False):
1✔
460
        """Returns True if handle is formatted ``user@domain.tld``, False otherwise.
461

462
        Example: ``@user@instance.com``
463

464
        Args:
465
          handle (str)
466
          allow_internal (bool): whether the domain can be a Bridgy Fed domain
467
        """
468
        parts = handle.split('@')
1✔
469
        if len(parts) != 2:
1✔
470
            return False
1✔
471

472
        user, domain = parts
1✔
473
        return bool(user and domain
1✔
474
                    and not cls.is_blocklisted(domain, allow_internal=allow_internal))
475

476
    @classmethod
1✔
477
    def bridged_web_url_for(cls, user, fallback=False):
1✔
478
        """Returns the web URL for a user's bridged profile in this protocol.
479

480
        For example, for Web user ``alice.com``, :meth:`ATProto.bridged_web_url_for`
481
        returns ``https://bsky.app/profile/alice.com.web.brid.gy``
482

483
        Args:
484
          user (models.User)
485
          fallback (bool): if True, and bridged users have no canonical user
486
            profile URL in this protocol, return the native protocol's profile URL
487

488
        Returns:
489
          str, or None if there isn't a canonical URL
490
        """
491
        if fallback:
1✔
492
            return user.web_url()
1✔
493

494
    @classmethod
1✔
495
    def actor_key(cls, obj, allow_opt_out=False):
1✔
496
        """Returns the :class:`User`: key for a given object's author or actor.
497

498
        Args:
499
          obj (models.Object)
500
          allow_opt_out (bool): whether to return a user key if they're opted out
501

502
        Returns:
503
          google.cloud.ndb.key.Key or None:
504
        """
505
        owner = as1.get_owner(obj.as1)
1✔
506
        if owner:
1✔
507
            return cls.key_for(owner, allow_opt_out=allow_opt_out)
1✔
508

509
    @classmethod
1✔
510
    def bot_user_id(cls):
1✔
511
        """Returns the Web user id for the bot user for this protocol.
512

513
        For example, ``'bsky.brid.gy'`` for ATProto.
514

515
        Returns:
516
          str:
517
        """
518
        return f'{cls.ABBREV}{common.SUPERDOMAIN}'
1✔
519

520
    @classmethod
1✔
521
    def create_for(cls, user):
1✔
522
        """Creates or re-activate a copy user in this protocol.
523

524
        Should add the copy user to :attr:`copies`.
525

526
        If the copy user already exists and active, should do nothing.
527

528
        Args:
529
          user (models.User): original source user. Shouldn't already have a
530
            copy user for this protocol in :attr:`copies`.
531

532
        Raises:
533
          ValueError: if we can't create a copy of the given user in this protocol
534
        """
535
        raise NotImplementedError()
×
536

537
    @classmethod
1✔
538
    def send(to_cls, obj, target, from_user=None, orig_obj_id=None):
1✔
539
        """Sends an outgoing activity.
540

541
        To be implemented by subclasses. Should call
542
        ``to_cls.translate_ids(obj.as1)`` before converting it to this Protocol's
543
        format.
544

545
        NOTE: if this protocol's ``HAS_COPIES`` is True, and this method creates
546
        a copy and sends it, it *must* add that copy to the *object*'s (not
547
        activity's) :attr:`copies`, and store it back in the datastore!
548

549
        Args:
550
          obj (models.Object): with activity to send
551
          target (str): destination URL to send to
552
          from_user (models.User): user (actor) this activity is from
553
          orig_obj_id (str): :class:`models.Object` key id of the "original object"
554
            that this object refers to, eg replies to or reposts or likes
555

556
        Returns:
557
          bool: True if the activity is sent successfully, False if it is
558
          ignored or otherwise unsent due to protocol logic, eg no webmention
559
          endpoint, protocol doesn't support the activity type. (Failures are
560
          raised as exceptions.)
561

562
        Raises:
563
          werkzeug.HTTPException if the request fails
564
        """
565
        raise NotImplementedError()
×
566

567
    @classmethod
1✔
568
    def fetch(cls, obj, **kwargs):
1✔
569
        """Fetches a protocol-specific object and populates it in an :class:`Object`.
570

571
        Errors are raised as exceptions. If this method returns False, the fetch
572
        didn't fail but didn't succeed either, eg the id isn't valid for this
573
        protocol, or the fetch didn't return valid data for this protocol.
574

575
        To be implemented by subclasses.
576

577
        Args:
578
          obj (models.Object): with the id to fetch. Data is filled into one of
579
            the protocol-specific properties, eg ``as2``, ``mf2``, ``bsky``.
580
          kwargs: subclass-specific
581

582
        Returns:
583
          bool: True if the object was fetched and populated successfully,
584
          False otherwise
585

586
        Raises:
587
          requests.RequestException, werkzeug.HTTPException,
588
          websockets.WebSocketException, etc: if the fetch fails
589
        """
590
        raise NotImplementedError()
×
591

592
    @classmethod
1✔
593
    def convert(cls, obj, from_user=None, **kwargs):
1✔
594
        """Converts an :class:`Object` to this protocol's data format.
595

596
        For example, an HTML string for :class:`Web`, or a dict with AS2 JSON
597
        and ``application/activity+json`` for :class:`ActivityPub`.
598

599
        Just passes through to :meth:`_convert`, then does minor
600
        protocol-independent postprocessing.
601

602
        Args:
603
          obj (models.Object):
604
          from_user (models.User): user (actor) this activity/object is from
605
          kwargs: protocol-specific, passed through to :meth:`_convert`
606

607
        Returns:
608
          converted object in the protocol's native format, often a dict
609
        """
610
        if not obj or not obj.as1:
1✔
611
            return {}
1✔
612

613
        id = obj.key.id() if obj.key else obj.as1.get('id')
1✔
614
        is_crud = obj.as1.get('verb') in as1.CRUD_VERBS
1✔
615
        base_obj = as1.get_object(obj.as1) if is_crud else obj.as1
1✔
616
        orig_our_as1 = obj.our_as1
1✔
617

618
        # mark bridged actors as bots and add "bridged by Bridgy Fed" to their bios
619
        if (from_user and base_obj
1✔
620
            and base_obj.get('objectType') in as1.ACTOR_TYPES
621
            and PROTOCOLS.get(obj.source_protocol) != cls
622
            and Protocol.for_bridgy_subdomain(id) not in DOMAINS
623
            # Web users are special cased, they don't get the label if they've
624
            # explicitly enabled Bridgy Fed with redirects or webmentions
625
            and not (from_user.LABEL == 'web'
626
                     and (from_user.last_webmention_in or from_user.has_redirects))):
627
            cls.add_source_links(obj=obj, from_user=from_user)
1✔
628

629
        converted = cls._convert(obj, from_user=from_user, **kwargs)
1✔
630
        obj.our_as1 = orig_our_as1
1✔
631
        return converted
1✔
632

633
    @classmethod
1✔
634
    def _convert(cls, obj, from_user=None, **kwargs):
1✔
635
        """Converts an :class:`Object` to this protocol's data format.
636

637
        To be implemented by subclasses. Implementations should generally call
638
        :meth:`Protocol.translate_ids` (as their own class) before converting to
639
        their format.
640

641
        Args:
642
          obj (models.Object):
643
          from_user (models.User): user (actor) this activity/object is from
644
          kwargs: protocol-specific
645

646
        Returns:
647
          converted object in the protocol's native format, often a dict. May
648
            return the ``{}`` empty dict if the object can't be converted.
649
        """
650
        raise NotImplementedError()
×
651

652
    @classmethod
1✔
653
    def add_source_links(cls, obj, from_user):
1✔
654
        """Adds "bridged from ... by Bridgy Fed" to the user's actor's ``summary``.
655

656
        Uses HTML for protocols that support it, plain text otherwise.
657

658
        Args:
659
          obj (models.Object): user's actor/profile object
660
          from_user (models.User): user (actor) this activity/object is from
661
        """
662
        assert obj and obj.as1
1✔
663
        assert from_user
1✔
664

665
        obj.our_as1 = copy.deepcopy(obj.as1)
1✔
666
        actor = (as1.get_object(obj.as1) if obj.as1.get('verb') in as1.CRUD_VERBS
1✔
667
                 else obj.as1)
668
        actor['objectType'] = 'person'
1✔
669

670
        orig_summary = actor.setdefault('summary', '')
1✔
671
        summary_text = html_to_text(orig_summary, ignore_links=True)
1✔
672

673
        # Check if we've already added source links
674
        if '🌉 bridged' in summary_text:
1✔
675
            return
1✔
676

677
        actor_id = actor.get('id')
1✔
678

679
        url = (as1.get_url(actor)
1✔
680
               or (from_user.web_url() if from_user.profile_id() == actor_id
681
                   else actor_id))
682

683
        from web import Web
1✔
684
        bot_user = Web.get_by_id(from_user.bot_user_id())
1✔
685

686
        if cls.HTML_PROFILES:
1✔
687
            if bot_user and from_user.LABEL not in cls.DEFAULT_ENABLED_PROTOCOLS:
1✔
688
                mention = bot_user.user_link(proto=cls, name=False, handle='short')
1✔
689
                suffix = f', follow {mention} to interact'
1✔
690
            else:
691
                suffix = f' by <a href="https://{PRIMARY_DOMAIN}/">Bridgy Fed</a>'
1✔
692

693
            separator = '<br><br>'
1✔
694

695
            is_user = from_user.key and actor_id in (from_user.key.id(),
1✔
696
                                                     from_user.profile_id())
697
            if is_user:
1✔
698
                bridged = f'🌉 <a href="https://{PRIMARY_DOMAIN}{from_user.user_page_path()}">bridged</a>'
1✔
699
                from_ = f'<a href="{from_user.web_url()}">{from_user.handle}</a>'
1✔
700
            else:
701
                bridged = '🌉 bridged'
1✔
702
                from_ = util.pretty_link(url) if url else '?'
1✔
703

704
        else:  # plain text
705
            # TODO: unify with above. which is right?
706
            id = obj.key.id() if obj.key else obj.our_as1.get('id')
1✔
707
            is_user = from_user.key and id in (from_user.key.id(),
1✔
708
                                               from_user.profile_id())
709
            from_ = (from_user.web_url() if is_user else url) or '?'
1✔
710

711
            bridged = '🌉 bridged'
1✔
712
            suffix = (
1✔
713
                f': https://{PRIMARY_DOMAIN}{from_user.user_page_path()}'
714
                # link web users to their user pages
715
                if from_user.LABEL == 'web'
716
                else f', follow @{bot_user.handle_as(cls)} to interact'
717
                if bot_user and from_user.LABEL not in cls.DEFAULT_ENABLED_PROTOCOLS
718
                else f' by https://{PRIMARY_DOMAIN}/')
719
            separator = '\n\n'
1✔
720
            orig_summary = summary_text
1✔
721

722
        logo = f'{from_user.LOGO_EMOJI} ' if from_user.LOGO_EMOJI else ''
1✔
723
        source_links = f'{separator if orig_summary else ""}{bridged} from {logo}{from_}{suffix}'
1✔
724
        actor['summary'] = orig_summary + source_links
1✔
725

726
    @classmethod
1✔
727
    def set_username(to_cls, user, username):
1✔
728
        """Sets a custom username for a user's bridged account in this protocol.
729

730
        Args:
731
          user (models.User)
732
          username (str)
733

734
        Raises:
735
          ValueError: if the username is invalid
736
          RuntimeError: if the username could not be set
737
        """
738
        raise NotImplementedError()
1✔
739

740
    @classmethod
1✔
741
    def migrate_out(cls, user, to_user_id):
1✔
742
        """Migrates a bridged account out to be a native account.
743

744
        Args:
745
          user (models.User)
746
          to_user_id (str)
747

748
        Raises:
749
          ValueError: eg if this protocol doesn't own ``to_user_id``, or if
750
            ``user`` is on this protocol or not bridged to this protocol
751
        """
752
        raise NotImplementedError()
×
753

754
    @classmethod
1✔
755
    def check_can_migrate_out(cls, user, to_user_id):
1✔
756
        """Raises an exception if a user can't yet migrate to a native account.
757

758
        For example, if ``to_user_id`` isn't on this protocol, or if ``user`` is on
759
        this protocol, or isn't bridged to this protocol.
760

761
        If the user is ready to migrate, returns ``None``.
762

763
        Subclasses may override this to add more criteria, but they should call this
764
        implementation first.
765

766
        Args:
767
          user (models.User)
768
          to_user_id (str)
769

770
        Raises:
771
          ValueError: if ``user`` isn't ready to migrate to this protocol yet
772
        """
773
        def _error(msg):
1✔
774
            logger.warning(msg)
1✔
775
            raise ValueError(msg)
1✔
776

777
        if cls.owns_id(to_user_id) is False:
1✔
778
            _error(f"{to_user_id} doesn't look like an {cls.LABEL} id")
1✔
779
        elif isinstance(user, cls):
1✔
780
            _error(f"{user.handle_or_id()} is on {cls.PHRASE}")
1✔
781
        elif not user.is_enabled(cls):
1✔
782
            _error(f"{user.handle_or_id()} isn't currently bridged to {cls.PHRASE}")
1✔
783

784
    @classmethod
1✔
785
    def migrate_in(cls, user, from_user_id, **kwargs):
1✔
786
        """Migrates a native account in to be a bridged account.
787

788
        The protocol independent parts are done here; protocol-specific parts are
789
        done in :meth:`_migrate_in`, which this wraps.
790

791
        Reloads the user's profile before calling :meth:`_migrate_in`.
792

793
        Args:
794
          user (models.User): native user on another protocol to attach the
795
            newly imported bridged account to
796
          from_user_id (str)
797
          kwargs: additional protocol-specific parameters
798

799
        Raises:
800
          ValueError: eg if this protocol doesn't own ``from_user_id``, or if
801
            ``user`` is on this protocol or already bridged to this protocol
802
        """
803
        def _error(msg):
1✔
804
            logger.warning(msg)
1✔
805
            raise ValueError(msg)
1✔
806

807
        logger.info(f"Migrating in {from_user_id} for {user.key.id()}")
1✔
808

809
        # check req'ts
810
        if cls.owns_id(from_user_id) is False:
1✔
811
            _error(f"{from_user_id} doesn't look like an {cls.LABEL} id")
1✔
812
        elif isinstance(user, cls):
1✔
813
            _error(f"{user.handle_or_id()} is on {cls.PHRASE}")
1✔
814
        elif cls.HAS_COPIES and cls.LABEL in user.enabled_protocols:
1✔
815
            _error(f"{user.handle_or_id()} is already bridged to {cls.PHRASE}")
1✔
816

817
        # reload profile
818
        try:
1✔
819
            user.reload_profile()
1✔
820
        except (RequestException, HTTPException) as e:
×
821
            _, msg = util.interpret_http_exception(e)
×
822

823
        # migrate!
824
        cls._migrate_in(user, from_user_id, **kwargs)
1✔
825
        user.add('enabled_protocols', cls.LABEL)
1✔
826
        user.put()
1✔
827

828
        # attach profile object
829
        if user.obj:
1✔
830
            if cls.HAS_COPIES:
1✔
831
                profile_id = ids.profile_id(id=from_user_id, proto=cls)
1✔
832
                user.obj.remove_copies_on(cls)
1✔
833
                user.obj.add('copies', Target(uri=profile_id, protocol=cls.LABEL))
1✔
834
                user.obj.put()
1✔
835

836
            common.create_task(queue='receive', obj_id=user.obj_key.id(),
1✔
837
                               authed_as=user.key.id())
838

839
    @classmethod
1✔
840
    def _migrate_in(cls, user, from_user_id, **kwargs):
1✔
841
        """Protocol-specific parts of migrating in external account.
842

843
        Called by :meth:`migrate_in`, which does most of the work, including calling
844
        :meth:`reload_profile` before this.
845

846
        Args:
847
          user (models.User): native user on another protocol to attach the
848
            newly imported account to. Unused.
849
          from_user_id (str): DID of the account to be migrated in
850
          kwargs: protocol dependent
851
        """
852
        raise NotImplementedError()
×
853

854
    @classmethod
1✔
855
    def target_for(cls, obj, shared=False):
1✔
856
        """Returns an :class:`Object`'s delivery target (endpoint).
857

858
        To be implemented by subclasses.
859

860
        Examples:
861

862
        * If obj has ``source_protocol`` ``web``, returns its URL, as a
863
          webmention target.
864
        * If obj is an ``activitypub`` actor, returns its inbox.
865
        * If obj is an ``activitypub`` object, returns it's author's or actor's
866
          inbox.
867

868
        Args:
869
          obj (models.Object):
870
          shared (bool): optional. If True, returns a common/shared
871
            endpoint, eg ActivityPub's ``sharedInbox``, that can be reused for
872
            multiple recipients for efficiency
873

874
        Returns:
875
          str: target endpoint, or None if not available.
876
        """
877
        raise NotImplementedError()
×
878

879
    @classmethod
1✔
880
    def is_blocklisted(cls, url, allow_internal=False):
1✔
881
        """Returns True if we block the given URL and shouldn't deliver to it.
882

883
        Default implementation here, subclasses may override.
884

885
        Args:
886
          url (str):
887
          allow_internal (bool): whether to return False for internal domains
888
            like ``fed.brid.gy``, ``bsky.brid.gy``, etc
889
        """
890
        blocklist = DOMAIN_BLOCKLIST
1✔
891
        if not DEBUG:
1✔
892
            blocklist += tuple(util.RESERVED_TLDS | util.LOCAL_TLDS)
×
893
        if not allow_internal:
1✔
894
            blocklist += DOMAINS
1✔
895
        return util.domain_or_parent_in(url, blocklist)
1✔
896

897
    @classmethod
1✔
898
    def translate_ids(to_cls, obj):
1✔
899
        """Translates all ids in an AS1 object to a specific protocol.
900

901
        Infers source protocol for each id value separately.
902

903
        For example, if ``proto`` is :class:`ActivityPub`, the ATProto URI
904
        ``at://did:plc:abc/coll/123`` will be converted to
905
        ``https://bsky.brid.gy/ap/at://did:plc:abc/coll/123``.
906

907
        Wraps these AS1 fields:
908

909
        * ``id``
910
        * ``actor``
911
        * ``author``
912
        * ``bcc``
913
        * ``bto``
914
        * ``cc``
915
        * ``featured[].items``, ``featured[].orderedItems``
916
        * ``object``
917
        * ``object.actor``
918
        * ``object.author``
919
        * ``object.id``
920
        * ``object.inReplyTo``
921
        * ``object.object``
922
        * ``attachments[].id``
923
        * ``tags[objectType=mention].url``
924
        * ``to``
925

926
        This is the inverse of :meth:`models.Object.resolve_ids`. Much of the
927
        same logic is duplicated there!
928

929
        TODO: unify with :meth:`Object.resolve_ids`,
930
        :meth:`models.Object.normalize_ids`.
931

932
        Args:
933
          to_proto (Protocol subclass)
934
          obj (dict): AS1 object or activity (not :class:`models.Object`!)
935

936
        Returns:
937
          dict: wrapped AS1 version of ``obj``
938
        """
939
        assert to_cls != Protocol
1✔
940
        if not obj:
1✔
941
            return obj
1✔
942

943
        outer_obj = copy.deepcopy(obj)
1✔
944
        inner_objs = outer_obj['object'] = as1.get_objects(outer_obj)
1✔
945

946
        def translate(elem, field, fn, uri=False):
1✔
947
            elem[field] = as1.get_objects(elem, field)
1✔
948
            for obj in elem[field]:
1✔
949
                if id := obj.get('id'):
1✔
950
                    if field in ('to', 'cc', 'bcc', 'bto') and as1.is_audience(id):
1✔
951
                        continue
1✔
952
                    from_cls = Protocol.for_id(id)
1✔
953
                    # TODO: what if from_cls is None? relax translate_object_id,
954
                    # make it a noop if we don't know enough about from/to?
955
                    if from_cls and from_cls != to_cls:
1✔
956
                        obj['id'] = fn(id=id, from_=from_cls, to=to_cls)
1✔
957
                    if obj['id'] and uri:
1✔
958
                        obj['id'] = to_cls(id=obj['id']).id_uri()
1✔
959

960
            elem[field] = [o['id'] if o.keys() == {'id'} else o
1✔
961
                           for o in elem[field]]
962

963
            if len(elem[field]) == 1 and field not in ('items', 'orderedItems'):
1✔
964
                elem[field] = elem[field][0]
1✔
965

966
        type = as1.object_type(outer_obj)
1✔
967
        translate(outer_obj, 'id',
1✔
968
                  ids.translate_user_id if type in as1.ACTOR_TYPES
969
                  else ids.translate_object_id)
970

971
        for o in inner_objs:
1✔
972
            is_actor = (as1.object_type(o) in as1.ACTOR_TYPES
1✔
973
                        or as1.get_owner(outer_obj) == o.get('id')
974
                        or type in ('follow', 'stop-following'))
975
            translate(o, 'id', (ids.translate_user_id if is_actor
1✔
976
                                else ids.translate_object_id))
977
            obj_is_actor = o.get('verb') in as1.VERBS_WITH_ACTOR_OBJECT
1✔
978
            translate(o, 'object', (ids.translate_user_id if obj_is_actor
1✔
979
                                    else ids.translate_object_id))
980

981
        for o in [outer_obj] + inner_objs:
1✔
982
            translate(o, 'inReplyTo', ids.translate_object_id)
1✔
983
            for field in 'actor', 'author', 'to', 'cc', 'bto', 'bcc':
1✔
984
                translate(o, field, ids.translate_user_id)
1✔
985
            for tag in as1.get_objects(o, 'tags'):
1✔
986
                if tag.get('objectType') == 'mention':
1✔
987
                    translate(tag, 'url', ids.translate_user_id, uri=True)
1✔
988
            for att in as1.get_objects(o, 'attachments'):
1✔
989
                translate(att, 'id', ids.translate_object_id)
1✔
990
                url = att.get('url')
1✔
991
                if url and not att.get('id'):
1✔
992
                    if from_cls := Protocol.for_id(url):
1✔
993
                        att['id'] = ids.translate_object_id(from_=from_cls, to=to_cls,
1✔
994
                                                            id=url)
995
            if feat := as1.get_object(o, 'featured'):
1✔
996
                translate(feat, 'orderedItems', ids.translate_object_id)
1✔
997
                translate(feat, 'items', ids.translate_object_id)
1✔
998

999
        outer_obj = util.trim_nulls(outer_obj)
1✔
1000

1001
        if objs := util.get_list(outer_obj ,'object'):
1✔
1002
            outer_obj['object'] = [o['id'] if o.keys() == {'id'} else o for o in objs]
1✔
1003
            if len(outer_obj['object']) == 1:
1✔
1004
                outer_obj['object'] = outer_obj['object'][0]
1✔
1005

1006
        return outer_obj
1✔
1007

1008
    @classmethod
1✔
1009
    def receive(from_cls, obj, authed_as=None, internal=False, received_at=None):
1✔
1010
        """Handles an incoming activity.
1011

1012
        If ``obj``'s key is unset, ``obj.as1``'s id field is used. If both are
1013
        unset, returns HTTP 299.
1014

1015
        Args:
1016
          obj (models.Object)
1017
          authed_as (str): authenticated actor id who sent this activity
1018
          internal (bool): whether to allow activity ids on internal domains,
1019
            from opted out/blocked users, etc.
1020
          received_at (datetime): when we first saw (received) this activity.
1021
            Right now only used for monitoring.
1022

1023
        Returns:
1024
          (str, int) tuple: (response body, HTTP status code) Flask response
1025

1026
        Raises:
1027
          werkzeug.HTTPException: if the request is invalid
1028
        """
1029
        # check some invariants
1030
        assert from_cls != Protocol
1✔
1031
        assert isinstance(obj, Object), obj
1✔
1032

1033
        if not obj.as1:
1✔
1034
            error('No object data provided')
1✔
1035

1036
        id = None
1✔
1037
        if obj.key and obj.key.id():
1✔
1038
            id = obj.key.id()
1✔
1039

1040
        if not id:
1✔
1041
            id = obj.as1.get('id')
1✔
1042
            obj.key = ndb.Key(Object, id)
1✔
1043

1044
        if not id:
1✔
1045
            error('No id provided')
×
1046
        elif from_cls.owns_id(id) is False:
1✔
1047
            error(f'Protocol {from_cls.LABEL} does not own id {id}')
1✔
1048
        elif from_cls.is_blocklisted(id, allow_internal=internal):
1✔
1049
            error(f'Activity {id} is blocklisted')
1✔
1050

1051
        # does this protocol support this activity/object type?
1052
        from_cls.check_supported(obj, 'receive')
1✔
1053

1054
        # lease this object, atomically
1055
        memcache_key = activity_id_memcache_key(id)
1✔
1056
        leased = memcache.memcache.add(
1✔
1057
            memcache_key, 'leased', noreply=False,
1058
            expire=int(MEMCACHE_LEASE_EXPIRATION.total_seconds()))
1059

1060
        # short circuit if we've already seen this activity id.
1061
        # (don't do this for bare objects since we need to check further down
1062
        # whether they've been updated since we saw them last.)
1063
        if (obj.as1.get('objectType') == 'activity'
1✔
1064
            and 'force' not in request.values
1065
            and (not leased
1066
                 or (obj.new is False and obj.changed is False))):
1067
            error(f'Already seen this activity {id}', status=204)
1✔
1068

1069
        pruned = {k: v for k, v in obj.as1.items()
1✔
1070
                  if k not in ('contentMap', 'replies', 'signature')}
1071
        delay = ''
1✔
1072
        retry = request.headers.get('X-AppEngine-TaskRetryCount')
1✔
1073
        if (received_at and retry in (None, '0')
1✔
1074
                and obj.type not in ('delete', 'undo')):  # we delay deletes/undos
1075
            delay_s = int((util.now().replace(tzinfo=None)
1✔
1076
                           - received_at.replace(tzinfo=None)
1077
                           ).total_seconds())
1078
            delay = f'({delay_s} s behind)'
1✔
1079
        logger.info(f'Receiving {from_cls.LABEL} {obj.type} {id} {delay} AS1: {json_dumps(pruned, indent=2)}')
1✔
1080

1081
        # check authorization
1082
        # https://www.w3.org/wiki/ActivityPub/Primer/Authentication_Authorization
1083
        actor = as1.get_owner(obj.as1)
1✔
1084
        if not actor:
1✔
1085
            error('Activity missing actor or author')
1✔
1086
        elif from_cls.owns_id(actor) is False:
1✔
1087
            error(f"{from_cls.LABEL} doesn't own actor {actor}, this is probably a bridged activity. Skipping.", status=204)
1✔
1088

1089
        assert authed_as
1✔
1090
        assert isinstance(authed_as, str)
1✔
1091
        authed_as = ids.normalize_user_id(id=authed_as, proto=from_cls)
1✔
1092
        actor = ids.normalize_user_id(id=actor, proto=from_cls)
1✔
1093
        if actor != authed_as:
1✔
1094
            report_error("Auth: receive: authed_as doesn't match owner",
1✔
1095
                         user=f'{id} authed_as {authed_as} owner {actor}')
1096
            error(f"actor {actor} isn't authed user {authed_as}")
1✔
1097

1098
        # update copy ids to originals
1099
        obj.normalize_ids()
1✔
1100
        obj.resolve_ids()
1✔
1101

1102
        if (obj.type == 'follow'
1✔
1103
                and Protocol.for_bridgy_subdomain(as1.get_object(obj.as1).get('id'))):
1104
            # follows of bot user; refresh user profile first
1105
            logger.info(f'Follow of bot user, reloading {actor}')
1✔
1106
            from_user = from_cls.get_or_create(id=actor, allow_opt_out=True)
1✔
1107
            from_user.reload_profile()
1✔
1108
        else:
1109
            # load actor user
1110
            from_user = from_cls.get_or_create(id=actor, allow_opt_out=internal)
1✔
1111

1112
        if not internal and (not from_user or from_user.manual_opt_out):
1✔
1113
            error(f"Couldn't load actor {actor}", status=204)
1✔
1114

1115
        # if this is an object, ie not an activity, wrap it in a create or update
1116
        obj = from_cls.handle_bare_object(obj, authed_as=authed_as)
1✔
1117
        obj.add('users', from_user.key)
1✔
1118

1119
        inner_obj_as1 = as1.get_object(obj.as1)
1✔
1120
        inner_obj_id = inner_obj_as1.get('id')
1✔
1121
        if obj.type in as1.CRUD_VERBS | as1.VERBS_WITH_OBJECT:
1✔
1122
            if not inner_obj_id:
1✔
1123
                error(f'{obj.type} object has no id!')
1✔
1124

1125
        # check age. we support backdated posts, but if they're over 2w old, we
1126
        # don't deliver them
1127
        if obj.type == 'post':
1✔
1128
            if published := inner_obj_as1.get('published'):
1✔
1129
                try:
1✔
1130
                    published_dt = util.parse_iso8601(published)
1✔
1131
                    if not published_dt.tzinfo:
1✔
1132
                        published_dt = published_dt.replace(tzinfo=timezone.utc)
×
1133
                    age = util.now() - published_dt
1✔
1134
                    if age > CREATE_MAX_AGE and 'force' not in request.values:
1✔
1135
                        error(f'Ignoring, too old, {age} is over {CREATE_MAX_AGE}',
×
1136
                              status=204)
1137
                except ValueError:  # from parse_iso8601
×
1138
                    logger.debug(f"Couldn't parse published {published}")
×
1139

1140
        # write Object to datastore
1141
        obj.source_protocol = from_cls.LABEL
1✔
1142
        if obj.type in STORE_AS1_TYPES:
1✔
1143
            obj.put()
1✔
1144

1145
        # store inner object
1146
        # TODO: unify with big obj.type conditional below. would have to merge
1147
        # this with the DM handling block lower down.
1148
        crud_obj = None
1✔
1149
        if obj.type in ('post', 'update') and inner_obj_as1.keys() > set(['id']):
1✔
1150
            crud_obj = Object.get_or_create(inner_obj_id, our_as1=inner_obj_as1,
1✔
1151
                                            source_protocol=from_cls.LABEL,
1152
                                            authed_as=actor, users=[from_user.key],
1153
                                            deleted=False)
1154

1155
        actor = as1.get_object(obj.as1, 'actor')
1✔
1156
        actor_id = actor.get('id')
1✔
1157

1158
        # handle activity!
1159
        if obj.type == 'stop-following':
1✔
1160
            # TODO: unify with handle_follow?
1161
            # TODO: handle multiple followees
1162
            if not actor_id or not inner_obj_id:
1✔
1163
                error(f'stop-following requires actor id and object id. Got: {actor_id} {inner_obj_id} {obj.as1}')
×
1164

1165
            # deactivate Follower
1166
            from_ = from_cls.key_for(actor_id)
1✔
1167
            if not (to_cls := Protocol.for_id(inner_obj_id)):
1✔
1168
                error(f"Can't determine protocol for {inner_obj_id} , giving up")
1✔
1169
            to = to_cls.key_for(inner_obj_id)
1✔
1170
            follower = Follower.query(Follower.to == to,
1✔
1171
                                      Follower.from_ == from_,
1172
                                      Follower.status == 'active').get()
1173
            if follower:
1✔
1174
                logger.info(f'Marking {follower} inactive')
1✔
1175
                follower.status = 'inactive'
1✔
1176
                follower.put()
1✔
1177
            else:
1178
                logger.warning(f'No Follower found for {from_} => {to}')
1✔
1179

1180
            # fall through to deliver to followee
1181
            # TODO: do we convert stop-following to webmention 410 of original
1182
            # follow?
1183

1184
            # fall through to deliver to followers
1185

1186
        elif obj.type in ('delete', 'undo'):
1✔
1187
            delete_obj_id = (from_user.profile_id()
1✔
1188
                            if inner_obj_id == from_user.key.id()
1189
                            else inner_obj_id)
1190

1191
            delete_obj = Object.get_by_id(delete_obj_id, authed_as=authed_as)
1✔
1192
            if not delete_obj:
1✔
1193
                logger.info(f"Ignoring, we don't have {delete_obj_id} stored")
1✔
1194
                return 'OK', 204
1✔
1195

1196
            # TODO: just delete altogether!
1197
            logger.info(f'Marking Object {delete_obj_id} deleted')
1✔
1198
            delete_obj.deleted = True
1✔
1199
            delete_obj.put()
1✔
1200

1201
            # if this is an actor, handle deleting it later so that
1202
            # in case it's from_user, user.enabled_protocols is still populated
1203
            #
1204
            # fall through to deliver to followers and delete copy if necessary.
1205
            # should happen via protocol-specific copy target and send of
1206
            # delete activity.
1207
            # https://github.com/snarfed/bridgy-fed/issues/63
1208

1209
        elif obj.type == 'block':
1✔
1210
            if proto := Protocol.for_bridgy_subdomain(inner_obj_id):
1✔
1211
                # blocking protocol bot user disables that protocol
1212
                from_user.delete(proto)
1✔
1213
                from_user.disable_protocol(proto)
1✔
1214
                return 'OK', 200
1✔
1215

1216
        elif obj.type == 'post':
1✔
1217
            # handle DMs to bot users
1218
            if as1.is_dm(obj.as1):
1✔
1219
                return dms.receive(from_user=from_user, obj=obj)
1✔
1220

1221
        # fetch actor if necessary
1222
        if (actor and actor.keys() == set(['id'])
1✔
1223
                and obj.type not in ('delete', 'undo')):
1224
            logger.debug('Fetching actor so we have name, profile photo, etc')
1✔
1225
            actor_obj = from_cls.load(ids.profile_id(id=actor['id'], proto=from_cls),
1✔
1226
                                      raise_=False)
1227
            if actor_obj and actor_obj.as1:
1✔
1228
                obj.our_as1 = {
1✔
1229
                    **obj.as1, 'actor': {
1230
                        **actor_obj.as1,
1231
                        # override profile id with actor id
1232
                        # https://github.com/snarfed/bridgy-fed/issues/1720
1233
                        'id': actor['id'],
1234
                    }
1235
                }
1236

1237
        # fetch object if necessary
1238
        if (obj.type in ('post', 'update', 'share')
1✔
1239
                and inner_obj_as1.keys() == set(['id'])
1240
                and from_cls.owns_id(inner_obj_id) is not False):
1241
            logger.debug('Fetching inner object')
1✔
1242
            inner_obj = from_cls.load(inner_obj_id, raise_=False,
1✔
1243
                                      remote=(obj.type in ('post', 'update')))
1244
            if obj.type in ('post', 'update'):
1✔
1245
                crud_obj = inner_obj
1✔
1246
            if inner_obj and inner_obj.as1:
1✔
1247
                obj.our_as1 = {
1✔
1248
                    **obj.as1,
1249
                    'object': {
1250
                        **inner_obj_as1,
1251
                        **inner_obj.as1,
1252
                    }
1253
                }
1254
            elif obj.type in ('post', 'update'):
1✔
1255
                error(f"Need object {inner_obj_id} but couldn't fetch, giving up")
1✔
1256

1257
        if obj.type == 'follow':
1✔
1258
            if proto := Protocol.for_bridgy_subdomain(inner_obj_id):
1✔
1259
                # follow of one of our protocol bot users; enable that protocol.
1260
                # fall through so that we send an accept.
1261
                try:
1✔
1262
                    from_user.enable_protocol(proto)
1✔
1263
                except ErrorButDoNotRetryTask:
1✔
1264
                    from web import Web
1✔
1265
                    bot = Web.get_by_id(proto.bot_user_id())
1✔
1266
                    from_cls.respond_to_follow('reject', follower=from_user,
1✔
1267
                                               followee=bot, follow=obj)
1268
                    raise
1✔
1269
                proto.bot_maybe_follow_back(from_user)
1✔
1270

1271
            from_cls.handle_follow(obj, from_user=from_user)
1✔
1272

1273
        # deliver to targets
1274
        resp = from_cls.deliver(obj, from_user=from_user, crud_obj=crud_obj)
1✔
1275

1276
        # if this is a user, deactivate its followers/followings
1277
        # https://github.com/snarfed/bridgy-fed/issues/1304
1278
        if obj.type == 'delete':
1✔
1279
            if user_key := from_cls.key_for(id=inner_obj_id):
1✔
1280
                if user := user_key.get():
1✔
1281
                    for proto in user.enabled_protocols:
1✔
1282
                        user.disable_protocol(PROTOCOLS[proto])
1✔
1283

1284
                    logger.info(f'Deactivating Followers from or to {user_key.id()}')
1✔
1285
                    followers = Follower.query(
1✔
1286
                        OR(Follower.to == user_key, Follower.from_ == user_key)
1287
                        ).fetch()
1288
                    for f in followers:
1✔
1289
                        f.status = 'inactive'
1✔
1290
                    ndb.put_multi(followers)
1✔
1291

1292
        memcache.memcache.set(memcache_key, 'done', expire=7 * 24 * 60 * 60)  # 1w
1✔
1293
        return resp
1✔
1294

1295
    @classmethod
1✔
1296
    def handle_follow(from_cls, obj, from_user):
1✔
1297
        """Handles an incoming follow activity.
1298

1299
        Sends an ``Accept`` back, but doesn't send the ``Follow`` itself. That
1300
        happens in :meth:`deliver`.
1301

1302
        Args:
1303
          obj (models.Object): follow activity
1304
        """
1305
        logger.debug('Got follow. storing Follow(s), sending accept(s)')
1✔
1306
        from_id = from_user.key.id()
1✔
1307

1308
        # Prepare followee (to) users' data
1309
        to_as1s = as1.get_objects(obj.as1)
1✔
1310
        if not to_as1s:
1✔
1311
            error(f'Follow activity requires object(s). Got: {obj.as1}')
×
1312

1313
        # Store Followers
1314
        for to_as1 in to_as1s:
1✔
1315
            to_id = to_as1.get('id')
1✔
1316
            if not to_id:
1✔
1317
                error(f'Follow activity requires object(s). Got: {obj.as1}')
×
1318

1319
            logger.info(f'Follow {from_id} => {to_id}')
1✔
1320

1321
            to_cls = Protocol.for_id(to_id)
1✔
1322
            if not to_cls:
1✔
1323
                error(f"Couldn't determine protocol for {to_id}")
×
1324
            elif from_cls == to_cls:
1✔
1325
                logger.info(f'Skipping same-protocol Follower {from_id} => {to_id}')
1✔
1326
                continue
1✔
1327

1328
            to_key = to_cls.key_for(to_id)
1✔
1329
            if not to_key:
1✔
1330
                logger.info(f'Skipping invalid {from_cls.LABEL} user key: {from_id}')
×
1331
                continue
×
1332

1333
            to_user = to_cls.get_or_create(id=to_key.id())
1✔
1334
            if not to_user or not to_user.is_enabled(from_user):
1✔
1335
                error(f'{to_id} not found')
1✔
1336

1337
            follower_obj = Follower.get_or_create(to=to_user, from_=from_user,
1✔
1338
                                                  follow=obj.key, status='active')
1339
            obj.add('notify', to_key)
1✔
1340
            from_cls.respond_to_follow('accept', follower=from_user,
1✔
1341
                                       followee=to_user, follow=obj)
1342

1343
    @classmethod
1✔
1344
    def respond_to_follow(_, verb, follower, followee, follow):
1✔
1345
        """Sends an accept or reject activity for a follow.
1346

1347
        ...if the follower's protocol supports accepts/rejects. Otherwise, does
1348
        nothing.
1349

1350
        Args:
1351
          verb (str): ``accept`` or  ``reject``
1352
          follower (models.User)
1353
          followee (models.User)
1354
          follow (models.Object)
1355
        """
1356
        assert verb in ('accept', 'reject')
1✔
1357
        if verb not in follower.SUPPORTED_AS1_TYPES:
1✔
1358
            return
1✔
1359

1360
        if not follower.obj or not (target := follower.target_for(follower.obj)):
1✔
1361
            error(f"Couldn't find delivery target for follower {follower.key.id()}")
1✔
1362

1363
        # send. note that this is one response for the whole follow, even if it
1364
        # has multiple followees!
1365
        id = f'{followee.key.id()}/followers#{verb}-{follow.key.id()}'
1✔
1366
        accept = {
1✔
1367
            'id': id,
1368
            'objectType': 'activity',
1369
            'verb': verb,
1370
            'actor': followee.key.id(),
1371
            'object': follow.as1,
1372
        }
1373
        common.create_task(queue='send', id=id, our_as1=accept, url=target,
1✔
1374
                           protocol=follower.LABEL, user=followee.key.urlsafe())
1375

1376
    @classmethod
1✔
1377
    def bot_maybe_follow_back(bot_cls, user):
1✔
1378
        """Follow a user from a protocol bot user, if their protocol needs that.
1379

1380
        ...so that the protocol starts sending us their activities, if it needs
1381
        a follow for that (eg ActivityPub).
1382

1383
        Args:
1384
          user (User)
1385
        """
1386
        if not user.BOTS_FOLLOW_BACK:
1✔
1387
            return
1✔
1388

1389
        from web import Web
1✔
1390
        bot = Web.get_by_id(bot_cls.bot_user_id())
1✔
1391
        now = util.now().isoformat()
1✔
1392
        logger.info(f'Following {user.key.id()} back from bot user {bot.key.id()}')
1✔
1393

1394
        if not user.obj:
1✔
1395
            logger.info("  can't follow, user has no profile obj")
1✔
1396
            return
1✔
1397

1398
        target = user.target_for(user.obj)
1✔
1399
        follow_back_id = f'https://{bot.key.id()}/#follow-back-{user.key.id()}-{now}'
1✔
1400
        follow_back_as1 = {
1✔
1401
            'objectType': 'activity',
1402
            'verb': 'follow',
1403
            'id': follow_back_id,
1404
            'actor': bot.key.id(),
1405
            'object': user.key.id(),
1406
        }
1407
        common.create_task(queue='send', id=follow_back_id,
1✔
1408
                           our_as1=follow_back_as1, url=target,
1409
                           source_protocol='web', protocol=user.LABEL,
1410
                           user=bot.key.urlsafe())
1411

1412
    @classmethod
1✔
1413
    def handle_bare_object(cls, obj, authed_as=None):
1✔
1414
        """If obj is a bare object, wraps it in a create or update activity.
1415

1416
        Checks if we've seen it before.
1417

1418
        Args:
1419
          obj (models.Object)
1420
          authed_as (str): authenticated actor id who sent this activity
1421

1422
        Returns:
1423
          models.Object: ``obj`` if it's an activity, otherwise a new object
1424
        """
1425
        is_actor = obj.type in as1.ACTOR_TYPES
1✔
1426
        if not is_actor and obj.type not in ('note', 'article', 'comment'):
1✔
1427
            return obj
1✔
1428

1429
        obj_actor = ids.normalize_user_id(id=as1.get_owner(obj.as1), proto=cls)
1✔
1430
        now = util.now().isoformat()
1✔
1431

1432
        # occasionally we override the object, eg if this is a profile object
1433
        # coming in via a user with use_instead set
1434
        obj_as1 = obj.as1
1✔
1435
        if obj_id := obj.key.id():
1✔
1436
            if obj_as1_id := obj_as1.get('id'):
1✔
1437
                if obj_id != obj_as1_id:
1✔
1438
                    logger.info(f'Overriding AS1 object id {obj_as1_id} with Object id {obj_id}')
1✔
1439
                    obj_as1['id'] = obj_id
1✔
1440

1441
        # this is a raw post; wrap it in a create or update activity
1442
        if obj.changed or is_actor:
1✔
1443
            if obj.changed:
1✔
1444
                logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes')
1✔
1445
            else:
1446
                logger.info(f'Got actor profile object, wrapping in update')
1✔
1447
            id = f'{obj.key.id()}#bridgy-fed-update-{now}'
1✔
1448
            update_as1 = {
1✔
1449
                'objectType': 'activity',
1450
                'verb': 'update',
1451
                'id': id,
1452
                'actor': obj_actor,
1453
                'object': {
1454
                    # Mastodon requires the updated field for Updates, so
1455
                    # add a default value.
1456
                    # https://docs.joinmastodon.org/spec/activitypub/#supported-activities-for-statuses
1457
                    # https://socialhub.activitypub.rocks/t/what-could-be-the-reason-that-my-update-activity-does-not-work/2893/4
1458
                    # https://github.com/mastodon/documentation/pull/1150
1459
                    'updated': now,
1460
                    **obj_as1,
1461
                },
1462
            }
1463
            logger.debug(f'  AS1: {json_dumps(update_as1, indent=2)}')
1✔
1464
            return Object(id=id, our_as1=update_as1,
1✔
1465
                          source_protocol=obj.source_protocol)
1466

1467
        if (obj.new
1✔
1468
                # HACK: force query param here is specific to webmention
1469
                or 'force' in request.form):
1470
            create_id = f'{obj.key.id()}#bridgy-fed-create'
1✔
1471
            create_as1 = {
1✔
1472
                'objectType': 'activity',
1473
                'verb': 'post',
1474
                'id': create_id,
1475
                'actor': obj_actor,
1476
                'object': obj_as1,
1477
                'published': now,
1478
            }
1479
            logger.info(f'Wrapping in post')
1✔
1480
            logger.debug(f'  AS1: {json_dumps(create_as1, indent=2)}')
1✔
1481
            return Object(id=create_id, our_as1=create_as1,
1✔
1482
                          source_protocol=obj.source_protocol)
1483

1484
        error(f'{obj.key.id()} is unchanged, nothing to do', status=204)
1✔
1485

1486
    @classmethod
1✔
1487
    def deliver(from_cls, obj, from_user, crud_obj=None, to_proto=None):
1✔
1488
        """Delivers an activity to its external recipients.
1489

1490
        Args:
1491
          obj (models.Object): activity to deliver
1492
          from_user (models.User): user (actor) this activity is from
1493
          crud_obj (models.Object): if this is a create, update, or delete/undo
1494
            activity, the inner object that's being written, otherwise None.
1495
            (This object's ``notify`` and ``feed`` properties may be updated.)
1496
          to_proto (protocol.Protocol): optional; if provided, only deliver to
1497
            targets on this protocol
1498

1499
        Returns:
1500
          (str, int) tuple: Flask response
1501
        """
1502
        if to_proto:
1✔
1503
            logger.info(f'Only delivering to {to_proto.LABEL}')
1✔
1504

1505
        # find delivery targets. maps Target to Object or None
1506
        #
1507
        # ...then write the relevant object, since targets() has a side effect of
1508
        # setting the notify and feed properties (and dirty attribute)
1509
        targets = from_cls.targets(obj, from_user=from_user, crud_obj=crud_obj)
1✔
1510
        if to_proto:
1✔
1511
            targets = {t: obj for t, obj in targets.items()
1✔
1512
                       if t.protocol == to_proto.LABEL}
1513
        if not targets:
1✔
1514
            return r'No targets, nothing to do ¯\_(ツ)_/¯', 204
1✔
1515

1516
        # store object that targets() updated
1517
        if crud_obj and crud_obj.dirty:
1✔
1518
            crud_obj.put()
1✔
1519
        elif obj.type in STORE_AS1_TYPES and obj.dirty:
1✔
1520
            obj.put()
1✔
1521

1522
        obj_params = ({'obj_id': obj.key.id()} if obj.type in STORE_AS1_TYPES
1✔
1523
                      else obj.to_request())
1524

1525
        # sort targets so order is deterministic for tests, debugging, etc
1526
        sorted_targets = sorted(targets.items(), key=lambda t: t[0].uri)
1✔
1527

1528
        # enqueue send task for each targets
1529
        logger.info(f'Delivering to: {[t for t, _ in sorted_targets]}')
1✔
1530
        user = from_user.key.urlsafe()
1✔
1531
        for i, (target, orig_obj) in enumerate(sorted_targets):
1✔
1532
            orig_obj_id = orig_obj.key.id() if orig_obj else None
1✔
1533
            common.create_task(queue='send', url=target.uri, protocol=target.protocol,
1✔
1534
                               orig_obj_id=orig_obj_id, user=user, **obj_params)
1535

1536
        return 'OK', 202
1✔
1537

1538
    @classmethod
1✔
1539
    def targets(from_cls, obj, from_user, crud_obj=None, internal=False):
1✔
1540
        """Collects the targets to send a :class:`models.Object` to.
1541

1542
        Targets are both objects - original posts, events, etc - and actors.
1543

1544
        Args:
1545
          obj (models.Object)
1546
          from_user (User)
1547
          crud_obj (models.Object): if this is a create, update, or delete/undo
1548
            activity, the inner object that's being written, otherwise None.
1549
            (This object's ``notify`` and ``feed`` properties may be updated.)
1550
          internal (bool): whether this is a recursive internal call
1551

1552
        Returns:
1553
          dict: maps :class:`models.Target` to original (in response to)
1554
          :class:`models.Object`, if any, otherwise None
1555
        """
1556
        logger.debug('Finding recipients and their targets')
1✔
1557

1558
        # we should only have crud_obj iff this is a create or update
1559
        assert (crud_obj is not None) == (obj.type in ('post', 'update')), obj.type
1✔
1560
        write_obj = crud_obj or obj
1✔
1561
        write_obj.dirty = False
1✔
1562

1563
        target_uris = as1.targets(obj.as1)
1✔
1564
        orig_obj = None
1✔
1565
        targets = {}  # maps Target to Object or None
1✔
1566
        owner = as1.get_owner(obj.as1)
1✔
1567
        allow_opt_out = (obj.type == 'delete')
1✔
1568
        inner_obj_as1 = as1.get_object(obj.as1)
1✔
1569
        inner_obj_id = inner_obj_as1.get('id')
1✔
1570
        in_reply_tos = as1.get_ids(inner_obj_as1, 'inReplyTo')
1✔
1571
        quoted_posts = as1.quoted_posts(inner_obj_as1)
1✔
1572
        mentioned_urls = as1.mentions(inner_obj_as1)
1✔
1573
        is_reply = obj.type == 'comment' or in_reply_tos
1✔
1574
        is_self_reply = False
1✔
1575

1576
        original_ids = []
1✔
1577
        if is_reply:
1✔
1578
            original_ids = in_reply_tos
1✔
1579
        elif inner_obj_id:
1✔
1580
            if inner_obj_id == from_user.key.id():
1✔
1581
                inner_obj_id = from_user.profile_id()
1✔
1582
            original_ids = [inner_obj_id]
1✔
1583

1584
        original_objs = {}
1✔
1585
        for id in original_ids:
1✔
1586
            if proto := Protocol.for_id(id):
1✔
1587
                original_objs[id] = proto.load(id, raise_=False)
1✔
1588

1589
        # for AP, add in-reply-tos' mentions
1590
        # https://github.com/snarfed/bridgy-fed/issues/1608
1591
        # https://github.com/snarfed/bridgy-fed/issues/1218
1592
        orig_post_mentions = {}  # maps mentioned id to original post Object
1✔
1593
        for id in in_reply_tos:
1✔
1594
            if ((in_reply_to_obj := original_objs.get(id))
1✔
1595
                    and (proto := PROTOCOLS.get(in_reply_to_obj.source_protocol))
1596
                    and proto.SEND_REPLIES_TO_ORIG_POSTS_MENTIONS
1597
                    and (mentions := as1.mentions(in_reply_to_obj.as1))):
1598
                logger.info(f"Adding in-reply-to {id} 's mentions to targets: {mentions}")
1✔
1599
                target_uris.extend(mentions)
1✔
1600
                for mention in mentions:
1✔
1601
                    orig_post_mentions[mention] = in_reply_to_obj
1✔
1602

1603
        target_uris = sorted(set(target_uris))
1✔
1604
        logger.info(f'Raw targets: {target_uris}')
1✔
1605

1606
        # which protocols should we allow delivering to?
1607
        to_protocols = []
1✔
1608
        for label in (list(from_user.DEFAULT_ENABLED_PROTOCOLS)
1✔
1609
                      + from_user.enabled_protocols):
1610
            if not (proto := PROTOCOLS.get(label)):
1✔
1611
                report_error(f'unknown enabled protocol {label} for {from_user.key.id()}')
1✔
1612
                continue
1✔
1613

1614
            if proto.HAS_COPIES and (obj.type in ('update', 'delete', 'share', 'undo')
1✔
1615
                                     or is_reply):
1616
                origs_could_bridge = None
1✔
1617

1618
                for id in original_ids:
1✔
1619
                    if not (orig := original_objs.get(id)):
1✔
1620
                        continue
1✔
1621
                    elif isinstance(orig, proto):
1✔
1622
                        logger.info(f'Allowing {label} for original post {id}')
×
1623
                        break
×
1624
                    elif orig.get_copy(proto):
1✔
1625
                        logger.info(f'Allowing {label}, original post {id} was bridged there')
1✔
1626
                        break
1✔
1627

1628
                    if (origs_could_bridge is not False
1✔
1629
                            and (orig_author_id := as1.get_owner(orig.as1))
1630
                            and (orig_proto := PROTOCOLS.get(orig.source_protocol))
1631
                            and (orig_author := orig_proto.get_by_id(orig_author_id))):
1632
                        origs_could_bridge = orig_author.is_enabled(proto)
1✔
1633

1634
                else:
1635
                    msg = f"original object(s) {original_ids} weren't bridged to {label}"
1✔
1636
                    if (proto.LABEL not in from_user.DEFAULT_ENABLED_PROTOCOLS
1✔
1637
                            and origs_could_bridge):
1638
                        # retry later; original obj may still be bridging
1639
                        # TODO: limit to brief window, eg no older than 2h? 1d?
1640
                        error(msg, status=304)
1✔
1641

1642
                    logger.info(msg)
1✔
1643
                    continue
1✔
1644

1645

1646
            util.add(to_protocols, proto)
1✔
1647

1648
        # process direct targets
1649
        for target_id in target_uris:
1✔
1650
            target_proto = Protocol.for_id(target_id)
1✔
1651
            if not target_proto:
1✔
1652
                logger.info(f"Can't determine protocol for {target_id}")
1✔
1653
                continue
1✔
1654
            elif target_proto.is_blocklisted(target_id):
1✔
1655
                logger.debug(f'{target_id} is blocklisted')
1✔
1656
                continue
1✔
1657

1658
            orig_obj = target_proto.load(target_id, raise_=False)
1✔
1659
            if not orig_obj or not orig_obj.as1:
1✔
1660
                logger.info(f"Couldn't load {target_id}")
1✔
1661
                continue
1✔
1662

1663
            target_author_key = (target_proto(id=target_id).key
1✔
1664
                                 if target_id in mentioned_urls
1665
                                 else target_proto.actor_key(orig_obj))
1666
            if not from_user.is_enabled(target_proto):
1✔
1667
                # if author isn't bridged and target user is, DM a prompt and
1668
                # add a notif for the target user
1669
                if (target_id in (in_reply_tos + quoted_posts + mentioned_urls)
1✔
1670
                        and target_author_key):
1671
                    if target_author := target_author_key.get():
1✔
1672
                        if target_author.is_enabled(from_cls):
1✔
1673
                            notifications.add_notification(target_author, write_obj)
1✔
1674
                            verb, noun = (
1✔
1675
                                ('replied to', 'replies') if target_id in in_reply_tos
1676
                                else ('quoted', 'quotes') if target_id in quoted_posts
1677
                                else ('mentioned', 'mentions'))
1678
                            dms.maybe_send(from_=target_proto, to_user=from_user,
1✔
1679
                                           type='replied_to_bridged_user', text=f"""\
1680
Hi! You <a href="{inner_obj_as1.get('url') or inner_obj_id}">recently {verb}</a> {target_author.user_link()}, who's bridged here from {target_proto.PHRASE}. If you want them to see your {noun}, you can bridge your account into {target_proto.PHRASE} by following this account. <a href="https://fed.brid.gy/docs">See the docs</a> for more information.""")
1681

1682
                continue
1✔
1683

1684
            # deliver self-replies to followers
1685
            # https://github.com/snarfed/bridgy-fed/issues/639
1686
            if target_id in in_reply_tos and owner == as1.get_owner(orig_obj.as1):
1✔
1687
                is_self_reply = True
1✔
1688
                logger.info(f'self reply!')
1✔
1689

1690
            # also add copies' targets
1691
            for copy in orig_obj.copies:
1✔
1692
                proto = PROTOCOLS[copy.protocol]
1✔
1693
                if proto in to_protocols:
1✔
1694
                    # copies generally won't have their own Objects
1695
                    if target := proto.target_for(Object(id=copy.uri)):
1✔
1696
                        logger.debug(f'Adding target {target} for copy {copy.uri} of original {target_id}')
1✔
1697
                        targets[Target(protocol=copy.protocol, uri=target)] = orig_obj
1✔
1698

1699
            if target_proto == from_cls:
1✔
1700
                logger.debug(f'Skipping same-protocol target {target_id}')
1✔
1701
                continue
1✔
1702

1703
            target = target_proto.target_for(orig_obj)
1✔
1704
            if not target:
1✔
1705
                # TODO: surface errors like this somehow?
1706
                logger.error(f"Can't find delivery target for {target_id}")
1✔
1707
                continue
1✔
1708

1709
            logger.debug(f'Target for {target_id} is {target}')
1✔
1710
            # only use orig_obj for inReplyTos, like/repost objects, reply's original
1711
            # post's mentions, etc
1712
            # https://github.com/snarfed/bridgy-fed/issues/1237
1713
            target_obj = None
1✔
1714
            if target_id in in_reply_tos + as1.get_ids(obj.as1, 'object'):
1✔
1715
                target_obj = orig_obj
1✔
1716
            elif target_id in orig_post_mentions:
1✔
1717
                target_obj = orig_post_mentions[target_id]
1✔
1718
            targets[Target(protocol=target_proto.LABEL, uri=target)] = target_obj
1✔
1719

1720
            if target_author_key:
1✔
1721
                logger.debug(f'Recipient is {target_author_key}')
1✔
1722
                if write_obj.add('notify', target_author_key):
1✔
1723
                    write_obj.dirty = True
1✔
1724

1725
        if obj.type == 'undo':
1✔
1726
            logger.debug('Object is an undo; adding targets for inner object')
1✔
1727
            if set(inner_obj_as1.keys()) == {'id'}:
1✔
1728
                inner_obj = from_cls.load(inner_obj_id, raise_=False)
1✔
1729
            else:
1730
                inner_obj = Object(id=inner_obj_id, our_as1=inner_obj_as1)
1✔
1731
            if inner_obj:
1✔
1732
                targets.update(from_cls.targets(inner_obj, from_user=from_user,
1✔
1733
                                                internal=True))
1734

1735
        logger.info(f'Direct targets: {[t.uri for t in targets.keys()]}')
1✔
1736

1737
        # deliver to followers, if appropriate
1738
        user_key = from_cls.actor_key(obj, allow_opt_out=allow_opt_out)
1✔
1739
        if not user_key:
1✔
1740
            logger.info("Can't tell who this is from! Skipping followers.")
1✔
1741
            return targets
1✔
1742

1743
        followers = []
1✔
1744
        if (obj.type in ('post', 'update', 'delete', 'move', 'share', 'undo')
1✔
1745
                and (not is_reply or is_self_reply)):
1746
            logger.info(f'Delivering to followers of {user_key}')
1✔
1747
            followers = []
1✔
1748
            for f in Follower.query(Follower.to == user_key,
1✔
1749
                                    Follower.status == 'active'):
1750
                proto = PROTOCOLS_BY_KIND[f.from_.kind()]
1✔
1751
                # skip protocol bot users
1752
                if (not Protocol.for_bridgy_subdomain(f.from_.id())
1✔
1753
                        # skip protocols this user hasn't enabled, or where the base
1754
                        # object of this activity hasn't been bridged
1755
                        and proto in to_protocols
1756
                        # we deliver to HAS_COPIES protocols separately, below. we
1757
                        # assume they have follower-independent targets.
1758
                        and not (proto.HAS_COPIES and proto.DEFAULT_TARGET)):
1759
                    followers.append(f)
1✔
1760

1761
            user_keys = [f.from_ for f in followers]
1✔
1762
            users = [u for u in ndb.get_multi(user_keys) if u]
1✔
1763
            User.load_multi(users)
1✔
1764

1765
            if (not followers and
1✔
1766
                (util.domain_or_parent_in(from_user.key.id(), LIMITED_DOMAINS)
1767
                 or util.domain_or_parent_in(obj.key.id(), LIMITED_DOMAINS))):
1768
                logger.info(f'skipping, {from_user.key.id()} is on a limited domain and has no followers')
1✔
1769
                return {}
1✔
1770

1771
            # add to followers' feeds, if any
1772
            if not internal and obj.type in ('post', 'update', 'share'):
1✔
1773
                if write_obj.type not in as1.ACTOR_TYPES:
1✔
1774
                    write_obj.feed = [u.key for u in users if u.USES_OBJECT_FEED]
1✔
1775
                    if write_obj.feed:
1✔
1776
                        write_obj.dirty = True
1✔
1777

1778
            # collect targets for followers
1779
            for user in users:
1✔
1780
                # TODO: should we pass remote=False through here to Protocol.load?
1781
                target = user.target_for(user.obj, shared=True) if user.obj else None
1✔
1782
                if not target:
1✔
1783
                    # logger.error(f'Follower {user.key} has no delivery target')
1784
                    continue
1✔
1785

1786
                # normalize URL (lower case hostname, etc)
1787
                # ...but preserve our PDS URL without trailing slash in path
1788
                # https://atproto.com/specs/did#did-documents
1789
                target = util.dedupe_urls([target], trailing_slash=False)[0]
1✔
1790

1791
                targets[Target(protocol=user.LABEL, uri=target)] = \
1✔
1792
                    Object.get_by_id(inner_obj_id) if obj.type == 'share' else None
1793

1794
        # deliver to enabled HAS_COPIES protocols proactively
1795
        if obj.type in ('post', 'update', 'delete', 'share'):
1✔
1796
            for proto in to_protocols:
1✔
1797
                if proto.HAS_COPIES and proto.DEFAULT_TARGET:
1✔
1798
                    logger.info(f'user has {proto.LABEL} enabled, adding {proto.DEFAULT_TARGET}')
1✔
1799
                    targets.setdefault(
1✔
1800
                        Target(protocol=proto.LABEL, uri=proto.DEFAULT_TARGET), None)
1801

1802
        # de-dupe targets, discard same-domain
1803
        # maps string target URL to (Target, Object) tuple
1804
        candidates = {t.uri: (t, obj) for t, obj in targets.items()}
1✔
1805
        # maps Target to Object or None
1806
        targets = {}
1✔
1807
        source_domains = [
1✔
1808
            util.domain_from_link(url) for url in
1809
            (obj.as1.get('id'), obj.as1.get('url'), as1.get_owner(obj.as1))
1810
            if util.is_web(url)
1811
        ]
1812
        for url in sorted(util.dedupe_urls(
1✔
1813
                candidates.keys(),
1814
                # preserve our PDS URL without trailing slash in path
1815
                # https://atproto.com/specs/did#did-documents
1816
                trailing_slash=False)):
1817
            if util.is_web(url) and util.domain_from_link(url) in source_domains:
1✔
1818
                logger.info(f'Skipping same-domain target {url}')
×
1819
                continue
×
1820
            target, obj = candidates[url]
1✔
1821
            targets[target] = obj
1✔
1822

1823
        return targets
1✔
1824

1825
    @classmethod
1✔
1826
    def load(cls, id, remote=None, local=True, raise_=True, **kwargs):
1✔
1827
        """Loads and returns an Object from datastore or HTTP fetch.
1828

1829
        Sets the :attr:`new` and :attr:`changed` attributes if we know either
1830
        one for the loaded object, ie local is True and remote is True or None.
1831

1832
        Args:
1833
          id (str)
1834
          remote (bool): whether to fetch the object over the network. If True,
1835
            fetches even if we already have the object stored, and updates our
1836
            stored copy. If False and we don't have the object stored, returns
1837
            None. Default (None) means to fetch over the network only if we
1838
            don't already have it stored.
1839
          local (bool): whether to load from the datastore before
1840
            fetching over the network. If False, still stores back to the
1841
            datastore after a successful remote fetch.
1842
          raise_ (bool): if False, catches any :class:`request.RequestException`
1843
            or :class:`HTTPException` raised by :meth:`fetch()` and returns
1844
            ``None`` instead
1845
          kwargs: passed through to :meth:`fetch()`
1846

1847
        Returns:
1848
          models.Object: loaded object, or None if it isn't fetchable, eg a
1849
          non-URL string for Web, or ``remote`` is False and it isn't in the
1850
          datastore
1851

1852
        Raises:
1853
          requests.HTTPError: anything that :meth:`fetch` raises, if ``raise_``
1854
            is True
1855
        """
1856
        assert id
1✔
1857
        assert local or remote is not False
1✔
1858
        # logger.debug(f'Loading Object {id} local={local} remote={remote}')
1859

1860
        obj = orig_as1 = None
1✔
1861
        if local:
1✔
1862
            obj = Object.get_by_id(id)
1✔
1863
            if not obj:
1✔
1864
                # logger.debug(f' {id} not in datastore')
1865
                pass
1✔
1866
            elif obj.as1 or obj.raw or obj.deleted:
1✔
1867
                # logger.debug(f'  {id} got from datastore')
1868
                obj.new = False
1✔
1869

1870
        if remote is False:
1✔
1871
            return obj
1✔
1872
        elif remote is None and obj:
1✔
1873
            if obj.updated < util.as_utc(util.now() - OBJECT_REFRESH_AGE):
1✔
1874
                # logger.debug(f'  last updated {obj.updated}, refreshing')
1875
                pass
1✔
1876
            else:
1877
                return obj
1✔
1878

1879
        if obj:
1✔
1880
            orig_as1 = obj.as1
1✔
1881
            obj.our_as1 = None
1✔
1882
            obj.new = False
1✔
1883
        else:
1884
            obj = Object(id=id)
1✔
1885
            if local:
1✔
1886
                # logger.debug(f'  {id} not in datastore')
1887
                obj.new = True
1✔
1888
                obj.changed = False
1✔
1889

1890
        try:
1✔
1891
            fetched = cls.fetch(obj, **kwargs)
1✔
1892
        except (RequestException, HTTPException) as e:
1✔
1893
            if raise_:
1✔
1894
                raise
1✔
1895
            util.interpret_http_exception(e)
1✔
1896
            return None
1✔
1897

1898
        if not fetched:
1✔
1899
            return None
1✔
1900

1901
        # https://stackoverflow.com/a/3042250/186123
1902
        size = len(_entity_to_protobuf(obj)._pb.SerializeToString())
1✔
1903
        if size > models.MAX_ENTITY_SIZE:
1✔
1904
            logger.warning(f'Object is too big! {size} bytes is over {models.MAX_ENTITY_SIZE}')
1✔
1905
            return None
1✔
1906

1907
        obj.resolve_ids()
1✔
1908
        obj.normalize_ids()
1✔
1909

1910
        if obj.new is False:
1✔
1911
            obj.changed = obj.activity_changed(orig_as1)
1✔
1912

1913
        if obj.source_protocol not in (cls.LABEL, cls.ABBREV):
1✔
1914
            if obj.source_protocol:
1✔
1915
                logger.warning(f'Object {obj.key.id()} changed protocol from {obj.source_protocol} to {cls.LABEL} ?!')
×
1916
            obj.source_protocol = cls.LABEL
1✔
1917

1918
        obj.put()
1✔
1919
        return obj
1✔
1920

1921
    @classmethod
1✔
1922
    def check_supported(cls, obj, direction):
1✔
1923
        """If this protocol doesn't support this activity, raises HTTP 204.
1924

1925
        Also reports an error.
1926

1927
        (This logic is duplicated in some protocols, eg ActivityPub, so that
1928
        they can short circuit out early. It generally uses their native formats
1929
        instead of AS1, before an :class:`models.Object` is created.)
1930

1931
        Args:
1932
          obj (Object)
1933
          direction (str): ``'receive'`` or  ``'send'``
1934

1935
        Raises:
1936
          werkzeug.HTTPException: if this protocol doesn't support this object
1937
        """
1938
        assert direction in ('receive', 'send')
1✔
1939
        if not obj.type:
1✔
1940
            return
×
1941

1942
        inner = as1.get_object(obj.as1)
1✔
1943
        inner_type = as1.object_type(inner) or ''
1✔
1944
        if (obj.type not in cls.SUPPORTED_AS1_TYPES
1✔
1945
            or (obj.type in as1.CRUD_VERBS
1946
                and inner_type
1947
                and inner_type not in cls.SUPPORTED_AS1_TYPES)):
1948
            error(f"Bridgy Fed for {cls.LABEL} doesn't support {obj.type} {inner_type} yet", status=204)
1✔
1949

1950
        # don't allow posts with blank content and no image/video/audio
1951
        crud_obj = (as1.get_object(obj.as1) if obj.type in ('post', 'update')
1✔
1952
                    else obj.as1)
1953
        if (crud_obj.get('objectType') in as1.POST_TYPES
1✔
1954
                and not util.get_url(crud_obj, key='image')
1955
                and not any(util.get_urls(crud_obj, 'attachments', inner_key='stream'))
1956
                # TODO: handle articles with displayName but not content
1957
                and not source.html_to_text(crud_obj.get('content')).strip()):
1958
            error('Blank content and no image or video or audio', status=204)
1✔
1959

1960
        # receiving DMs is only allowed to protocol bot accounts
1961
        if direction == 'receive':
1✔
1962
            if recip := as1.recipient_if_dm(obj.as1):
1✔
1963
                owner = as1.get_owner(obj.as1)
1✔
1964
                if (not cls.SUPPORTS_DMS or (recip not in common.bot_user_ids()
1✔
1965
                                             and owner not in common.bot_user_ids())):
1966
                    # reply and say DMs aren't supported
1967
                    from_proto = PROTOCOLS.get(obj.source_protocol)
1✔
1968
                    to_proto = Protocol.for_id(recip)
1✔
1969
                    if owner and from_proto and to_proto:
1✔
1970
                        if ((from_user := from_proto.get_or_create(id=owner))
1✔
1971
                                and (to_user := to_proto.get_or_create(id=recip))):
1972
                            in_reply_to = (inner.get('id') if obj.type == 'post'
1✔
1973
                                           else obj.as1.get('id'))
1974
                            text = f"Hi! Sorry, this account is bridged from {to_user.PHRASE}, so it doesn't support DMs. Try getting in touch another way!"
1✔
1975
                            type = f'dms_not_supported-{to_user.key.id()}'
1✔
1976
                            dms.maybe_send(from_=to_user, to_user=from_user,
1✔
1977
                                           text=text, type=type,
1978
                                           in_reply_to=in_reply_to)
1979

1980
                    error("Bridgy Fed doesn't support DMs", status=204)
1✔
1981

1982
            # check that this activity is public. only do this for some activities,
1983
            # not eg likes or follows, since Mastodon doesn't currently mark those
1984
            # as explicitly public.
1985
            elif (obj.type in set(('post', 'update')) | as1.POST_TYPES | as1.ACTOR_TYPES
1✔
1986
                      and not as1.is_public(obj.as1, unlisted=False)):
1987
                  error('Bridgy Fed only supports public activities', status=204)
1✔
1988

1989

1990
@cloud_tasks_only(log=None)
1✔
1991
def receive_task():
1✔
1992
    """Task handler for a newly received :class:`models.Object`.
1993

1994
    Calls :meth:`Protocol.receive` with the form parameters.
1995

1996
    Parameters:
1997
      authed_as (str): passed to :meth:`Protocol.receive`
1998
      obj_id (str): key id of :class:`models.Object` to handle
1999
      received_at (str, ISO 8601 timestamp): when we first saw (received)
2000
        this activity
2001
      *: If ``obj_id`` is unset, all other parameters are properties for a new
2002
        :class:`models.Object` to handle
2003

2004
    TODO: migrate incoming webmentions to this. See how we did it for AP. The
2005
    difficulty is that parts of :meth:`protocol.Protocol.receive` depend on
2006
    setup in :func:`web.webmention`, eg :class:`models.Object` with ``new`` and
2007
    ``changed``, HTTP request details, etc. See stash for attempt at this for
2008
    :class:`web.Web`.
2009
    """
2010
    common.log_request()
1✔
2011
    form = request.form.to_dict()
1✔
2012

2013
    authed_as = form.pop('authed_as', None)
1✔
2014
    internal = (authed_as == common.PRIMARY_DOMAIN
1✔
2015
                or authed_as in common.PROTOCOL_DOMAINS)
2016

2017
    obj = Object.from_request()
1✔
2018
    assert obj
1✔
2019
    assert obj.source_protocol
1✔
2020
    obj.new = True
1✔
2021

2022
    if received_at := form.pop('received_at', None):
1✔
2023
        received_at = datetime.fromisoformat(received_at)
1✔
2024

2025
    try:
1✔
2026
        return PROTOCOLS[obj.source_protocol].receive(
1✔
2027
            obj=obj, authed_as=authed_as, internal=internal, received_at=received_at)
2028
    except RequestException as e:
1✔
2029
        util.interpret_http_exception(e)
1✔
2030
        error(e, status=304)
1✔
2031
    except ValueError as e:
1✔
2032
        logger.warning(e, exc_info=True)
×
2033
        error(e, status=304)
×
2034

2035

2036
@cloud_tasks_only(log=None)
1✔
2037
def send_task():
1✔
2038
    """Task handler for sending an activity to a single specific destination.
2039

2040
    Calls :meth:`Protocol.send` with the form parameters.
2041

2042
    Parameters:
2043
      protocol (str): :class:`Protocol` to send to
2044
      url (str): destination URL to send to
2045
      obj_id (str): key id of :class:`models.Object` to send
2046
      orig_obj_id (str): optional, :class:`models.Object` key id of the
2047
        "original object" that this object refers to, eg replies to or reposts
2048
        or likes
2049
      user (url-safe google.cloud.ndb.key.Key): :class:`models.User` (actor)
2050
        this activity is from
2051
      *: If ``obj_id`` is unset, all other parameters are properties for a new
2052
        :class:`models.Object` to handle
2053
    """
2054
    common.log_request()
1✔
2055

2056
    # prepare
2057
    form = request.form.to_dict()
1✔
2058
    url = form.get('url')
1✔
2059
    protocol = form.get('protocol')
1✔
2060
    if not url or not protocol:
1✔
2061
        logger.warning(f'Missing protocol or url; got {protocol} {url}')
1✔
2062
        return '', 204
1✔
2063

2064
    target = Target(uri=url, protocol=protocol)
1✔
2065
    obj = Object.from_request()
1✔
2066
    assert obj and obj.key and obj.key.id()
1✔
2067

2068
    PROTOCOLS[protocol].check_supported(obj, 'send')
1✔
2069
    allow_opt_out = (obj.type == 'delete')
1✔
2070

2071
    user = None
1✔
2072
    if user_key := form.get('user'):
1✔
2073
        key = ndb.Key(urlsafe=user_key)
1✔
2074
        # use get_by_id so that we follow use_instead
2075
        user = PROTOCOLS_BY_KIND[key.kind()].get_by_id(
1✔
2076
            key.id(), allow_opt_out=allow_opt_out)
2077

2078
    # send
2079
    delay = ''
1✔
2080
    if request.headers.get('X-AppEngine-TaskRetryCount') == '0' and obj.created:
1✔
2081
        delay_s = int((util.now().replace(tzinfo=None) - obj.created).total_seconds())
1✔
2082
        delay = f'({delay_s} s behind)'
1✔
2083
    logger.info(f'Sending {obj.source_protocol} {obj.type} {obj.key.id()} to {protocol} {url} {delay}')
1✔
2084
    logger.debug(f'  AS1: {json_dumps(obj.as1, indent=2)}')
1✔
2085
    sent = None
1✔
2086
    try:
1✔
2087
        sent = PROTOCOLS[protocol].send(obj, url, from_user=user,
1✔
2088
                                        orig_obj_id=form.get('orig_obj_id'))
2089
    except BaseException as e:
1✔
2090
        code, body = util.interpret_http_exception(e)
1✔
2091
        if not code and not body:
1✔
2092
            raise
1✔
2093

2094
    if sent is False:
1✔
2095
        logger.info(f'Failed sending!')
1✔
2096

2097
    return '', 200 if sent else 204 if sent is False else 304
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc