81bd7efa-7e2f-44be-91cc-76a322ca2564

Committed 28 May 2025 08:14PM UTC coverage: 85.374% (+0.02%) from 85.353%

Build # 81bd7efa-7e2f-44be-91cc-76a322ca2564

Build Type

Pull #5572

circleci

Committed by

groovecoder

Commit Message

for MPP-3439: add ensure_ascii_email and call before sending an email

Pull Request Pull Request #5572: for MPP-3439: add IDNAEmailCleaner to clean email address domains with non-ASCII chars

Run Details

2479 of 3627 branches covered (68.35%)

Branch coverage included in aggregate %.

105 of 107 new or added lines in 6 files covered. (98.13%)

10 existing lines in 2 files now uncovered.

17607 of 19900 relevant lines covered (88.48%)

9.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.65

/privaterelay/cleaners.py

"""Tasks that detect and fix data issues in privaterelay app or 3rd party apps."""

from django.contrib.auth.models import User
from django.db import transaction
from django.db.models import Q, QuerySet, Value
from django.db.models.functions import Coalesce, NullIf

from allauth.socialaccount.models import EmailAddress, SocialAccount, SocialApp

from .cleaner_task import CleanerTask, DataBisectSpec, DataModelSpec


class MissingEmailCleaner(CleanerTask):
    slug = "missing-email"
    title = "Ensure all users have an email"
    check_description = (
        "When User.email is empty, we are unable to forward email to the Relay user."
        " We can get the email from the FxA profile if available."
    )

    # The Firefox Accounts default provider identifier is `fxa`. Firefox Accounts was
    # the name for Mozilla Accounts before 2023. This query returns the value, as a
    # one-element list, of the `SocialApp.provider_id` if set, and `fxa` if not.
    #
    # The `provider` field for a SocialAccount is a CharField, not a ForeignKey.  The
    # default `provider` value is the `id` of the SocialAccount provider. This `id` is
    # used in the django-allauth URLs. The `provider` value can be overridden by setting
    # the `SocialApp.provider_id`. This supports generic providers like OpenID Connect
    # and SAML. When it is set on a non-generic provider, it changes the value of the
    # SocialAccount's `provider`, but not the URLs. When django-allauth needs the
    # SocialApp for a given SocialAccount, it uses an adapter to look it up at runtime.
    #
    # See:
    # - The Firefox Account Provider docs
    # https://docs.allauth.org/en/latest/socialaccount/providers/fxa.html
    # - The OpenID Connect docs
    # https://docs.allauth.org/en/latest/socialaccount/providers/openid_connect.html
    # - The DefaultSocialAccountAdapter docs
    # https://docs.allauth.org/en/latest/socialaccount/adapter.html#allauth.socialaccount.adapter.DefaultSocialAccountAdapter.get_provider

    _fxa_provider_id = SocialApp.objects.filter(provider="fxa").values_list(
        Coalesce(NullIf("provider_id", Value("")), "provider"), flat=True
    )

    data_specification = [
        # Report on how many users do not have an email
        DataModelSpec(
            model=User,
            subdivisions=[
                DataBisectSpec("email", ~Q(email__exact="")),
                DataBisectSpec(
                    "!email.fxa", Q(socialaccount__provider__in=_fxa_provider_id)
                ),
            ],
            omit_key_prefixes=["!email.!fxa"],
            report_name_overrides={"!email": "No Email", "!email.fxa": "Has FxA"},
            ok_key="email",
            needs_cleaning_key="!email.fxa",
        )
    ]

    def clean_users(self, queryset: QuerySet[User]) -> int:
        fixed = 0
        for user in queryset:
            try:
                fxa_account = SocialAccount.objects.get(
                    provider__in=self._fxa_provider_id, user=user
                )
            except SocialAccount.DoesNotExist:
                continue
            if fxa_email := fxa_account.extra_data.get("email"):
                user.email = fxa_email
                user.save(update_fields=["email"])
                fixed += 1
        return fixed


class IDNAEmailCleaner(CleanerTask):
    slug = "idna-email"
    title = "Fix non-ASCII domains in user emails"
    check_description = (
        "Users with non-ASCII characters in their email domain cannot receive emails"
        "via AWS SES. Convert these domains to ASCII-compatible Punycode using IDNA."
    )

    def has_non_ascii_domain(self, email: str) -> bool:
        if not email or "@" not in email:
            return False
        try:
            domain = email.split("@", 1)[1]
            domain.encode("ascii")
            return False
        except UnicodeEncodeError:
            return True

    def punycode_email(self, email: str) -> str:
        local, domain = email.split("@", 1)
        domain_ascii = domain.encode("idna").decode("ascii")
        return f"{local}@{domain_ascii}"

    data_specification = [
        DataModelSpec(
            model=User,
            subdivisions=[
                DataBisectSpec(
                    "ascii_domain",
                    ~Q(email__regex=r".*@.*[^\x00-\x7F].*"),
                )
            ],
            report_name_overrides={
                "ascii_domain": "Has ASCII Domain",
                "!ascii_domain": "Non-ASCII Domain",
            },
            ok_key="ascii_domain",
            needs_cleaning_key="!ascii_domain",
        )
    ]

    def clean_users(self, queryset: QuerySet[User]) -> int:
        fixed = 0
        for user in queryset:
            if user.email and self.has_non_ascii_domain(user.email):
                updated_email = self.punycode_email(user.email)
                with transaction.atomic():
                    user.email = updated_email
                    user.save(update_fields=["email"])
                    EmailAddress.objects.filter(user=user).update(email=updated_email)
                    sa = SocialAccount.objects.filter(user=user).first()
                    if sa is not None:
                        sa.extra_data["email"] = updated_email
                        sa.save(update_fields=["extra_data"])
                fixed += 1
        return fixed

1	"""Tasks that detect and fix data issues in privaterelay app or 3rd party apps."""
2
3	from django.contrib.auth.models import User	1✔
4	from django.db import transaction	1✔
5	from django.db.models import Q, QuerySet, Value	1✔
6	from django.db.models.functions import Coalesce, NullIf	1✔
7
8	from allauth.socialaccount.models import EmailAddress, SocialAccount, SocialApp	1✔
9
10	from .cleaner_task import CleanerTask, DataBisectSpec, DataModelSpec	1✔
11
12
13	class MissingEmailCleaner(CleanerTask):	1✔
14	slug = "missing-email"	1✔
15	title = "Ensure all users have an email"	1✔
16	check_description = (	1✔
17	"When User.email is empty, we are unable to forward email to the Relay user."
18	" We can get the email from the FxA profile if available."
19	)
20
21	# The Firefox Accounts default provider identifier is `fxa`. Firefox Accounts was
22	# the name for Mozilla Accounts before 2023. This query returns the value, as a
23	# one-element list, of the `SocialApp.provider_id` if set, and `fxa` if not.
24	#
25	# The `provider` field for a SocialAccount is a CharField, not a ForeignKey. The
26	# default `provider` value is the `id` of the SocialAccount provider. This `id` is
27	# used in the django-allauth URLs. The `provider` value can be overridden by setting
28	# the `SocialApp.provider_id`. This supports generic providers like OpenID Connect
29	# and SAML. When it is set on a non-generic provider, it changes the value of the
30	# SocialAccount's `provider`, but not the URLs. When django-allauth needs the
31	# SocialApp for a given SocialAccount, it uses an adapter to look it up at runtime.
32	#
33	# See:
34	# - The Firefox Account Provider docs
35	# https://docs.allauth.org/en/latest/socialaccount/providers/fxa.html
36	# - The OpenID Connect docs
37	# https://docs.allauth.org/en/latest/socialaccount/providers/openid_connect.html
38	# - The DefaultSocialAccountAdapter docs
39	# https://docs.allauth.org/en/latest/socialaccount/adapter.html#allauth.socialaccount.adapter.DefaultSocialAccountAdapter.get_provider
40
41	_fxa_provider_id = SocialApp.objects.filter(provider="fxa").values_list(	1✔
42	Coalesce(NullIf("provider_id", Value("")), "provider"), flat=True
43	)
44
45	data_specification = [	1✔
46	# Report on how many users do not have an email
47	DataModelSpec(
48	model=User,
49	subdivisions=[
50	DataBisectSpec("email", ~Q(email__exact="")),
51	DataBisectSpec(
52	"!email.fxa", Q(socialaccount__provider__in=_fxa_provider_id)
53	),
54	],
55	omit_key_prefixes=["!email.!fxa"],
56	report_name_overrides={"!email": "No Email", "!email.fxa": "Has FxA"},
57	ok_key="email",
58	needs_cleaning_key="!email.fxa",
59	)
60	]
61
62	def clean_users(self, queryset: QuerySet[User]) -> int:	1✔
63	fixed = 0	1✔
64	for user in queryset:	1✔
65	try:	1✔
66	fxa_account = SocialAccount.objects.get(	1✔
67	provider__in=self._fxa_provider_id, user=user
68	)
69	except SocialAccount.DoesNotExist:	1✔
70	continue	1✔
71	if fxa_email := fxa_account.extra_data.get("email"):	1✔
72	user.email = fxa_email	1✔
73	user.save(update_fields=["email"])	1✔
74	fixed += 1	1✔
75	return fixed	1✔
76
77
78	class IDNAEmailCleaner(CleanerTask):	1✔
79	slug = "idna-email"	1✔
80	title = "Fix non-ASCII domains in user emails"	1✔
81	check_description = (	1✔
82	"Users with non-ASCII characters in their email domain cannot receive emails"
83	"via AWS SES. Convert these domains to ASCII-compatible Punycode using IDNA."
84	)
85
86	def has_non_ascii_domain(self, email: str) -> bool:	1✔
87	if not email or "@" not in email:	1!
NEW 88	return False	×
89	try:	1✔
90	domain = email.split("@", 1)[1]	1✔
91	domain.encode("ascii")	1✔
92	return False	1✔
93	except UnicodeEncodeError:	1✔
94	return True	1✔
95
96	def punycode_email(self, email: str) -> str:	1✔
97	local, domain = email.split("@", 1)	1✔
98	domain_ascii = domain.encode("idna").decode("ascii")	1✔
99	return f"{local}@{domain_ascii}"	1✔
100
101	data_specification = [	1✔
102	DataModelSpec(
103	model=User,
104	subdivisions=[
105	DataBisectSpec(
106	"ascii_domain",
107	~Q(email__regex=r".@.[^\x00-\x7F].*"),
108	)
109	],
110	report_name_overrides={
111	"ascii_domain": "Has ASCII Domain",
112	"!ascii_domain": "Non-ASCII Domain",
113	},
114	ok_key="ascii_domain",
115	needs_cleaning_key="!ascii_domain",
116	)
117	]
118
119	def clean_users(self, queryset: QuerySet[User]) -> int:	1✔
120	fixed = 0	1✔
121	for user in queryset:	1✔
122	if user.email and self.has_non_ascii_domain(user.email):	1!
123	updated_email = self.punycode_email(user.email)	1✔
124	with transaction.atomic():	1✔
125	user.email = updated_email	1✔
126	user.save(update_fields=["email"])	1✔
127	EmailAddress.objects.filter(user=user).update(email=updated_email)	1✔
128	sa = SocialAccount.objects.filter(user=user).first()	1✔
129	if sa is not None:	1✔
130	sa.extra_data["email"] = updated_email	1✔
131	sa.save(update_fields=["extra_data"])	1✔
132	fixed += 1	1✔
133	return fixed	1✔

mozilla / fx-private-relay / 81bd7efa-7e2f-44be-91cc-76a322ca2564

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous