3ce233ef-696e-43c5-8086-1f352694b276

Committed 28 May 2025 02:40PM UTC coverage: 85.366% (+0.01%) from 85.353%

Build # 3ce233ef-696e-43c5-8086-1f352694b276

Build Type

Pull #5572

circleci

Committed by

groovecoder

Commit Message

for MPP-3439: add IDNAEmailCleaner to clean email address domains with non-ASCII chars

Pull Request Pull Request #5572: for MPP-3439: add IDNAEmailCleaner to clean email address domains with non-ASCII chars

Run Details

2473 of 3619 branches covered (68.33%)

Branch coverage included in aggregate %.

73 of 74 new or added lines in 4 files covered. (98.65%)

9 existing lines in 1 file now uncovered.

17576 of 19867 relevant lines covered (88.47%)

9.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.38

/privaterelay/cleaners.py

"""Tasks that detect and fix data issues in privaterelay app or 3rd party apps."""

from django.contrib.auth.models import User
from django.db import transaction
from django.db.models import Q, QuerySet, Value
from django.db.models.functions import Coalesce, NullIf

from allauth.socialaccount.models import EmailAddress, SocialAccount, SocialApp

from .cleaner_task import CleanerTask, DataBisectSpec, DataModelSpec


class MissingEmailCleaner(CleanerTask):
    slug = "missing-email"
    title = "Ensure all users have an email"
    check_description = (
        "When User.email is empty, we are unable to forward email to the Relay user."
        " We can get the email from the FxA profile if available."
    )

    # The Firefox Accounts default provider identifier is `fxa`. Firefox Accounts was
    # the name for Mozilla Accounts before 2023. This query returns the value, as a
    # one-element list, of the `SocialApp.provider_id` if set, and `fxa` if not.
    #
    # The `provider` field for a SocialAccount is a CharField, not a ForeignKey.  The
    # default `provider` value is the `id` of the SocialAccount provider. This `id` is
    # used in the django-allauth URLs. The `provider` value can be overridden by setting
    # the `SocialApp.provider_id`. This supports generic providers like OpenID Connect
    # and SAML. When it is set on a non-generic provider, it changes the value of the
    # SocialAccount's `provider`, but not the URLs. When django-allauth needs the
    # SocialApp for a given SocialAccount, it uses an adapter to look it up at runtime.
    #
    # See:
    # - The Firefox Account Provider docs
    # https://docs.allauth.org/en/latest/socialaccount/providers/fxa.html
    # - The OpenID Connect docs
    # https://docs.allauth.org/en/latest/socialaccount/providers/openid_connect.html
    # - The DefaultSocialAccountAdapter docs
    # https://docs.allauth.org/en/latest/socialaccount/adapter.html#allauth.socialaccount.adapter.DefaultSocialAccountAdapter.get_provider

    _fxa_provider_id = SocialApp.objects.filter(provider="fxa").values_list(
        Coalesce(NullIf("provider_id", Value("")), "provider"), flat=True
    )

    data_specification = [
        # Report on how many users do not have an email
        DataModelSpec(
            model=User,
            subdivisions=[
                DataBisectSpec("email", ~Q(email__exact="")),
                DataBisectSpec(
                    "!email.fxa", Q(socialaccount__provider__in=_fxa_provider_id)
                ),
            ],
            omit_key_prefixes=["!email.!fxa"],
            report_name_overrides={"!email": "No Email", "!email.fxa": "Has FxA"},
            ok_key="email",
            needs_cleaning_key="!email.fxa",
        )
    ]

    def clean_users(self, queryset: QuerySet[User]) -> int:
        fixed = 0
        for user in queryset:
            try:
                fxa_account = SocialAccount.objects.get(
                    provider__in=self._fxa_provider_id, user=user
                )
            except SocialAccount.DoesNotExist:
                continue
            if fxa_email := fxa_account.extra_data.get("email"):
                user.email = fxa_email
                user.save(update_fields=["email"])
                fixed += 1
        return fixed


class IDNAEmailCleaner(CleanerTask):
    slug = "idna-email"
    title = "Fix non-ASCII domains in user emails"
    check_description = (
        "Users with non-ASCII characters in their email domain cannot receive emails"
        "via AWS SES. Convert these domains to ASCII-compatible Punycode using IDNA."
    )

    def has_non_ascii_domain(self, email: str) -> bool:
        try:
            domain = email.split("@", 1)[1]
            domain.encode("ascii")
            return False
        except (IndexError, UnicodeEncodeError):
            return True

    def punycode_email(self, email: str) -> str:
        local, domain = email.split("@", 1)
        domain_ascii = domain.encode("idna").decode("ascii")
        return f"{local}@{domain_ascii}"

    data_specification = [
        DataModelSpec(
            model=User,
            subdivisions=[
                DataBisectSpec(
                    "ascii_domain",
                    ~Q(email__regex=r".*@.*[^\x00-\x7F].*"),
                )
            ],
            report_name_overrides={
                "ascii_domain": "Has ASCII Domain",
                "!ascii_domain": "Non-ASCII Domain",
            },
            ok_key="ascii_domain",
            needs_cleaning_key="!ascii_domain",
        )
    ]

    def clean_users(self, queryset: QuerySet[User]) -> int:
        fixed = 0
        for user in queryset:
            if user.email and self.has_non_ascii_domain(user.email):
                updated_email = self.punycode_email(user.email)
                with transaction.atomic():
                    user.email = updated_email
                    user.save(update_fields=["email"])
                    EmailAddress.objects.filter(user=user).update(email=updated_email)
                    sa = SocialAccount.objects.filter(user=user).first()
                    if sa is not None:
                        sa.extra_data["email"] = updated_email
                        sa.save(update_fields=["extra_data"])
                fixed += 1
        return fixed

1	"""Tasks that detect and fix data issues in privaterelay app or 3rd party apps."""
2
3	from django.contrib.auth.models import User	1✔
4	from django.db import transaction	1✔
5	from django.db.models import Q, QuerySet, Value	1✔
6	from django.db.models.functions import Coalesce, NullIf	1✔
7
8	from allauth.socialaccount.models import EmailAddress, SocialAccount, SocialApp	1✔
9
10	from .cleaner_task import CleanerTask, DataBisectSpec, DataModelSpec	1✔
11
12
13	class MissingEmailCleaner(CleanerTask):	1✔
14	slug = "missing-email"	1✔
15	title = "Ensure all users have an email"	1✔
16	check_description = (	1✔
17	"When User.email is empty, we are unable to forward email to the Relay user."
18	" We can get the email from the FxA profile if available."
19	)
20
21	# The Firefox Accounts default provider identifier is `fxa`. Firefox Accounts was
22	# the name for Mozilla Accounts before 2023. This query returns the value, as a
23	# one-element list, of the `SocialApp.provider_id` if set, and `fxa` if not.
24	#
25	# The `provider` field for a SocialAccount is a CharField, not a ForeignKey. The
26	# default `provider` value is the `id` of the SocialAccount provider. This `id` is
27	# used in the django-allauth URLs. The `provider` value can be overridden by setting
28	# the `SocialApp.provider_id`. This supports generic providers like OpenID Connect
29	# and SAML. When it is set on a non-generic provider, it changes the value of the
30	# SocialAccount's `provider`, but not the URLs. When django-allauth needs the
31	# SocialApp for a given SocialAccount, it uses an adapter to look it up at runtime.
32	#
33	# See:
34	# - The Firefox Account Provider docs
35	# https://docs.allauth.org/en/latest/socialaccount/providers/fxa.html
36	# - The OpenID Connect docs
37	# https://docs.allauth.org/en/latest/socialaccount/providers/openid_connect.html
38	# - The DefaultSocialAccountAdapter docs
39	# https://docs.allauth.org/en/latest/socialaccount/adapter.html#allauth.socialaccount.adapter.DefaultSocialAccountAdapter.get_provider
40
41	_fxa_provider_id = SocialApp.objects.filter(provider="fxa").values_list(	1✔
42	Coalesce(NullIf("provider_id", Value("")), "provider"), flat=True
43	)
44
45	data_specification = [	1✔
46	# Report on how many users do not have an email
47	DataModelSpec(
48	model=User,
49	subdivisions=[
50	DataBisectSpec("email", ~Q(email__exact="")),
51	DataBisectSpec(
52	"!email.fxa", Q(socialaccount__provider__in=_fxa_provider_id)
53	),
54	],
55	omit_key_prefixes=["!email.!fxa"],
56	report_name_overrides={"!email": "No Email", "!email.fxa": "Has FxA"},
57	ok_key="email",
58	needs_cleaning_key="!email.fxa",
59	)
60	]
61
62	def clean_users(self, queryset: QuerySet[User]) -> int:	1✔
63	fixed = 0	1✔
64	for user in queryset:	1✔
65	try:	1✔
66	fxa_account = SocialAccount.objects.get(	1✔
67	provider__in=self._fxa_provider_id, user=user
68	)
69	except SocialAccount.DoesNotExist:	1✔
70	continue	1✔
71	if fxa_email := fxa_account.extra_data.get("email"):	1✔
72	user.email = fxa_email	1✔
73	user.save(update_fields=["email"])	1✔
74	fixed += 1	1✔
75	return fixed	1✔
76
77
78	class IDNAEmailCleaner(CleanerTask):	1✔
79	slug = "idna-email"	1✔
80	title = "Fix non-ASCII domains in user emails"	1✔
81	check_description = (	1✔
82	"Users with non-ASCII characters in their email domain cannot receive emails"
83	"via AWS SES. Convert these domains to ASCII-compatible Punycode using IDNA."
84	)
85
86	def has_non_ascii_domain(self, email: str) -> bool:	1✔
87	try:	1✔
88	domain = email.split("@", 1)[1]	1✔
89	domain.encode("ascii")	1✔
NEW 90	return False	×
91	except (IndexError, UnicodeEncodeError):	1✔
92	return True	1✔
93
94	def punycode_email(self, email: str) -> str:	1✔
95	local, domain = email.split("@", 1)	1✔
96	domain_ascii = domain.encode("idna").decode("ascii")	1✔
97	return f"{local}@{domain_ascii}"	1✔
98
99	data_specification = [	1✔
100	DataModelSpec(
101	model=User,
102	subdivisions=[
103	DataBisectSpec(
104	"ascii_domain",
105	~Q(email__regex=r".@.[^\x00-\x7F].*"),
106	)
107	],
108	report_name_overrides={
109	"ascii_domain": "Has ASCII Domain",
110	"!ascii_domain": "Non-ASCII Domain",
111	},
112	ok_key="ascii_domain",
113	needs_cleaning_key="!ascii_domain",
114	)
115	]
116
117	def clean_users(self, queryset: QuerySet[User]) -> int:	1✔
118	fixed = 0	1✔
119	for user in queryset:	1✔
120	if user.email and self.has_non_ascii_domain(user.email):	1!
121	updated_email = self.punycode_email(user.email)	1✔
122	with transaction.atomic():	1✔
123	user.email = updated_email	1✔
124	user.save(update_fields=["email"])	1✔
125	EmailAddress.objects.filter(user=user).update(email=updated_email)	1✔
126	sa = SocialAccount.objects.filter(user=user).first()	1✔
127	if sa is not None:	1!
128	sa.extra_data["email"] = updated_email	1✔
129	sa.save(update_fields=["extra_data"])	1✔
130	fixed += 1	1✔
131	return fixed	1✔

mozilla / fx-private-relay / 3ce233ef-696e-43c5-8086-1f352694b276

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous