#4377

Committed 27 Nov 2024 02:27PM UTC coverage: 89.507% (+0.04%) from 89.463%

Build # #4377

Build Type

push

circleci

Committed by

web-flow

Commit Message

Add support for mirror URLs for GMP plugins. (#3251)

This patch adds support for alternative URLs from which the GMP plugin
can be downloaded. This is useful for GMP plugins, particularly
Widevine, because there are many different mirrors from Google. Some
users have trouble accessing the preferred mirror due to networking
issues, so it would be ideal to supply alternative domains that are more
likely to resolve/be reachable as a backup.

Run Details

2140 of 2528 branches covered (84.65%)

Branch coverage included in aggregate %.

14 of 14 new or added lines in 1 file covered. (100.0%)

4 existing lines in 1 file now uncovered.

5699 of 6230 relevant lines covered (91.48%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.55

/src/auslib/AUS.py

import functools
import logging
import re
from random import randint
from urllib.parse import urlparse

from auslib.blobs.base import ServeUpdate, createBlob
from auslib.global_state import cache, dbo
from auslib.services import releases
from auslib.util.versions import PinVersion


class ForceResult(object):
    """Enumerated "result" class that represents a non-random result chosen by a caller."""

    def __init__(self, name, query_value):
        self.name = name
        self.query_value = query_value


# Magic constants that callers can use to choose a specific "random" result.
FORCE_MAIN_MAPPING = ForceResult("succeed", "1")
FORCE_FALLBACK_MAPPING = ForceResult("fail", "-1")


def isSpecialURL(url, specialForceHosts):
    if not specialForceHosts:
        return False
    for s in specialForceHosts:
        if url.startswith(s):
            return True
    return False


def isForbiddenUrl(url, product, allowlistedDomains):
    if allowlistedDomains is None:
        allowlistedDomains = []
    parsedUrl = urlparse(url)
    domain = parsedUrl.netloc
    if domain not in allowlistedDomains:
        logging.warning("Forbidden domain: %s", domain)
        return True
    allowlistedDomain = allowlistedDomains[domain]
    if isinstance(allowlistedDomain, tuple):
        if product in allowlistedDomain:
            return False
        logging.warning("Forbidden domain for product %s: %s", product, domain)
    elif isinstance(allowlistedDomain, dict):
        path = parsedUrl.path
        for pathRegex in allowlistedDomain:
            if not re.fullmatch(pathRegex, path):
                continue
            if product in allowlistedDomain[pathRegex]:
                return False
            logging.warning("Forbidden domain/path for product %s: %s (%s)", product, domain, path)
            return True
        logging.warning("Forbidden domain/path: %s (%s)", domain, path)
    else:
        logging.warning("Forbidden domain, malformed entry: %s", domain)
    return True


def getFallbackChannel(channel):
    return channel.split("-cck-")[0]


class AUS:
    def __init__(self):
        self.specialForceHosts = None
        self.rand = functools.partial(randint, 0, 99)
        self.log = logging.getLogger(self.__class__.__name__)

    def updates_are_disabled(self, product, channel, transaction=None):
        cache_key = (product, channel)
        v = cache.get("updates_disabled", cache_key)
        if v is not None:
            return v

        where = dict(product=product, channel=channel)
        emergency_shutoffs = dbo.emergencyShutoffs.select(where=where, transaction=transaction)
        v = bool(emergency_shutoffs)
        cache.put("updates_disabled", cache_key, v)
        return v

    def evaluateRules(self, updateQuery, transaction=None):
        self.log.debug("Looking for rules that apply to:")
        self.log.debug(updateQuery)

        eval_metadata = dict(rule_id="unknown", rule_data_version="unknown")

        if self.updates_are_disabled(updateQuery["product"], updateQuery["channel"], transaction) or self.updates_are_disabled(
            updateQuery["product"], getFallbackChannel(updateQuery["channel"]), transaction
        ):
            log_message = "Updates are disabled for {}/{}.".format(updateQuery["product"], updateQuery["channel"])
            self.log.debug(log_message)
            return None, None, eval_metadata

        rules = dbo.rules.getRulesMatchingQuery(updateQuery, fallbackChannel=getFallbackChannel(updateQuery["channel"]), transaction=transaction)

        # TODO: throw any N->N update rules and keep the highest priority remaining one?
        if len(rules) < 1:
            return None, None, eval_metadata

        rules = sorted(rules, key=lambda rule: rule["priority"], reverse=True)
        rule = rules[0]

        eval_metadata["rule_id"] = rule["rule_id"]
        eval_metadata["rule_data_version"] = rule["data_version"]

        self.log.debug("Matching rule: %s" % rule)

        # There's a few cases where we have a matching rule but don't want
        # to serve an update:
        # 1) No mapping.
        if not rule["mapping"]:
            self.log.debug("Matching rule points at null mapping.")
            return None, None, eval_metadata
        mapping = rule["mapping"]

        # 2) For background checks (force=1 missing from query), we might not
        # serve every request an update
        # backgroundRate=100 means all requests are served
        # backgroundRate=25 means only one quarter of requests are served
        if not updateQuery["force"] == FORCE_MAIN_MAPPING and rule["backgroundRate"] < 100:
            self.log.debug("backgroundRate < 100, rolling the dice")
            if updateQuery["force"] == FORCE_FALLBACK_MAPPING or self.rand() >= rule["backgroundRate"]:
                fallbackReleaseName = rule["fallbackMapping"]
                if not fallbackReleaseName:
                    self.log.debug("No fallback releases. Request was dropped")
                    return None, None, eval_metadata

                self.log.debug("Using fallback release %s", fallbackReleaseName)
                mapping = fallbackReleaseName

        # 3) Incoming release is older than the one in the mapping, defined as one of:
        #    * version decreases
        #    * version is the same and buildID doesn't increase
        def get_blob(mapping):
            release = releases.get_release(mapping, transaction, include_sc=False)
            blob = None
            if release:
                blob = createBlob(release["blob"])
            # TODO: remove me when old releases table dies
            else:
                release = dbo.releases.getReleases(name=mapping, limit=1, transaction=transaction)[0]
                blob = release["data"]
            return blob

        blob = get_blob(mapping)
        if not blob:
            return None, None, eval_metadata
        candidate = blob.shouldServeUpdate(updateQuery)
        if not candidate:
            return None, None, eval_metadata

        if candidate == ServeUpdate.Maybe:
            version_pin = PinVersion(updateQuery.get("pin"))
            pin_mapping = dbo.pinnable_releases.getPinMapping(updateQuery["product"], getFallbackChannel(updateQuery["channel"]), str(version_pin))
            # Note that we fall back to serving the original update if the pin is not found
            # in the pin table, even if the version that we will serve is past the pinned
            # version. This is because, if there is something wrong with the update pin,
            # we would rather default to keeping the user up-to-date.
            # The alternative would mean that setting invalid pin would be a subtle problem
            # that would potentially be difficult to recognize. For example:
            #  - A sysadmin installs Firefox ESR 120.0 and sets the pin to '120.15.'.
            #  - Every time that a new minor version of 120 is available, the sysadmin
            #    updates to it promptly.
            #  - Eventually, the sysadmin updates to 120.14, but 120.15 is never released.
            #    The next version is ESR 134.0.
            #  - The relevant update rule now points to ESR 134, but it isn't returned as
            #    an available update so, as far as the sysadmin knows, the pin is working
            #    as expected.
            #  - A loaner laptop is given out that still has ESR 120.0 installed.
            #  - When the loaner laptop asks for updates, Balrog sees that the newest
            #    version is ESR 134.0, but that would be newer than the '120.15' pin,
            #    so no update is returned.
            #  - The loaner laptop never updates and stays at ESR 120.0.
            # This situation hides the problem from the sysadmin and leaves some
            # installations vulnerable.
            if pin_mapping is not None:
                mapping = pin_mapping
                blob = get_blob(mapping)
                if not blob or not blob.shouldServeUpdate(updateQuery):
                    return None, None, eval_metadata

        self.log.debug("Returning release %s", mapping)
        return blob, rule["update_type"], eval_metadata

1	import functools	1✔
2	import logging	1✔
3	import re	1✔
4	from random import randint	1✔
5	from urllib.parse import urlparse	1✔
6
7	from auslib.blobs.base import ServeUpdate, createBlob	1✔
8	from auslib.global_state import cache, dbo	1✔
9	from auslib.services import releases	1✔
10	from auslib.util.versions import PinVersion	1✔
11
12
13	class ForceResult(object):	1✔
14	"""Enumerated "result" class that represents a non-random result chosen by a caller."""
15
16	def __init__(self, name, query_value):	1✔
17	self.name = name	1✔
18	self.query_value = query_value	1✔
19
20
21	# Magic constants that callers can use to choose a specific "random" result.
22	FORCE_MAIN_MAPPING = ForceResult("succeed", "1")	1✔
23	FORCE_FALLBACK_MAPPING = ForceResult("fail", "-1")	1✔
24
25
26	def isSpecialURL(url, specialForceHosts):	1✔
27	if not specialForceHosts:	1!
UNCOV 28	return False	×
29	for s in specialForceHosts:	1✔
30	if url.startswith(s):	1✔
31	return True	1✔
32	return False	1✔
33
34
35	def isForbiddenUrl(url, product, allowlistedDomains):	1✔
36	if allowlistedDomains is None:	1!
UNCOV 37	allowlistedDomains = []	×
38	parsedUrl = urlparse(url)	1✔
39	domain = parsedUrl.netloc	1✔
40	if domain not in allowlistedDomains:	1✔
41	logging.warning("Forbidden domain: %s", domain)	1✔
42	return True	1✔
43	allowlistedDomain = allowlistedDomains[domain]	1✔
44	if isinstance(allowlistedDomain, tuple):	1✔
45	if product in allowlistedDomain:	1✔
46	return False	1✔
47	logging.warning("Forbidden domain for product %s: %s", product, domain)	1✔
48	elif isinstance(allowlistedDomain, dict):	1!
49	path = parsedUrl.path	1✔
50	for pathRegex in allowlistedDomain:	1✔
51	if not re.fullmatch(pathRegex, path):	1✔
52	continue	1✔
53	if product in allowlistedDomain[pathRegex]:	1✔
54	return False	1✔
55	logging.warning("Forbidden domain/path for product %s: %s (%s)", product, domain, path)	1✔
56	return True	1✔
57	logging.warning("Forbidden domain/path: %s (%s)", domain, path)	1✔
58	else:
UNCOV 59	logging.warning("Forbidden domain, malformed entry: %s", domain)	×
60	return True	1✔
61
62
63	def getFallbackChannel(channel):	1✔
64	return channel.split("-cck-")[0]	1✔
65
66
67	class AUS:	1✔
68	def __init__(self):	1✔
69	self.specialForceHosts = None	1✔
70	self.rand = functools.partial(randint, 0, 99)	1✔
71	self.log = logging.getLogger(self.__class__.__name__)	1✔
72
73	def updates_are_disabled(self, product, channel, transaction=None):	1✔
74	cache_key = (product, channel)	1✔
75	v = cache.get("updates_disabled", cache_key)	1✔
76	if v is not None:	1✔
77	return v	1✔
78
79	where = dict(product=product, channel=channel)	1✔
80	emergency_shutoffs = dbo.emergencyShutoffs.select(where=where, transaction=transaction)	1✔
81	v = bool(emergency_shutoffs)	1✔
82	cache.put("updates_disabled", cache_key, v)	1✔
83	return v	1✔
84
85	def evaluateRules(self, updateQuery, transaction=None):	1✔
86	self.log.debug("Looking for rules that apply to:")	1✔
87	self.log.debug(updateQuery)	1✔
88
89	eval_metadata = dict(rule_id="unknown", rule_data_version="unknown")	1✔
90
91	if self.updates_are_disabled(updateQuery["product"], updateQuery["channel"], transaction) or self.updates_are_disabled(	1✔
92	updateQuery["product"], getFallbackChannel(updateQuery["channel"]), transaction
93	):
94	log_message = "Updates are disabled for {}/{}.".format(updateQuery["product"], updateQuery["channel"])	1✔
95	self.log.debug(log_message)	1✔
96	return None, None, eval_metadata	1✔
97
98	rules = dbo.rules.getRulesMatchingQuery(updateQuery, fallbackChannel=getFallbackChannel(updateQuery["channel"]), transaction=transaction)	1✔
99
100	# TODO: throw any N->N update rules and keep the highest priority remaining one?
101	if len(rules) < 1:	1✔
102	return None, None, eval_metadata	1✔
103
104	rules = sorted(rules, key=lambda rule: rule["priority"], reverse=True)	1✔
105	rule = rules[0]	1✔
106
107	eval_metadata["rule_id"] = rule["rule_id"]	1✔
108	eval_metadata["rule_data_version"] = rule["data_version"]	1✔
109
110	self.log.debug("Matching rule: %s" % rule)	1✔
111
112	# There's a few cases where we have a matching rule but don't want
113	# to serve an update:
114	# 1) No mapping.
115	if not rule["mapping"]:	1✔
116	self.log.debug("Matching rule points at null mapping.")	1✔
117	return None, None, eval_metadata	1✔
118	mapping = rule["mapping"]	1✔
119
120	# 2) For background checks (force=1 missing from query), we might not
121	# serve every request an update
122	# backgroundRate=100 means all requests are served
123	# backgroundRate=25 means only one quarter of requests are served
124	if not updateQuery["force"] == FORCE_MAIN_MAPPING and rule["backgroundRate"] < 100:	1✔
125	self.log.debug("backgroundRate < 100, rolling the dice")	1✔
126	if updateQuery["force"] == FORCE_FALLBACK_MAPPING or self.rand() >= rule["backgroundRate"]:	1✔
127	fallbackReleaseName = rule["fallbackMapping"]	1✔
128	if not fallbackReleaseName:	1✔
129	self.log.debug("No fallback releases. Request was dropped")	1✔
130	return None, None, eval_metadata	1✔
131
132	self.log.debug("Using fallback release %s", fallbackReleaseName)	1✔
133	mapping = fallbackReleaseName	1✔
134
135	# 3) Incoming release is older than the one in the mapping, defined as one of:
136	# * version decreases
137	# * version is the same and buildID doesn't increase
138	def get_blob(mapping):	1✔
139	release = releases.get_release(mapping, transaction, include_sc=False)	1✔
140	blob = None	1✔
141	if release:	1✔
142	blob = createBlob(release["blob"])	1✔
143	# TODO: remove me when old releases table dies
144	else:
145	release = dbo.releases.getReleases(name=mapping, limit=1, transaction=transaction)[0]	1✔
146	blob = release["data"]	1✔
147	return blob	1✔
148
149	blob = get_blob(mapping)	1✔
150	if not blob:	1!
UNCOV 151	return None, None, eval_metadata	×
152	candidate = blob.shouldServeUpdate(updateQuery)	1✔
153	if not candidate:	1✔
154	return None, None, eval_metadata	1✔
155
156	if candidate == ServeUpdate.Maybe:	1✔
157	version_pin = PinVersion(updateQuery.get("pin"))	1✔
158	pin_mapping = dbo.pinnable_releases.getPinMapping(updateQuery["product"], getFallbackChannel(updateQuery["channel"]), str(version_pin))	1✔
159	# Note that we fall back to serving the original update if the pin is not found
160	# in the pin table, even if the version that we will serve is past the pinned
161	# version. This is because, if there is something wrong with the update pin,
162	# we would rather default to keeping the user up-to-date.
163	# The alternative would mean that setting invalid pin would be a subtle problem
164	# that would potentially be difficult to recognize. For example:
165	# - A sysadmin installs Firefox ESR 120.0 and sets the pin to '120.15.'.
166	# - Every time that a new minor version of 120 is available, the sysadmin
167	# updates to it promptly.
168	# - Eventually, the sysadmin updates to 120.14, but 120.15 is never released.
169	# The next version is ESR 134.0.
170	# - The relevant update rule now points to ESR 134, but it isn't returned as
171	# an available update so, as far as the sysadmin knows, the pin is working
172	# as expected.
173	# - A loaner laptop is given out that still has ESR 120.0 installed.
174	# - When the loaner laptop asks for updates, Balrog sees that the newest
175	# version is ESR 134.0, but that would be newer than the '120.15' pin,
176	# so no update is returned.
177	# - The loaner laptop never updates and stays at ESR 120.0.
178	# This situation hides the problem from the sysadmin and leaves some
179	# installations vulnerable.
180	if pin_mapping is not None:	1!
181	mapping = pin_mapping	1✔
182	blob = get_blob(mapping)	1✔
183	if not blob or not blob.shouldServeUpdate(updateQuery):	1✔
184	return None, None, eval_metadata	1✔
185
186	self.log.debug("Returning release %s", mapping)	1✔
187	return blob, rule["update_type"], eval_metadata	1✔

mozilla-releng / balrog / #4377

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous