#4795

Committed 23 Jul 2025 02:48PM UTC coverage: 61.584%. First build

Build # #4795

Build Type

Pull #3426

circleci

Committed by

jcristau

Commit Message

admin: port statsd timers to a middleware

Pull Request Pull Request #3426: update to connexion 3.x

Run Details

2034 of 3464 branches covered (58.72%)

Branch coverage included in aggregate %.

33 of 49 new or added lines in 3 files covered. (67.35%)

3907 of 6183 relevant lines covered (63.19%)

0.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.02

/src/auslib/web/admin/base.py

import logging
import re
from os import path

import connexion
from connexion.middleware import MiddlewarePosition
from connexion.options import SwaggerUIOptions
from flask import g, request
from sentry_sdk import capture_exception
from specsynthase.specbuilder import SpecBuilder
from starlette.middleware.cors import CORSMiddleware
from statsd.defaults.env import statsd

import auslib
import auslib.web.admin.views.validators  # noqa
from auslib.db import ChangeScheduledError, OutdatedDataError, UpdateMergeError
from auslib.dockerflow import create_dockerflow_endpoints
from auslib.errors import BlobValidationError, PermissionDeniedError, ReadOnlyError, SignoffRequiredError
from auslib.util.auth import AuthError, verified_userinfo
from auslib.web.admin.views.problem import problem
from auslib.web.common import middlewares

log = logging.getLogger(__name__)

current_dir = path.dirname(__file__)
web_dir = path.dirname(auslib.web.__file__)

spec = (
    SpecBuilder()
    .add_spec(path.join(current_dir, "swagger/api.yml"))
    .add_spec(path.join(web_dir, "common/swagger/definitions.yml"))
    .add_spec(path.join(web_dir, "common/swagger/parameters.yml"))
    .add_spec(path.join(web_dir, "common/swagger/responses.yml"))
)

swagger_ui_options = SwaggerUIOptions(swagger_ui=False)


def create_app(allow_origins=None):
    connexion_app = connexion.App(__name__, swagger_ui_options=swagger_ui_options, middlewares=middlewares[:])
    connexion_app.app.debug = False
    connexion_app.add_api(spec, strict_validation=True)
    connexion_app.add_api(path.join(current_dir, "swagger", "api_v2.yml"), base_path="/v2", strict_validation=True, validate_responses=True)
    flask_app = connexion_app.app

    create_dockerflow_endpoints(flask_app)

    @flask_app.before_request
    def setup_request():
        if request.full_path.startswith("/v2"):
            from auslib.global_state import dbo

            request.transaction = dbo.begin()

            if request.method in ("POST", "PUT", "DELETE"):
                username = verified_userinfo(request, flask_app.config["AUTH_DOMAIN"], flask_app.config["AUTH_AUDIENCE"])["email"]
                if not username:
                    log.warning("Login Required")
                    return problem(401, "Unauthenticated", "Login Required")
                # Machine to machine accounts are identified by uninformative clientIds
                # In order to keep Balrog permissions more readable, we map them to
                # more useful usernames, which are stored in the app config.
                if "@" not in username:
                    username = flask_app.config["M2M_ACCOUNT_MAPPING"].get(username, username)
                # Even if the user has provided a valid access token, we don't want to assume
                # that person should be able to access Balrog (in case auth0 is not configured
                # to be restrictive enough.
                elif not dbo.isKnownUser(username):
                    log.warning("Authorization Required")
                    return problem(403, "Forbidden", "Authorization Required")

                request.username = username

    @flask_app.after_request
    def complete_request(response):
        if hasattr(request, "transaction"):
            try:
                if response.status_code >= 400:
                    request.transaction.rollback()
                else:
                    request.transaction.commit()
            finally:
                request.transaction.close()

        return response

    @flask_app.errorhandler(OutdatedDataError)
    def outdated_data_error(error):
        msg = "Couldn't perform the request %s. Outdated Data Version. old_data_version doesn't match current data_version" % request.method
        log.warning("Bad input: %s", msg)
        log.warning(error)
        return problem(400, "Bad Request", "OutdatedDataError", ext={"exception": msg})

    @flask_app.errorhandler(UpdateMergeError)
    def update_merge_error(error):
        msg = "Couldn't perform the request %s due to merge error. Is there a scheduled change that conflicts with yours?" % request.method
        log.warning("Bad input: %s", msg)
        log.warning(error)
        return problem(400, "Bad Request", "UpdateMergeError", ext={"exception": msg})

    @flask_app.errorhandler(ChangeScheduledError)
    def change_scheduled_error(error):
        msg = "Couldn't perform the request %s due a conflict with a scheduled change. " % request.method
        msg += str(error)
        log.warning("Bad input: %s", msg)
        log.warning(error)
        return problem(400, "Bad Request", "ChangeScheduledError", ext={"exception": msg})

    @flask_app.errorhandler(AuthError)
    def auth_error(error):
        msg = "Permission denied to perform the request. {}".format(error.error)
        log.warning(msg)
        return problem(error.status_code, "Forbidden", "PermissionDeniedError", ext={"exception": msg})

    @flask_app.errorhandler(BlobValidationError)
    def blob_validation_error(error):
        return problem(400, "Bad Request", "Invalid Blob", ext={"exception": error.errors})

    @flask_app.errorhandler(SignoffRequiredError)
    def signoff_required_error(error):
        return problem(400, "Bad Request", "Signoff Required", ext={"exception": f"{error}"})

    @flask_app.errorhandler(ReadOnlyError)
    def read_only_error(error):
        return problem(400, "Bad Request", "Read only", ext={"exception": f"{error}"})

    @flask_app.errorhandler(PermissionDeniedError)
    def permission_denied_error(error):
        return problem(403, "Forbidden", "Permission Denied", ext={"exception": f"{error}"})

    @flask_app.errorhandler(ValueError)
    def value_error(error):
        return problem(400, "Bad Request", "Unknown error", ext={"exception": f"{error}"})

    # Connexion's error handling sometimes breaks when parameters contain
    # unicode characters (https://github.com/zalando/connexion/issues/604).
    # To work around, we catch them and return a 400 (which is what Connexion
    # would do if it didn't hit this error).
    @flask_app.errorhandler(UnicodeEncodeError)
    def unicode(error):
        return problem(400, "Unicode Error", "Connexion was unable to parse some unicode data correctly.")

    @flask_app.errorhandler(Exception)
    def ise(error):
        capture_exception(error)
        log.exception("Caught ISE 500 error: %r", error)
        log.debug("Request path is: %s", request.path)
        log.debug("Request environment is: %s", request.environ)
        log.debug("Request headers are: %s", request.headers)
        return problem(500, "Internal Server Error", "Internal Server Error")

    @flask_app.after_request
    def add_security_headers(response):
        response.headers["X-Frame-Options"] = "DENY"
        response.headers["X-Content-Type-Options"] = "nosniff"
        response.headers["Strict-Transport-Security"] = flask_app.config.get("STRICT_TRANSPORT_SECURITY", "max-age=31536000;")
        if re.match("^/ui/", request.path):
            # This enables swagger-ui to dynamically fetch and
            # load the swagger specification JSON file containing API definition and examples.
            response.headers["X-Frame-Options"] = "SAMEORIGIN"
        else:
            response.headers["Content-Security-Policy"] = flask_app.config.get("CONTENT_SECURITY_POLICY", "default-src 'none'; frame-ancestors 'none'")
        return response

    class StatsdMiddleware:
        def __init__(self, app):
            self.app = app

        def metric_name(self, scope):
            if scope["method"] == "OPTIONS":
                return
            breakpoint()
            op = scope.get("extensions", {}).get("connexion_routing", {}).get("operation_id")
            if op is None:
                return
            # do some massaging to get the metric name right
            # * remove various module prefixes
            # * add a common prefix to ensure that we can mark these metrics as gauges for
            #   statsd
            metric = op.replace(".", "_").removeprefix("auslib_web_admin_views_").removeprefix("auslib_web_admin_").removeprefix("auslib_web_common_")
            return f"endpoint_{metric}"

        async def __call__(self, scope, receive, send):
            if scope["type"] != "http":
                await self.app(scope, receive, send)
                return

            metric = self.metric_name(scope)
            if not metric:
                await self.app(scope, receive, send)
                return

            timer = statsd.timer(metric)
            timer.start()
            try:
                await self.app(scope, receive, send)
            finally:
                timer.stop()

    connexion_app.add_middleware(StatsdMiddleware, MiddlewarePosition.BEFORE_VALIDATION)

    if allow_origins:
        connexion_app.add_middleware(
            CORSMiddleware,
            MiddlewarePosition.BEFORE_ROUTING,
            allow_origins=allow_origins,
            allow_headers=["Authorization", "Content-Type"],
            allow_methods=["OPTIONS", "GET", "POST", "PUT", "DELETE"],
        )

    return connexion_app

1	import logging	1✔
2	import re	1✔
3	from os import path	1✔
4
5	import connexion	1✔
6	from connexion.middleware import MiddlewarePosition	1✔
7	from connexion.options import SwaggerUIOptions	1✔
8	from flask import g, request	1✔
9	from sentry_sdk import capture_exception	1✔
10	from specsynthase.specbuilder import SpecBuilder	1✔
11	from starlette.middleware.cors import CORSMiddleware	1✔
12	from statsd.defaults.env import statsd	1✔
13
14	import auslib	1✔
15	import auslib.web.admin.views.validators # noqa	1✔
16	from auslib.db import ChangeScheduledError, OutdatedDataError, UpdateMergeError	1✔
17	from auslib.dockerflow import create_dockerflow_endpoints	1✔
18	from auslib.errors import BlobValidationError, PermissionDeniedError, ReadOnlyError, SignoffRequiredError	1✔
19	from auslib.util.auth import AuthError, verified_userinfo	1✔
20	from auslib.web.admin.views.problem import problem	1✔
21	from auslib.web.common import middlewares	1✔
22
23	log = logging.getLogger(__name__)	1✔
24
25	current_dir = path.dirname(__file__)	1✔
26	web_dir = path.dirname(auslib.web.__file__)	1✔
27
28	spec = (	1✔
29	SpecBuilder()
30	.add_spec(path.join(current_dir, "swagger/api.yml"))
31	.add_spec(path.join(web_dir, "common/swagger/definitions.yml"))
32	.add_spec(path.join(web_dir, "common/swagger/parameters.yml"))
33	.add_spec(path.join(web_dir, "common/swagger/responses.yml"))
34	)
35
36	swagger_ui_options = SwaggerUIOptions(swagger_ui=False)	1✔
37
38
39	def create_app(allow_origins=None):	1✔
40	connexion_app = connexion.App(__name__, swagger_ui_options=swagger_ui_options, middlewares=middlewares[:])	1✔
41	connexion_app.app.debug = False	1✔
42	connexion_app.add_api(spec, strict_validation=True)	1✔
43	connexion_app.add_api(path.join(current_dir, "swagger", "api_v2.yml"), base_path="/v2", strict_validation=True, validate_responses=True)	1✔
44	flask_app = connexion_app.app	1✔
45
46	create_dockerflow_endpoints(flask_app)	1✔
47
48	@flask_app.before_request	1✔
49	def setup_request():	1✔
50	if request.full_path.startswith("/v2"):	×
51	from auslib.global_state import dbo	×
52
53	request.transaction = dbo.begin()	×
54
55	if request.method in ("POST", "PUT", "DELETE"):	×
56	username = verified_userinfo(request, flask_app.config["AUTH_DOMAIN"], flask_app.config["AUTH_AUDIENCE"])["email"]	×
57	if not username:	×
58	log.warning("Login Required")	×
59	return problem(401, "Unauthenticated", "Login Required")	×
60	# Machine to machine accounts are identified by uninformative clientIds
61	# In order to keep Balrog permissions more readable, we map them to
62	# more useful usernames, which are stored in the app config.
63	if "@" not in username:	×
64	username = flask_app.config["M2M_ACCOUNT_MAPPING"].get(username, username)	×
65	# Even if the user has provided a valid access token, we don't want to assume
66	# that person should be able to access Balrog (in case auth0 is not configured
67	# to be restrictive enough.
68	elif not dbo.isKnownUser(username):	×
69	log.warning("Authorization Required")	×
70	return problem(403, "Forbidden", "Authorization Required")	×
71
72	request.username = username	×
73
74	@flask_app.after_request	1✔
75	def complete_request(response):	1✔
76	if hasattr(request, "transaction"):	×
77	try:	×
78	if response.status_code >= 400:	×
79	request.transaction.rollback()	×
80	else:
81	request.transaction.commit()	×
82	finally:
83	request.transaction.close()	×
84
85	return response	×
86
87	@flask_app.errorhandler(OutdatedDataError)	1✔
88	def outdated_data_error(error):	1✔
89	msg = "Couldn't perform the request %s. Outdated Data Version. old_data_version doesn't match current data_version" % request.method	×
90	log.warning("Bad input: %s", msg)	×
91	log.warning(error)	×
92	return problem(400, "Bad Request", "OutdatedDataError", ext={"exception": msg})	×
93
94	@flask_app.errorhandler(UpdateMergeError)	1✔
95	def update_merge_error(error):	1✔
96	msg = "Couldn't perform the request %s due to merge error. Is there a scheduled change that conflicts with yours?" % request.method	×
97	log.warning("Bad input: %s", msg)	×
98	log.warning(error)	×
99	return problem(400, "Bad Request", "UpdateMergeError", ext={"exception": msg})	×
100
101	@flask_app.errorhandler(ChangeScheduledError)	1✔
102	def change_scheduled_error(error):	1✔
103	msg = "Couldn't perform the request %s due a conflict with a scheduled change. " % request.method	×
104	msg += str(error)	×
105	log.warning("Bad input: %s", msg)	×
106	log.warning(error)	×
107	return problem(400, "Bad Request", "ChangeScheduledError", ext={"exception": msg})	×
108
109	@flask_app.errorhandler(AuthError)	1✔
110	def auth_error(error):	1✔
111	msg = "Permission denied to perform the request. {}".format(error.error)	×
112	log.warning(msg)	×
113	return problem(error.status_code, "Forbidden", "PermissionDeniedError", ext={"exception": msg})	×
114
115	@flask_app.errorhandler(BlobValidationError)	1✔
116	def blob_validation_error(error):	1✔
117	return problem(400, "Bad Request", "Invalid Blob", ext={"exception": error.errors})	×
118
119	@flask_app.errorhandler(SignoffRequiredError)	1✔
120	def signoff_required_error(error):	1✔
121	return problem(400, "Bad Request", "Signoff Required", ext={"exception": f"{error}"})	×
122
123	@flask_app.errorhandler(ReadOnlyError)	1✔
124	def read_only_error(error):	1✔
125	return problem(400, "Bad Request", "Read only", ext={"exception": f"{error}"})	×
126
127	@flask_app.errorhandler(PermissionDeniedError)	1✔
128	def permission_denied_error(error):	1✔
129	return problem(403, "Forbidden", "Permission Denied", ext={"exception": f"{error}"})	×
130
131	@flask_app.errorhandler(ValueError)	1✔
132	def value_error(error):	1✔
133	return problem(400, "Bad Request", "Unknown error", ext={"exception": f"{error}"})	×
134
135	# Connexion's error handling sometimes breaks when parameters contain
136	# unicode characters (https://github.com/zalando/connexion/issues/604).
137	# To work around, we catch them and return a 400 (which is what Connexion
138	# would do if it didn't hit this error).
139	@flask_app.errorhandler(UnicodeEncodeError)	1✔
140	def unicode(error):	1✔
141	return problem(400, "Unicode Error", "Connexion was unable to parse some unicode data correctly.")	×
142
143	@flask_app.errorhandler(Exception)	1✔
144	def ise(error):	1✔
145	capture_exception(error)	×
146	log.exception("Caught ISE 500 error: %r", error)	×
147	log.debug("Request path is: %s", request.path)	×
148	log.debug("Request environment is: %s", request.environ)	×
149	log.debug("Request headers are: %s", request.headers)	×
150	return problem(500, "Internal Server Error", "Internal Server Error")	×
151
152	@flask_app.after_request	1✔
153	def add_security_headers(response):	1✔
154	response.headers["X-Frame-Options"] = "DENY"	×
155	response.headers["X-Content-Type-Options"] = "nosniff"	×
156	response.headers["Strict-Transport-Security"] = flask_app.config.get("STRICT_TRANSPORT_SECURITY", "max-age=31536000;")	×
157	if re.match("^/ui/", request.path):	×
158	# This enables swagger-ui to dynamically fetch and
159	# load the swagger specification JSON file containing API definition and examples.
160	response.headers["X-Frame-Options"] = "SAMEORIGIN"	×
161	else:
162	response.headers["Content-Security-Policy"] = flask_app.config.get("CONTENT_SECURITY_POLICY", "default-src 'none'; frame-ancestors 'none'")	×
163	return response	×
164
165	class StatsdMiddleware:	1✔
166	def __init__(self, app):	1✔
167	self.app = app	1✔
168
169	def metric_name(self, scope):	1✔
170	if scope["method"] == "OPTIONS":	1!
NEW 171	return	×
172	breakpoint()	1✔
NEW 173	op = scope.get("extensions", {}).get("connexion_routing", {}).get("operation_id")	×
NEW 174	if op is None:	×
NEW 175	return	×
176	# do some massaging to get the metric name right
177	# * remove various module prefixes
178	# * add a common prefix to ensure that we can mark these metrics as gauges for
179	# statsd
NEW 180	metric = op.replace(".", "_").removeprefix("auslib_web_admin_views_").removeprefix("auslib_web_admin_").removeprefix("auslib_web_common_")	×
NEW 181	return f"endpoint_{metric}"	×
182
183	async def __call__(self, scope, receive, send):	1✔
184	if scope["type"] != "http":	1!
NEW 185	await self.app(scope, receive, send)	×
NEW 186	return	×
187
188	metric = self.metric_name(scope)	1✔
NEW 189	if not metric:	×
NEW 190	await self.app(scope, receive, send)	×
NEW 191	return	×
192
NEW 193	timer = statsd.timer(metric)	×
NEW 194	timer.start()	×
NEW 195	try:	×
NEW 196	await self.app(scope, receive, send)	×
197	finally:
NEW 198	timer.stop()	×
199
200	connexion_app.add_middleware(StatsdMiddleware, MiddlewarePosition.BEFORE_VALIDATION)	1✔
201
202	if allow_origins:	1!
203	connexion_app.add_middleware(	1✔
204	CORSMiddleware,
205	MiddlewarePosition.BEFORE_ROUTING,
206	allow_origins=allow_origins,
207	allow_headers=["Authorization", "Content-Type"],
208	allow_methods=["OPTIONS", "GET", "POST", "PUT", "DELETE"],
209	)
210
211	return connexion_app	1✔

mozilla-releng / balrog / #4795

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous