15300796687

Committed 28 May 2025 01:00PM UTC coverage: 87.028% (-0.05%) from 87.075%

Build # 15300796687

Build Type

push

github

Committed by

web-flow

Commit Message

chore: Improve logging setup (#873)

Introduces a logging setup to centralize its configuration and removes
changes to the logging config from other places in the code base.

The logging config is called in the bases __init__ file to be as early
as possible applied. For sanic applications, it must also run for
workers and the main process. The configuration sets up logging by
default as follows:

- our code is logged at level INFO
- all third-party code is logged at level WARNING

What is "our" code is determined by the logger name: our loggers
should all be children of the renku_data_services logger. Using the
provided getLogger function makes sure this is the case.

The config can be adjusted using environment variables, as written in
`Config.from_env()`.

Run Details

255 of 300 new or added lines in 42 files covered. (85.0%)

8 existing lines in 4 files now uncovered.

21825 of 25078 relevant lines covered (87.03%)

1.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.47

/components/renku_data_services/base_api/error_handler.py

"""The error handler for the application."""

import os
import sys
import traceback
from collections.abc import Mapping, Set
from sqlite3 import Error as SqliteError
from typing import Any, Optional, Protocol, TypeVar, Union

import jwt
from asyncpg import exceptions as postgres_exceptions
from pydantic import ValidationError as PydanticValidationError
from sanic import HTTPResponse, Request, SanicException, json
from sanic.errorpages import BaseRenderer, TextRenderer
from sanic.handlers import ErrorHandler
from sanic_ext.exceptions import ValidationError
from sqlalchemy.exc import SQLAlchemyError

from renku_data_services import errors


class BaseError(Protocol):
    """Protocol for the error type of an apispec module."""

    code: int
    message: str
    detail: Optional[str]
    quiet: bool


class BaseErrorResponse(Protocol):
    """Protocol for the error response class of an apispec module."""

    error: BaseError

    def dict(
        self,
        *,
        include: Optional[Union[Set[Union[int, str]], Mapping[Union[int, str], Any]]] = None,
        exclude: Optional[Union[Set[Union[int, str]], Mapping[Union[int, str], Any]]] = None,
        by_alias: bool = False,
        skip_defaults: Optional[bool] = None,
        exclude_unset: bool = False,
        exclude_defaults: bool = False,
        exclude_none: bool = False,
    ) -> dict[str, Any]:
        """Turn the response to dict."""
        ...


BError = TypeVar("BError", bound=BaseError)
BErrorResponse = TypeVar("BErrorResponse", bound=BaseErrorResponse)


class ApiSpec(Protocol[BErrorResponse, BError]):
    """Protocol for an apispec with error data."""

    ErrorResponse: BErrorResponse
    Error: BError


class CustomErrorHandler(ErrorHandler):
    """Central error handling."""

    def __init__(self, api_spec: ApiSpec, base: type[BaseRenderer] = TextRenderer) -> None:
        self.api_spec = api_spec
        super().__init__(base)

    def default(self, request: Request, exception: Exception) -> HTTPResponse:
        """Overrides the default error handler."""
        formatted_exception = errors.BaseError()
        match exception:
            case errors.BaseError():
                formatted_exception = exception
            case ValidationError():
                extra_exception = None if exception.extra is None else exception.extra["exception"]
                match extra_exception:
                    case TypeError():
                        formatted_exception = errors.ValidationError(
                            message="The validation failed because the provided input has the wrong type"
                        )
                    case PydanticValidationError():
                        parts = [
                            ".".join(str(i) for i in field["loc"]) + ": " + field["msg"]
                            for field in extra_exception.errors()
                        ]
                        message = f"There are errors in the following fields, {', '.join(parts)}"
                        formatted_exception = errors.ValidationError(message=message)
            case SanicException():
                message = exception.message
                if message == "" or message is None:
                    message = ", ".join([str(i) for i in exception.args])
                formatted_exception = errors.BaseError(
                    message=message,
                    status_code=exception.status_code,
                    code=1000 + exception.status_code,
                    quiet=exception.quiet or False,
                )
            case SqliteError():
                formatted_exception = errors.BaseError(
                    message=f"Database error occurred: {exception.sqlite_errorname}",
                    detail=f"Error code: {exception.sqlite_errorcode}",
                )
            case postgres_exceptions.PostgresError():
                formatted_exception = errors.BaseError(
                    message=f"Database error occurred: {exception.msg}", detail=f"Error code: {exception.pgcode}"
                )
            case SQLAlchemyError():
                message = ", ".join([str(i) for i in exception.args])
                if "CharacterNotInRepertoireError" in message:
                    # NOTE: This message is usually triggered if a string field for the database contains
                    # NULL - i.e \u0000 or other invalid characters that are not UTF-8 compatible
                    formatted_exception = errors.ValidationError(
                        message="The payload contains characters that are incompatible with the database",
                        detail=message,
                    )
                elif "value out of int32 range" in message:
                    formatted_exception = errors.ValidationError(
                        message="The payload contains integers with values that are "
                        "too large or small for the database",
                        detail=message,
                    )
                else:
                    formatted_exception = errors.BaseError(message=f"Database error occurred: {message}")
            case PydanticValidationError():
                parts = [".".join(str(i) for i in field["loc"]) + ": " + field["msg"] for field in exception.errors()]
                message = f"There are errors in the following fields, {', '.join(parts)}"
                formatted_exception = errors.ValidationError(message=message)
            case OverflowError():
                formatted_exception = errors.ValidationError(
                    message="The provided input is too large to be stored in the database"
                )
            case jwt.exceptions.InvalidTokenError():
                formatted_exception = errors.InvalidTokenError()
        self.log(request, formatted_exception)
        if formatted_exception.status_code == 500 and "PYTEST_CURRENT_TEST" in os.environ:
            # TODO: Figure out how to do logging properly in here, I could not get the sanic logs to show up from here
            # at all when running schemathesis. So 500 errors are hard to debug but print statements do show up.
            # The above log statement does not show up in the logs that pytest shows after a test is done.
            sys.stderr.write(f"A 500 error was raised because of {type(exception)} on request {request}\n")
            traceback.print_exception(exception)
        return json(
            self.api_spec.ErrorResponse(
                error=self.api_spec.Error(
                    code=formatted_exception.code,
                    message=formatted_exception.message,
                    detail=formatted_exception.detail,
                )
            ).model_dump(exclude_none=True),
            status=formatted_exception.status_code,
        )

1	"""The error handler for the application."""
2
3	import os	2✔
4	import sys	2✔
5	import traceback	2✔
6	from collections.abc import Mapping, Set	2✔
7	from sqlite3 import Error as SqliteError	2✔
8	from typing import Any, Optional, Protocol, TypeVar, Union	2✔
9
10	import jwt	2✔
11	from asyncpg import exceptions as postgres_exceptions	2✔
12	from pydantic import ValidationError as PydanticValidationError	2✔
13	from sanic import HTTPResponse, Request, SanicException, json	2✔
14	from sanic.errorpages import BaseRenderer, TextRenderer	2✔
15	from sanic.handlers import ErrorHandler	2✔
16	from sanic_ext.exceptions import ValidationError	2✔
17	from sqlalchemy.exc import SQLAlchemyError	2✔
18
19	from renku_data_services import errors	2✔
20
21
22	class BaseError(Protocol):	2✔
23	"""Protocol for the error type of an apispec module."""
24
25	code: int	2✔
26	message: str	2✔
27	detail: Optional[str]	2✔
28	quiet: bool	2✔
29
30
31	class BaseErrorResponse(Protocol):	2✔
32	"""Protocol for the error response class of an apispec module."""
33
34	error: BaseError	2✔
35
36	def dict(	2✔
37	self,
38	*,
39	include: Optional[Union[Set[Union[int, str]], Mapping[Union[int, str], Any]]] = None,
40	exclude: Optional[Union[Set[Union[int, str]], Mapping[Union[int, str], Any]]] = None,
41	by_alias: bool = False,
42	skip_defaults: Optional[bool] = None,
43	exclude_unset: bool = False,
44	exclude_defaults: bool = False,
45	exclude_none: bool = False,
46	) -> dict[str, Any]:
47	"""Turn the response to dict."""
48	...	×
49
50
51	BError = TypeVar("BError", bound=BaseError)	2✔
52	BErrorResponse = TypeVar("BErrorResponse", bound=BaseErrorResponse)	2✔
53
54
55	class ApiSpec(Protocol[BErrorResponse, BError]):	2✔
56	"""Protocol for an apispec with error data."""
57
58	ErrorResponse: BErrorResponse	2✔
59	Error: BError	2✔
60
61
62	class CustomErrorHandler(ErrorHandler):	2✔
63	"""Central error handling."""
64
65	def __init__(self, api_spec: ApiSpec, base: type[BaseRenderer] = TextRenderer) -> None:	2✔
66	self.api_spec = api_spec	2✔
67	super().__init__(base)	2✔
68
69	def default(self, request: Request, exception: Exception) -> HTTPResponse:	2✔
70	"""Overrides the default error handler."""
71	formatted_exception = errors.BaseError()	2✔
72	match exception:	2✔
73	case errors.BaseError():	2✔
74	formatted_exception = exception	2✔
75	case ValidationError():	2✔
76	extra_exception = None if exception.extra is None else exception.extra["exception"]	2✔
77	match extra_exception:	2✔
78	case TypeError():	2✔
79	formatted_exception = errors.ValidationError(	2✔
80	message="The validation failed because the provided input has the wrong type"
81	)
82	case PydanticValidationError():	2✔
83	parts = [	2✔
84	".".join(str(i) for i in field["loc"]) + ": " + field["msg"]
85	for field in extra_exception.errors()
86	]
87	message = f"There are errors in the following fields, {', '.join(parts)}"	2✔
88	formatted_exception = errors.ValidationError(message=message)	2✔
89	case SanicException():	2✔
90	message = exception.message	2✔
91	if message == "" or message is None:	2✔
92	message = ", ".join([str(i) for i in exception.args])	×
93	formatted_exception = errors.BaseError(	2✔
94	message=message,
95	status_code=exception.status_code,
96	code=1000 + exception.status_code,
97	quiet=exception.quiet or False,
98	)
99	case SqliteError():	2✔
100	formatted_exception = errors.BaseError(	×
101	message=f"Database error occurred: {exception.sqlite_errorname}",
102	detail=f"Error code: {exception.sqlite_errorcode}",
103	)
104	case postgres_exceptions.PostgresError():	2✔
105	formatted_exception = errors.BaseError(	×
106	message=f"Database error occurred: {exception.msg}", detail=f"Error code: {exception.pgcode}"
107	)
108	case SQLAlchemyError():	2✔
109	message = ", ".join([str(i) for i in exception.args])	1✔
110	if "CharacterNotInRepertoireError" in message:	1✔
111	# NOTE: This message is usually triggered if a string field for the database contains
112	# NULL - i.e \u0000 or other invalid characters that are not UTF-8 compatible
113	formatted_exception = errors.ValidationError(	×
114	message="The payload contains characters that are incompatible with the database",
115	detail=message,
116	)
117	elif "value out of int32 range" in message:	1✔
UNCOV 118	formatted_exception = errors.ValidationError(	×
119	message="The payload contains integers with values that are "
120	"too large or small for the database",
121	detail=message,
122	)
123	else:
124	formatted_exception = errors.BaseError(message=f"Database error occurred: {message}")	1✔
125	case PydanticValidationError():	2✔
126	parts = [".".join(str(i) for i in field["loc"]) + ": " + field["msg"] for field in exception.errors()]	2✔
127	message = f"There are errors in the following fields, {', '.join(parts)}"	2✔
128	formatted_exception = errors.ValidationError(message=message)	2✔
129	case OverflowError():	1✔
130	formatted_exception = errors.ValidationError(	×
131	message="The provided input is too large to be stored in the database"
132	)
133	case jwt.exceptions.InvalidTokenError():	1✔
134	formatted_exception = errors.InvalidTokenError()	×
135	self.log(request, formatted_exception)	2✔
136	if formatted_exception.status_code == 500 and "PYTEST_CURRENT_TEST" in os.environ:	2✔
137	# TODO: Figure out how to do logging properly in here, I could not get the sanic logs to show up from here
138	# at all when running schemathesis. So 500 errors are hard to debug but print statements do show up.
139	# The above log statement does not show up in the logs that pytest shows after a test is done.
140	sys.stderr.write(f"A 500 error was raised because of {type(exception)} on request {request}\n")	1✔
141	traceback.print_exception(exception)	1✔
142	return json(	2✔
143	self.api_spec.ErrorResponse(
144	error=self.api_spec.Error(
145	code=formatted_exception.code,
146	message=formatted_exception.message,
147	detail=formatted_exception.detail,
148	)
149	).model_dump(exclude_none=True),
150	status=formatted_exception.status_code,
151	)

SwissDataScienceCenter / renku-data-services / 15300796687

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous