16621609190

Committed 30 Jul 2025 11:43AM UTC coverage: 84.491% (+0.008%) from 84.483%

Build # 16621609190

Build Type

push

github

Committed by

rerowep

Commit Message

chore: update dependencies

Co-Authored-by: Peter Weber <peter.weber@rero.ch>

Run Details

4560 of 5397 relevant lines covered (84.49%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.39

/rero_mef/monitoring/views.py

# RERO MEF
# Copyright (C) 2022 RERO
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

"""Monitoring utilities."""

import time
from functools import wraps

from flask import Blueprint, current_app, jsonify, request, url_for
from flask_login import current_user
from invenio_cache import current_cache
from invenio_db import db
from invenio_search import current_search_client
from redis import Redis

from ..permissions import monitoring_permission
from .api import Monitoring
from .utils import DB_CONNECTION_COUNTS_QUERY, DB_CONNECTION_QUERY

api_blueprint = Blueprint("api_monitoring", __name__, url_prefix="/monitoring")


def check_authentication(func):
    """Decorator to check authentication for monitoring HTTP API."""

    @wraps(func)
    def decorated_view(*args, **kwargs):
        if not current_user.is_authenticated:
            return jsonify({"status": "error: Unauthorized"}), 401
        if not monitoring_permission.require().can():
            return jsonify({"status": "error: Forbidden"}), 403
        return func(*args, **kwargs)

    return decorated_view


@api_blueprint.route("/db_connection_counts")
@check_authentication
def db_connection_counts():
    """Display DB connection counts.

    :return: jsonified count for db connections
    """
    try:
        max_conn, used, res_for_super, free = db.session.execute(
            DB_CONNECTION_COUNTS_QUERY
        ).first()
    except Exception as error:
        return jsonify({"ERROR": error})
    return jsonify(
        {
            "data": {
                "max": max_conn,
                "used": used,
                "res_super": res_for_super,
                "free": free,
            }
        }
    )


@api_blueprint.route("/db_connections")
@check_authentication
def db_connections():
    """Display DB connections.

    :return: jsonified connections for db
    """
    try:
        results = db.session.execute(DB_CONNECTION_QUERY).fetchall()
    except Exception as error:
        return jsonify({"ERROR": error})
    data = {}
    for (
        pid,
        application_name,
        client_addr,
        client_port,
        backend_start,
        xact_start,
        query_start,
        wait_event,
        state,
        left,
    ) in results:
        data[pid] = {
            "application_name": application_name,
            "client_addr": client_addr,
            "client_port": client_port,
            "backend_start": backend_start,
            "xact_start": xact_start,
            "query_start": query_start,
            "wait_event": wait_event,
            "state": state,
            "left": left,
        }
    return jsonify({"data": data})


@api_blueprint.route("/es_db_counts")
def es_db_counts():
    """Display count for elasticsearch and documents.

    Displays for all document types defind in config.py following informations:
    - index name for document type
    - count of records in database
    - count of records in elasticsearch
    - difference between the count in elasticsearch and database
    :return: jsonified count for elasticsearch and documents
    """
    return jsonify(
        {
            "data": Monitoring().info(
                with_deleted=request.args.get(
                    "deleted", default=False, type=lambda v: v.lower() in ["true", "1"]
                ),
                difference_db_es=request.args.get(
                    "diff", default=False, type=lambda v: v.lower() in ["true", "1"]
                ),
            )
        }
    )


@api_blueprint.route("/mef_counts")
def mef_counts():
    """Display count for mef and documents.

    Displays for all document types defind in config.py following informations:
    - count of records in database
    - count of records in MEF
    - difference between the count in database and MEF
    :return: jsonified count for MEF and documents
    """
    return jsonify({"data": Monitoring().check_mef()})


@api_blueprint.route("/check_es_db_counts")
def check_es_db_counts():
    """Displays health status for elasticsearch and database counts.

    If there are no problems the status in returned data will be `green`,
    otherwise the status will be `red` and in the returned error
    links will be provided with more detailed informations.
    :return: jsonified health status for elasticsearch and database counts
    """
    result = {"data": {"status": "green"}}
    if checks := Monitoring().check(
        with_deleted=request.args.get(
            "deleted", default=False, type=lambda v: v.lower() in ["true", "1"]
        ),
        difference_db_es=request.args.get(
            "diff", default=False, type=lambda v: v.lower() in ["true", "1"]
        ),
    ):
        errors = []
        for doc_type, doc_type_data in checks.items():
            links = {
                "about": url_for("api_monitoring.check_es_db_counts", _external=True)
            }
            for info, count in doc_type_data.items():
                if info == "db_es":
                    links[doc_type] = url_for(
                        "api_monitoring.missing_pids", doc_type=doc_type, _external=True
                    )
                    errors.append(
                        {
                            "id": "DB_ES_COUNTER_MISSMATCH",
                            "links": links,
                            "code": "DB_ES_COUNTER_MISSMATCH",
                            "title": "DB items counts don't match ES items count.",
                            "details": f"There are {count} items from "
                            f"{doc_type} missing in ES.",
                        }
                    )
                elif info == "db-":
                    links[doc_type] = url_for(
                        "api_monitoring.missing_pids", doc_type=doc_type, _external=True
                    )
                    errors.append(
                        {
                            "id": "DB_ES_UNEQUAL",
                            "links": links,
                            "code": "DB_ES_UNEQUAL",
                            "title": "DB items unequal ES items.",
                            "details": f"There are {count} items from "
                            f"{doc_type} missing in DB.",
                        }
                    )
                elif info == "es-":
                    links[doc_type] = url_for(
                        "api_monitoring.missing_pids", doc_type=doc_type, _external=True
                    )
                    errors.append(
                        {
                            "id": "DB_ES_UNEQUAL",
                            "links": links,
                            "code": "DB_ES_UNEQUAL",
                            "title": "DB items unequal ES items.",
                            "details": f"There are {count} items from "
                            f"{doc_type} missing in ES.",
                        }
                    )
        result = {"data": {"status": "red"}, "errors": errors}
    return jsonify(result)


@api_blueprint.route("/missing_pids/<doc_type>")
@check_authentication
def missing_pids(doc_type):
    """Displays details of counts for document type.

    Following informations will be displayed:
    - missing pids in database
    - missing pids in elasticsearch
    - pids indexed multiple times in elasticsearch
    If possible, direct links will be provieded to the corresponding records.
    This view needs an logged in system admin.

    :param doc_type: Document type to display.
    :return: jsonified details of counts for document type
    """
    try:
        api_url = url_for(f"invenio_records_rest.{doc_type}_list", _external=True)
    except Exception:
        api_url = None
    delay = request.args.get("delay", default=1, type=int)
    mon = Monitoring(time_delta=delay).missing(doc_type)
    if mon.get("ERROR"):
        return {
            "error": {
                "id": "DOCUMENT_TYPE_NOT_FOUND",
                "code": "DOCUMENT_TYPE_NOT_FOUND",
                "title": "Document type not found.",
                "details": mon.get("ERROR"),
            }
        }
    data = {"DB": [], "ES": [], "ES duplicate": []}
    for pid in mon.get("DB"):
        if api_url:
            data["DB"].append(f"{api_url}?q=pid:{pid}")
        else:
            data["DB"].append(pid)
    for pid in mon.get("ES"):
        if api_url:
            data["ES"].append(f"{api_url}{pid}")
        else:
            data["ES"].append(pid)
    for pid in mon.get("ES duplicate"):
        if api_url:
            url = f"{api_url}?q=pid:{pid}"
            data["ES duplicate"][url] = len(mon.get("ES duplicate"))
        else:
            data["ES duplicate"][pid] = len(mon.get("ES duplicate"))
    return jsonify({"data": data})


@api_blueprint.route("/redis")
@check_authentication
def redis():
    """Displays redis info.

    :return: jsonified redis info.
    """
    url = current_app.config.get("ACCOUNTS_SESSION_REDIS_URL", "redis://localhost:6379")
    redis = Redis.from_url(url)
    info = redis.info()
    return jsonify({"data": info})


@api_blueprint.route("/es_indices")
@check_authentication
def elastic_search_indices():
    """Displays Elasticsearch indices info.

    :return: jsonified Elasticsearch indices info.
    """
    info = current_search_client.cat.indices(bytes="b", format="json", s="index")
    info = {data["index"]: data for data in info}
    return jsonify({"data": info})


@api_blueprint.route("/timestamps")
@check_authentication
def timestamps():
    """Get time stamps from current cache.

    Makes the saved timestamps accessible via url requests.

    :return: jsonified timestamps.
    """
    data = {}
    if time_stamps := current_cache.get("timestamps"):
        for name, values in time_stamps.items():
            data[name] = {}
            for key, value in values.items():
                if key == "time":
                    data[name]["utctime"] = value.strftime("%Y-%m-%d %H:%M:%S")
                    data[name]["unixtime"] = time.mktime(value.timetuple())
                else:
                    data[name][key] = value

    return jsonify({"data": data})


@api_blueprint.route("/es")
@check_authentication
def elastic_search():
    """Displays elastic search cluster info.

    :return: jsonified elastic search cluster info.
    """
    info = current_search_client.cluster.health()
    return jsonify({"data": info})

1	# RERO MEF
2	# Copyright (C) 2022 RERO
3	#
4	# This program is free software: you can redistribute it and/or modify
5	# it under the terms of the GNU Affero General Public License as published by
6	# the Free Software Foundation, version 3 of the License.
7	#
8	# This program is distributed in the hope that it will be useful,
9	# but WITHOUT ANY WARRANTY; without even the implied warranty of
10	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11	# GNU Affero General Public License for more details.
12	#
13	# You should have received a copy of the GNU Affero General Public License
14	# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
16	"""Monitoring utilities."""
17
18	import time	1✔
19	from functools import wraps	1✔
20
21	from flask import Blueprint, current_app, jsonify, request, url_for	1✔
22	from flask_login import current_user	1✔
23	from invenio_cache import current_cache	1✔
24	from invenio_db import db	1✔
25	from invenio_search import current_search_client	1✔
26	from redis import Redis	1✔
27
28	from ..permissions import monitoring_permission	1✔
29	from .api import Monitoring	1✔
30	from .utils import DB_CONNECTION_COUNTS_QUERY, DB_CONNECTION_QUERY	1✔
31
32	api_blueprint = Blueprint("api_monitoring", __name__, url_prefix="/monitoring")	1✔
33
34
35	def check_authentication(func):	1✔
36	"""Decorator to check authentication for monitoring HTTP API."""
37
38	@wraps(func)	1✔
39	def decorated_view(args, *kwargs):	1✔
40	if not current_user.is_authenticated:	1✔
41	return jsonify({"status": "error: Unauthorized"}), 401	1✔
42	if not monitoring_permission.require().can():	1✔
43	return jsonify({"status": "error: Forbidden"}), 403	×
44	return func(args, *kwargs)	1✔
45
46	return decorated_view	1✔
47
48
49	@api_blueprint.route("/db_connection_counts")	1✔
50	@check_authentication	1✔
51	def db_connection_counts():	1✔
52	"""Display DB connection counts.
53
54	:return: jsonified count for db connections
55	"""
56	try:	×
57	max_conn, used, res_for_super, free = db.session.execute(	×
58	DB_CONNECTION_COUNTS_QUERY
59	).first()
60	except Exception as error:	×
61	return jsonify({"ERROR": error})	×
62	return jsonify(	×
63	{
64	"data": {
65	"max": max_conn,
66	"used": used,
67	"res_super": res_for_super,
68	"free": free,
69	}
70	}
71	)
72
73
74	@api_blueprint.route("/db_connections")	1✔
75	@check_authentication	1✔
76	def db_connections():	1✔
77	"""Display DB connections.
78
79	:return: jsonified connections for db
80	"""
81	try:	×
82	results = db.session.execute(DB_CONNECTION_QUERY).fetchall()	×
83	except Exception as error:	×
84	return jsonify({"ERROR": error})	×
85	data = {}	×
86	for (	×
87	pid,
88	application_name,
89	client_addr,
90	client_port,
91	backend_start,
92	xact_start,
93	query_start,
94	wait_event,
95	state,
96	left,
97	) in results:
98	data[pid] = {	×
99	"application_name": application_name,
100	"client_addr": client_addr,
101	"client_port": client_port,
102	"backend_start": backend_start,
103	"xact_start": xact_start,
104	"query_start": query_start,
105	"wait_event": wait_event,
106	"state": state,
107	"left": left,
108	}
109	return jsonify({"data": data})	×
110
111
112	@api_blueprint.route("/es_db_counts")	1✔
113	def es_db_counts():	1✔
114	"""Display count for elasticsearch and documents.
115
116	Displays for all document types defind in config.py following informations:
117	- index name for document type
118	- count of records in database
119	- count of records in elasticsearch
120	- difference between the count in elasticsearch and database
121	:return: jsonified count for elasticsearch and documents
122	"""
123	return jsonify(	1✔
124	{
125	"data": Monitoring().info(
126	with_deleted=request.args.get(
127	"deleted", default=False, type=lambda v: v.lower() in ["true", "1"]
128	),
129	difference_db_es=request.args.get(
130	"diff", default=False, type=lambda v: v.lower() in ["true", "1"]
131	),
132	)
133	}
134	)
135
136
137	@api_blueprint.route("/mef_counts")	1✔
138	def mef_counts():	1✔
139	"""Display count for mef and documents.
140
141	Displays for all document types defind in config.py following informations:
142	- count of records in database
143	- count of records in MEF
144	- difference between the count in database and MEF
145	:return: jsonified count for MEF and documents
146	"""
147	return jsonify({"data": Monitoring().check_mef()})	×
148
149
150	@api_blueprint.route("/check_es_db_counts")	1✔
151	def check_es_db_counts():	1✔
152	"""Displays health status for elasticsearch and database counts.
153
154	If there are no problems the status in returned data will be `green`,
155	otherwise the status will be `red` and in the returned error
156	links will be provided with more detailed informations.
157	:return: jsonified health status for elasticsearch and database counts
158	"""
159	result = {"data": {"status": "green"}}	1✔
160	if checks := Monitoring().check(	1✔
161	with_deleted=request.args.get(
162	"deleted", default=False, type=lambda v: v.lower() in ["true", "1"]
163	),
164	difference_db_es=request.args.get(
165	"diff", default=False, type=lambda v: v.lower() in ["true", "1"]
166	),
167	):
168	errors = []	1✔
169	for doc_type, doc_type_data in checks.items():	1✔
170	links = {	1✔
171	"about": url_for("api_monitoring.check_es_db_counts", _external=True)
172	}
173	for info, count in doc_type_data.items():	1✔
174	if info == "db_es":	1✔
175	links[doc_type] = url_for(	1✔
176	"api_monitoring.missing_pids", doc_type=doc_type, _external=True
177	)
178	errors.append(	1✔
179	{
180	"id": "DB_ES_COUNTER_MISSMATCH",
181	"links": links,
182	"code": "DB_ES_COUNTER_MISSMATCH",
183	"title": "DB items counts don't match ES items count.",
184	"details": f"There are {count} items from "
185	f"{doc_type} missing in ES.",
186	}
187	)
188	elif info == "db-":	×
189	links[doc_type] = url_for(	×
190	"api_monitoring.missing_pids", doc_type=doc_type, _external=True
191	)
192	errors.append(	×
193	{
194	"id": "DB_ES_UNEQUAL",
195	"links": links,
196	"code": "DB_ES_UNEQUAL",
197	"title": "DB items unequal ES items.",
198	"details": f"There are {count} items from "
199	f"{doc_type} missing in DB.",
200	}
201	)
202	elif info == "es-":	×
203	links[doc_type] = url_for(	×
204	"api_monitoring.missing_pids", doc_type=doc_type, _external=True
205	)
206	errors.append(	×
207	{
208	"id": "DB_ES_UNEQUAL",
209	"links": links,
210	"code": "DB_ES_UNEQUAL",
211	"title": "DB items unequal ES items.",
212	"details": f"There are {count} items from "
213	f"{doc_type} missing in ES.",
214	}
215	)
216	result = {"data": {"status": "red"}, "errors": errors}	1✔
217	return jsonify(result)	1✔
218
219
220	@api_blueprint.route("/missing_pids/<doc_type>")	1✔
221	@check_authentication	1✔
222	def missing_pids(doc_type):	1✔
223	"""Displays details of counts for document type.
224
225	Following informations will be displayed:
226	- missing pids in database
227	- missing pids in elasticsearch
228	- pids indexed multiple times in elasticsearch
229	If possible, direct links will be provieded to the corresponding records.
230	This view needs an logged in system admin.
231
232	:param doc_type: Document type to display.
233	:return: jsonified details of counts for document type
234	"""
235	try:	1✔
236	api_url = url_for(f"invenio_records_rest.{doc_type}_list", _external=True)	1✔
237	except Exception:	×
238	api_url = None	×
239	delay = request.args.get("delay", default=1, type=int)	1✔
240	mon = Monitoring(time_delta=delay).missing(doc_type)	1✔
241	if mon.get("ERROR"):	1✔
242	return {	×
243	"error": {
244	"id": "DOCUMENT_TYPE_NOT_FOUND",
245	"code": "DOCUMENT_TYPE_NOT_FOUND",
246	"title": "Document type not found.",
247	"details": mon.get("ERROR"),
248	}
249	}
250	data = {"DB": [], "ES": [], "ES duplicate": []}	1✔
251	for pid in mon.get("DB"):	1✔
252	if api_url:	×
253	data["DB"].append(f"{api_url}?q=pid:{pid}")	×
254	else:
255	data["DB"].append(pid)	×
256	for pid in mon.get("ES"):	1✔
257	if api_url:	1✔
258	data["ES"].append(f"{api_url}{pid}")	1✔
259	else:
260	data["ES"].append(pid)	×
261	for pid in mon.get("ES duplicate"):	1✔
262	if api_url:	×
263	url = f"{api_url}?q=pid:{pid}"	×
264	data["ES duplicate"][url] = len(mon.get("ES duplicate"))	×
265	else:
266	data["ES duplicate"][pid] = len(mon.get("ES duplicate"))	×
267	return jsonify({"data": data})	1✔
268
269
270	@api_blueprint.route("/redis")	1✔
271	@check_authentication	1✔
272	def redis():	1✔
273	"""Displays redis info.
274
275	:return: jsonified redis info.
276	"""
277	url = current_app.config.get("ACCOUNTS_SESSION_REDIS_URL", "redis://localhost:6379")	×
278	redis = Redis.from_url(url)	×
279	info = redis.info()	×
280	return jsonify({"data": info})	×
281
282
283	@api_blueprint.route("/es_indices")	1✔
284	@check_authentication	1✔
285	def elastic_search_indices():	1✔
286	"""Displays Elasticsearch indices info.
287
288	:return: jsonified Elasticsearch indices info.
289	"""
290	info = current_search_client.cat.indices(bytes="b", format="json", s="index")	×
291	info = {data["index"]: data for data in info}	×
292	return jsonify({"data": info})	×
293
294
295	@api_blueprint.route("/timestamps")	1✔
296	@check_authentication	1✔
297	def timestamps():	1✔
298	"""Get time stamps from current cache.
299
300	Makes the saved timestamps accessible via url requests.
301
302	:return: jsonified timestamps.
303	"""
304	data = {}	1✔
305	if time_stamps := current_cache.get("timestamps"):	1✔
306	for name, values in time_stamps.items():	1✔
307	data[name] = {}	1✔
308	for key, value in values.items():	1✔
309	if key == "time":	1✔
310	data[name]["utctime"] = value.strftime("%Y-%m-%d %H:%M:%S")	1✔
311	data[name]["unixtime"] = time.mktime(value.timetuple())	1✔
312	else:
313	data[name][key] = value	1✔
314
315	return jsonify({"data": data})	1✔
316
317
318	@api_blueprint.route("/es")	1✔
319	@check_authentication	1✔
320	def elastic_search():	1✔
321	"""Displays elastic search cluster info.
322
323	:return: jsonified elastic search cluster info.
324	"""
325	info = current_search_client.cluster.health()	×
326	return jsonify({"data": info})	×

rero / rero-mef / 16621609190

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous