16621609190

Committed 30 Jul 2025 11:43AM UTC coverage: 84.491% (+0.008%) from 84.483%

Build # 16621609190

Build Type

push

github

Committed by

rerowep

Commit Message

chore: update dependencies

Co-Authored-by: Peter Weber <peter.weber@rero.ch>

Run Details

4560 of 5397 relevant lines covered (84.49%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.42

/rero_mef/monitoring/cli.py

# RERO MEF
# Copyright (C) 2022 RERO
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

"""Monitoring utilities."""

import click
from flask import current_app
from flask.cli import with_appcontext
from invenio_db import db
from invenio_search import current_search_client
from redis import Redis

from .api import Monitoring
from .utils import DB_CONNECTION_COUNTS_QUERY, DB_CONNECTION_QUERY


@click.group()
def monitoring():
    """Monitoring commands."""


@monitoring.command("es_db_counts")
@click.option(
    "-m",
    "--missing",
    "missing",
    is_flag=True,
    default=False,
    help="display missing pids",
)
@click.option(
    "-d",
    "--delay",
    "delay",
    default=1,
    help="Get ES and DB counts from delay miniutes in the past.",
)
@with_appcontext
def es_db_counts_cli(missing, delay):
    """Print ES and DB counts.

    Prints a table representation of database and elasticsearch counts.
    Columes:
    1. database count minus elasticsearch count
    2. document type
    3. database count
    4. elasticsearch index
    5. elasticsearch count
    """
    missing_doc_types = []
    mon = Monitoring(time_delta=delay)
    msg_head = f"DB - ES  {'type':>6} {'count':>10}"
    msg_head += f"  {'index':>25} {'count_es':>10}\n"
    msg_head += f"{'':-^64s}"
    click.echo(msg_head)
    info = mon.info(with_deleted=False, difference_db_es=False)
    for doc_type in sorted(info):
        db_es = info[doc_type].get("db-es", "")
        msg = f"{db_es:>7}  {doc_type:>6} {info[doc_type].get('db', ''):>10}"
        index = info[doc_type].get("index", "")
        if index:
            msg += f"  {index:>25} {info[doc_type].get('es', ''):>10}"
        if db_es not in [0, ""]:
            click.secho(msg, fg="red")
        else:
            click.echo(msg)
        if missing and index:
            missing_doc_types.append(doc_type)
    for missing_doc_type in missing_doc_types:
        mon.print_missing(missing_doc_type)


@monitoring.command("mef_counts")
@with_appcontext
@click.option(
    "-d",
    "--delay",
    "delay",
    default=1,
    help="Get ES and DB counts from delay miniutes in the past.",
)
def mef_counts_cli(delay):
    """Print MEF counts.

    Prints a table representation of MEF counts.
    Columes:
    1. MEF count minus database count
    2. document type
    3. database count
    5. MEF count
    """
    mon = Monitoring(time_delta=delay)
    msg_head = f"MEF - DB  {'type':>6} {'DB':>10}  {'MEF':>10}"
    click.echo(msg_head)
    for entity, data in mon.check_mef().items():
        mef_db = data.get("mef-db", "")
        db = data.get("db", "")
        mef = data.get("mef", "")
        msg = f"{mef_db:>8}  {entity:>6} {db:>10}  {mef:>10}"
        if mef_db not in [0, ""]:
            click.secho(msg, fg="red")
        else:
            click.echo(msg)


@monitoring.command("es_db_missing")
@click.argument("doc_type")
@click.option(
    "-d",
    "--delay",
    "delay",
    default=1,
    help="Get ES and DB counts from delay miniutes in the past.",
)
@with_appcontext
def es_db_missing_cli(doc_type, delay):
    """Print missing pids informations."""
    Monitoring(time_delta=delay).print_missing(doc_type)


@monitoring.command()
@with_appcontext
def es():
    """Displays elastic search cluster info."""
    for key, value in current_search_client.cluster.health().items():
        click.echo(f"{key:<33}: {value}")


@monitoring.command()
@with_appcontext
def redis():
    """Displays redis info."""
    url = current_app.config.get("ACCOUNTS_SESSION_REDIS_URL", "redis://localhost:6379")
    redis = Redis.from_url(url)
    for key, value in redis.info().items():
        click.echo(f"{key:<33}: {value}")


@monitoring.command("es_indices")
@with_appcontext
def es_indices():
    """Displays Elasticsearch indices info."""
    click.echo(current_search_client.cat.indices(s="index"))


@monitoring.command("db_connection_counts")
@with_appcontext
def db_connection_counts():
    """Display DB connection counts."""
    try:
        max_conn, used, res_for_super, free = db.session.execute(
            DB_CONNECTION_COUNTS_QUERY
        ).first()
    except Exception as error:
        click.secho(f"ERROR: {error}", fg="red")
    return click.secho(
        f"max: {max_conn}, used: {used}, res_super: {res_for_super}, free: {free}"
    )


@monitoring.command("db_connections")
@with_appcontext
def db_connections():
    """Display DB connections."""
    try:
        results = db.session.execute(DB_CONNECTION_QUERY).fetchall()
    except Exception as error:
        click.secho(f"ERROR: {error}", fg="red")
    for (
        pid,
        application_name,
        client_addr,
        client_port,
        backend_start,
        xact_start,
        query_start,
        wait_event,
        state,
        left,
    ) in results:
        click.secho(
            f"application_name: {application_name}\n"
            f"client_addr: {client_addr}\n"
            f"client_port: {client_port}\n"
            f"backend_start: {backend_start}\n"
            f"xact_start: {xact_start}\n"
            f"query_start: {query_start}\n"
            f"wait_event: {wait_event}\n"
            f"state: {state}\n"
            f"left: {left}\n"
        )

1	# RERO MEF
2	# Copyright (C) 2022 RERO
3	#
4	# This program is free software: you can redistribute it and/or modify
5	# it under the terms of the GNU Affero General Public License as published by
6	# the Free Software Foundation, version 3 of the License.
7	#
8	# This program is distributed in the hope that it will be useful,
9	# but WITHOUT ANY WARRANTY; without even the implied warranty of
10	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11	# GNU Affero General Public License for more details.
12	#
13	# You should have received a copy of the GNU Affero General Public License
14	# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
16	"""Monitoring utilities."""
17
18	import click	1✔
19	from flask import current_app	1✔
20	from flask.cli import with_appcontext	1✔
21	from invenio_db import db	1✔
22	from invenio_search import current_search_client	1✔
23	from redis import Redis	1✔
24
25	from .api import Monitoring	1✔
26	from .utils import DB_CONNECTION_COUNTS_QUERY, DB_CONNECTION_QUERY	1✔
27
28
29	@click.group()	1✔
30	def monitoring():	1✔
31	"""Monitoring commands."""
32
33
34	@monitoring.command("es_db_counts")	1✔
35	@click.option(	1✔
36	"-m",
37	"--missing",
38	"missing",
39	is_flag=True,
40	default=False,
41	help="display missing pids",
42	)
43	@click.option(	1✔
44	"-d",
45	"--delay",
46	"delay",
47	default=1,
48	help="Get ES and DB counts from delay miniutes in the past.",
49	)
50	@with_appcontext	1✔
51	def es_db_counts_cli(missing, delay):	1✔
52	"""Print ES and DB counts.
53
54	Prints a table representation of database and elasticsearch counts.
55	Columes:
56	1. database count minus elasticsearch count
57	2. document type
58	3. database count
59	4. elasticsearch index
60	5. elasticsearch count
61	"""
62	missing_doc_types = []	1✔
63	mon = Monitoring(time_delta=delay)	1✔
64	msg_head = f"DB - ES {'type':>6} {'count':>10}"	1✔
65	msg_head += f" {'index':>25} {'count_es':>10}\n"	1✔
66	msg_head += f"{'':-^64s}"	1✔
67	click.echo(msg_head)	1✔
68	info = mon.info(with_deleted=False, difference_db_es=False)	1✔
69	for doc_type in sorted(info):	1✔
70	db_es = info[doc_type].get("db-es", "")	1✔
71	msg = f"{db_es:>7} {doc_type:>6} {info[doc_type].get('db', ''):>10}"	1✔
72	index = info[doc_type].get("index", "")	1✔
73	if index:	1✔
74	msg += f" {index:>25} {info[doc_type].get('es', ''):>10}"	1✔
75	if db_es not in [0, ""]:	1✔
76	click.secho(msg, fg="red")	1✔
77	else:
78	click.echo(msg)	1✔
79	if missing and index:	1✔
80	missing_doc_types.append(doc_type)	1✔
81	for missing_doc_type in missing_doc_types:	1✔
82	mon.print_missing(missing_doc_type)	1✔
83
84
85	@monitoring.command("mef_counts")	1✔
86	@with_appcontext	1✔
87	@click.option(	1✔
88	"-d",
89	"--delay",
90	"delay",
91	default=1,
92	help="Get ES and DB counts from delay miniutes in the past.",
93	)
94	def mef_counts_cli(delay):	1✔
95	"""Print MEF counts.
96
97	Prints a table representation of MEF counts.
98	Columes:
99	1. MEF count minus database count
100	2. document type
101	3. database count
102	5. MEF count
103	"""
104	mon = Monitoring(time_delta=delay)	×
105	msg_head = f"MEF - DB {'type':>6} {'DB':>10} {'MEF':>10}"	×
106	click.echo(msg_head)	×
107	for entity, data in mon.check_mef().items():	×
108	mef_db = data.get("mef-db", "")	×
109	db = data.get("db", "")	×
110	mef = data.get("mef", "")	×
111	msg = f"{mef_db:>8} {entity:>6} {db:>10} {mef:>10}"	×
112	if mef_db not in [0, ""]:	×
113	click.secho(msg, fg="red")	×
114	else:
115	click.echo(msg)	×
116
117
118	@monitoring.command("es_db_missing")	1✔
119	@click.argument("doc_type")	1✔
120	@click.option(	1✔
121	"-d",
122	"--delay",
123	"delay",
124	default=1,
125	help="Get ES and DB counts from delay miniutes in the past.",
126	)
127	@with_appcontext	1✔
128	def es_db_missing_cli(doc_type, delay):	1✔
129	"""Print missing pids informations."""
130	Monitoring(time_delta=delay).print_missing(doc_type)	1✔
131
132
133	@monitoring.command()	1✔
134	@with_appcontext	1✔
135	def es():	1✔
136	"""Displays elastic search cluster info."""
137	for key, value in current_search_client.cluster.health().items():	×
138	click.echo(f"{key:<33}: {value}")	×
139
140
141	@monitoring.command()	1✔
142	@with_appcontext	1✔
143	def redis():	1✔
144	"""Displays redis info."""
145	url = current_app.config.get("ACCOUNTS_SESSION_REDIS_URL", "redis://localhost:6379")	×
146	redis = Redis.from_url(url)	×
147	for key, value in redis.info().items():	×
148	click.echo(f"{key:<33}: {value}")	×
149
150
151	@monitoring.command("es_indices")	1✔
152	@with_appcontext	1✔
153	def es_indices():	1✔
154	"""Displays Elasticsearch indices info."""
155	click.echo(current_search_client.cat.indices(s="index"))	×
156
157
158	@monitoring.command("db_connection_counts")	1✔
159	@with_appcontext	1✔
160	def db_connection_counts():	1✔
161	"""Display DB connection counts."""
162	try:	×
163	max_conn, used, res_for_super, free = db.session.execute(	×
164	DB_CONNECTION_COUNTS_QUERY
165	).first()
166	except Exception as error:	×
167	click.secho(f"ERROR: {error}", fg="red")	×
168	return click.secho(	×
169	f"max: {max_conn}, used: {used}, res_super: {res_for_super}, free: {free}"
170	)
171
172
173	@monitoring.command("db_connections")	1✔
174	@with_appcontext	1✔
175	def db_connections():	1✔
176	"""Display DB connections."""
177	try:	×
178	results = db.session.execute(DB_CONNECTION_QUERY).fetchall()	×
179	except Exception as error:	×
180	click.secho(f"ERROR: {error}", fg="red")	×
181	for (	×
182	pid,
183	application_name,
184	client_addr,
185	client_port,
186	backend_start,
187	xact_start,
188	query_start,
189	wait_event,
190	state,
191	left,
192	) in results:
193	click.secho(	×
194	f"application_name: {application_name}\n"
195	f"client_addr: {client_addr}\n"
196	f"client_port: {client_port}\n"
197	f"backend_start: {backend_start}\n"
198	f"xact_start: {xact_start}\n"
199	f"query_start: {query_start}\n"
200	f"wait_event: {wait_event}\n"
201	f"state: {state}\n"
202	f"left: {left}\n"
203	)

rero / rero-mef / 16621609190

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous