30269105825

Committed 27 Jul 2026 01:11PM UTC coverage: 91.264% (-0.03%) from 91.292%

Build # 30269105825

Build Type

Pull #4186

github

Committed by

web-flow

Commit Message

Merge 237ac5265 into edbd95eec

Pull Request Pull Request #4186: build: update dependencies

Coverage Stats

82 of 109 new or added lines in 29 files covered. (75.23%)

24238 of 26558 relevant lines covered (91.26%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.79

/rero_ils/modules/migrations/data/cli.py

# SPDX-FileCopyrightText: Fondation RERO+
# SPDX-License-Identifier: AGPL-3.0-or-later

"""Command line interface for migration data record management."""

import json
from pprint import pformat
from random import choice

import click
from elasticsearch.exceptions import NotFoundError
from elasticsearch_dsl import Index
from flask.cli import with_appcontext

from rero_ils.modules.utils import JsonWriter

from ..api import Migration
from .api import DeduplicationCandidate, DeduplicationStatus


@click.group()
def data():
    """Migration data commands."""


@data.command()
@click.argument("migration")
@click.argument("infile", type=click.File("r"))
@click.option("-n", "--dry-run", is_flag=True, default=False)
@with_appcontext
def load(migration, infile, dry_run):
    """Load the data for a given migration."""
    try:
        migration = Migration.get(migration)
    except NotFoundError:
        click.secho(f"Migration with: {migration} does not exists", fg="yellow")
        raise click.Abort()
    ConvertClass = migration.conversion_class
    MigrationData = migration.data_class
    with click.progressbar(ConvertClass.loads(infile)) as bar:
        for record in bar:
            if dry_run:
                print(ConvertClass.markdown(record), "\n")
            else:
                migration_data = MigrationData(raw=record, migration_id=migration.meta.id)
                migration_data.save()
    index = Index(migration.data_index_name)
    index.refresh()
    n_data = MigrationData.search().filter("term", migration_id=migration.meta.id).count()
    click.secho(f"Migration ({migration}) has now {n_data} converted files.")


@data.command()
@click.argument("migration")
@click.option("-i", "--id")
@click.option("-n", "--dry-run", is_flag=True, default=False)
@click.option("-f", "--force", is_flag=True, default=False)
@with_appcontext
def dedup(migration, id, dry_run, force):
    """Load the data for a given migration."""
    try:
        migration = Migration.get(migration)
    except NotFoundError:
        click.secho(f"Migration with: {migration} does not exists", fg="yellow")
        raise click.Abort()
    ConvertClass = migration.conversion_class
    MigrationData = migration.data_class
    search = MigrationData.search()
    if id:
        search = search.filter("ids", values=id.split(","))
    with click.progressbar(list(search.scan()), length=search.count()) as bar:
        for record in bar:
            ils_pid, logs, status, candidates = ConvertClass.dedup(record, force)
            if dry_run:
                print(
                    "\n======================================\n",
                    f"ID: {record.meta.id}\n",
                    f"Status: {status}",
                )
                for pid, json, score, detailed_score in candidates:
                    print(f"ILS pid: {pid}", f"{score:.2f}")
                    print(pformat(detailed_score, indent=2))
            else:
                record.deduplication.candidates = [
                    DeduplicationCandidate(
                        pid=pid,
                        json=json,
                        score=score,
                        detailed_score=detailed_score,
                    )
                    for pid, json, score, detailed_score in candidates
                ]
                record.deduplication.logs = logs
                record.deduplication.status = status
                record.deduplication.ils_pid = ils_pid
                record.save()


@data.command()
@click.argument("migration")
@click.option("-i", "--id")
@click.option("-c", "--conversion-status")
@click.option(
    "-f",
    "--format",
    type=click.Choice(["raw", "json", "markdown", "ids", "logs", "conversion-status"]),
    default="json",
)
@click.option("-o", "--out-file")
@with_appcontext
def get(migration, id, conversion_status, format, out_file):
    """Get migration data."""
    try:
        migration = Migration.get(migration)
    except NotFoundError:
        click.secho(f"Migration with: {migration} does not exists", fg="yellow")
        raise click.Abort()
    DataMigration = migration.data_class

    def format_data(data, format=json):
        """Display the data for a given format."""
        if format == "json":
            return data.conversion.to_dict().get("json")
        if format == "ids":
            return data.meta.id
        if format == "raw":
            return data.raw.decode()
        if format == "markdown":
            ConversionClass = migration.conversion_class
            return ConversionClass.markdown(data.raw)
        if format == "conversion-status" and hasattr(data.conversion, "status"):
            return (data.meta.id, data.conversion.status)
        return None

    if out_file:
        out_file = JsonWriter(out_file) if format == "json" else open(out_file, "w")
    search = DataMigration.search()
    if id:
        search = search.filter("term", _id=id)
    if conversion_status:
        search = search.filter("term", conversion__status=conversion_status)
    for data in search.scan():
        msg = format_data(data=data, format=format)
        if format == "logs":
            msg = f"Logs for {data.meta.id}"
            print(msg)
            if out_file:
                out_file.write(f"{msg}\n")
            if hasattr(data.conversion.logs, "info"):
                msg = "\n".join(data.conversion.logs.info)
                click.secho(msg, fg="green")
            if hasattr(data.conversion.logs, "warnings"):
                msg = "\n".join(data.conversion.logs.warnings)
                click.secho(msg, fg="yellow")
            if hasattr(data.conversion.logs, "error"):
                msg = "\n".join(data.conversion.logs.error)
                click.secho(msg, fg="red")
            if out_file:
                out_file.write(f"{msg}\n")
        else:
            if format == "json":
                print(json.dumps(msg, indent=2))
            else:
                print(msg)
            if out_file:
                out_file.write(msg)
                if format != "json":
                    out_file.write("\n")
    if out_file:
        out_file.close()


@data.command()
@click.argument("migration")
@click.argument("sets", type=str)
@click.option("-s", "--status", type=click.Choice([item.value for item in DeduplicationStatus]))
@with_appcontext
def subsets(migration, sets, status):
    """Get migration data."""
    try:
        migration = Migration.get(migration)
    except NotFoundError:
        click.secho(f"Migration with: {migration} does not exists", fg="yellow")
        raise click.Abort()
    DataMigration = migration.data_class
    sets = sets.split(",")

    search = DataMigration.search()
    if status:
        search = search.filter("term", deduplication__status=status)

    n = 0
    with click.progressbar(list(search.scan()), length=search.count()) as bar:
        for record in bar:
            record.deduplication.subset = choice(sets)
            record.save()
            n += 1
    click.secho(f"{n} records has been assigned to {len(sets)} sets.")

1	# SPDX-FileCopyrightText: Fondation RERO+
2	# SPDX-License-Identifier: AGPL-3.0-or-later
3
4	"""Command line interface for migration data record management."""
5
6	import json	1✔
7	from pprint import pformat	1✔
8	from random import choice	1✔
9
10	import click	1✔
11	from elasticsearch.exceptions import NotFoundError	1✔
12	from elasticsearch_dsl import Index	1✔
13	from flask.cli import with_appcontext	1✔
14
15	from rero_ils.modules.utils import JsonWriter	1✔
16
17	from ..api import Migration	1✔
18	from .api import DeduplicationCandidate, DeduplicationStatus	1✔
19
20
21	@click.group()	1✔
22	def data():	1✔
23	"""Migration data commands."""
24
25
26	@data.command()	1✔
27	@click.argument("migration")	1✔
28	@click.argument("infile", type=click.File("r"))	1✔
29	@click.option("-n", "--dry-run", is_flag=True, default=False)	1✔
30	@with_appcontext	1✔
31	def load(migration, infile, dry_run):	1✔
32	"""Load the data for a given migration."""
33	try:	1✔
34	migration = Migration.get(migration)	1✔
35	except NotFoundError:	×
36	click.secho(f"Migration with: {migration} does not exists", fg="yellow")	×
37	raise click.Abort()	×
38	ConvertClass = migration.conversion_class	1✔
39	MigrationData = migration.data_class	1✔
40	with click.progressbar(ConvertClass.loads(infile)) as bar:	1✔
41	for record in bar:	1✔
42	if dry_run:	1✔
43	print(ConvertClass.markdown(record), "\n")	1✔
44	else:
45	migration_data = MigrationData(raw=record, migration_id=migration.meta.id)	1✔
46	migration_data.save()	1✔
47	index = Index(migration.data_index_name)	1✔
48	index.refresh()	1✔
49	n_data = MigrationData.search().filter("term", migration_id=migration.meta.id).count()	1✔
50	click.secho(f"Migration ({migration}) has now {n_data} converted files.")	1✔
51
52
53	@data.command()	1✔
54	@click.argument("migration")	1✔
55	@click.option("-i", "--id")	1✔
56	@click.option("-n", "--dry-run", is_flag=True, default=False)	1✔
57	@click.option("-f", "--force", is_flag=True, default=False)	1✔
58	@with_appcontext	1✔
59	def dedup(migration, id, dry_run, force):	1✔
60	"""Load the data for a given migration."""
61	try:	1✔
62	migration = Migration.get(migration)	1✔
63	except NotFoundError:	×
64	click.secho(f"Migration with: {migration} does not exists", fg="yellow")	×
65	raise click.Abort()	×
66	ConvertClass = migration.conversion_class	1✔
67	MigrationData = migration.data_class	1✔
68	search = MigrationData.search()	1✔
69	if id:	1✔
70	search = search.filter("ids", values=id.split(","))	×
71	with click.progressbar(list(search.scan()), length=search.count()) as bar:	1✔
72	for record in bar:	1✔
73	ils_pid, logs, status, candidates = ConvertClass.dedup(record, force)	1✔
74	if dry_run:	1✔
75	print(	1✔
76	"\n======================================\n",
77	f"ID: {record.meta.id}\n",
78	f"Status: {status}",
79	)
80	for pid, json, score, detailed_score in candidates:	1✔
81	print(f"ILS pid: {pid}", f"{score:.2f}")	×
NEW 82	print(pformat(detailed_score, indent=2))	×
83	else:
84	record.deduplication.candidates = [	1✔
85	DeduplicationCandidate(
86	pid=pid,
87	json=json,
88	score=score,
89	detailed_score=detailed_score,
90	)
91	for pid, json, score, detailed_score in candidates
92	]
93	record.deduplication.logs = logs	1✔
94	record.deduplication.status = status	1✔
95	record.deduplication.ils_pid = ils_pid	1✔
96	record.save()	1✔
97
98
99	@data.command()	1✔
100	@click.argument("migration")	1✔
101	@click.option("-i", "--id")	1✔
102	@click.option("-c", "--conversion-status")	1✔
103	@click.option(	1✔
104	"-f",
105	"--format",
106	type=click.Choice(["raw", "json", "markdown", "ids", "logs", "conversion-status"]),
107	default="json",
108	)
109	@click.option("-o", "--out-file")	1✔
110	@with_appcontext	1✔
111	def get(migration, id, conversion_status, format, out_file):	1✔
112	"""Get migration data."""
113	try:	1✔
114	migration = Migration.get(migration)	1✔
115	except NotFoundError:	×
116	click.secho(f"Migration with: {migration} does not exists", fg="yellow")	×
117	raise click.Abort()	×
118	DataMigration = migration.data_class	1✔
119
120	def format_data(data, format=json):	1✔
121	"""Display the data for a given format."""
122	if format == "json":	1✔
123	return data.conversion.to_dict().get("json")	×
124	if format == "ids":	1✔
125	return data.meta.id	1✔
126	if format == "raw":	×
127	return data.raw.decode()	×
128	if format == "markdown":	×
129	ConversionClass = migration.conversion_class	×
130	return ConversionClass.markdown(data.raw)	×
131	if format == "conversion-status" and hasattr(data.conversion, "status"):	×
132	return (data.meta.id, data.conversion.status)	×
133	return None	×
134
135	if out_file:	1✔
136	out_file = JsonWriter(out_file) if format == "json" else open(out_file, "w")	×
137	search = DataMigration.search()	1✔
138	if id:	1✔
139	search = search.filter("term", _id=id)	×
140	if conversion_status:	1✔
141	search = search.filter("term", conversion__status=conversion_status)	×
142	for data in search.scan():	1✔
143	msg = format_data(data=data, format=format)	1✔
144	if format == "logs":	1✔
145	msg = f"Logs for {data.meta.id}"	×
146	print(msg)	×
147	if out_file:	×
148	out_file.write(f"{msg}\n")	×
149	if hasattr(data.conversion.logs, "info"):	×
150	msg = "\n".join(data.conversion.logs.info)	×
151	click.secho(msg, fg="green")	×
152	if hasattr(data.conversion.logs, "warnings"):	×
153	msg = "\n".join(data.conversion.logs.warnings)	×
154	click.secho(msg, fg="yellow")	×
155	if hasattr(data.conversion.logs, "error"):	×
156	msg = "\n".join(data.conversion.logs.error)	×
157	click.secho(msg, fg="red")	×
158	if out_file:	×
159	out_file.write(f"{msg}\n")	×
160	else:
161	if format == "json":	1✔
162	print(json.dumps(msg, indent=2))	×
163	else:
164	print(msg)	1✔
165	if out_file:	1✔
166	out_file.write(msg)	×
167	if format != "json":	×
168	out_file.write("\n")	×
169	if out_file:	1✔
170	out_file.close()	×
171
172
173	@data.command()	1✔
174	@click.argument("migration")	1✔
175	@click.argument("sets", type=str)	1✔
176	@click.option("-s", "--status", type=click.Choice([item.value for item in DeduplicationStatus]))	1✔
177	@with_appcontext	1✔
178	def subsets(migration, sets, status):	1✔
179	"""Get migration data."""
180	try:	1✔
181	migration = Migration.get(migration)	1✔
182	except NotFoundError:	×
183	click.secho(f"Migration with: {migration} does not exists", fg="yellow")	×
184	raise click.Abort()	×
185	DataMigration = migration.data_class	1✔
186	sets = sets.split(",")	1✔
187
188	search = DataMigration.search()	1✔
189	if status:	1✔
190	search = search.filter("term", deduplication__status=status)	×
191
192	n = 0	1✔
193	with click.progressbar(list(search.scan()), length=search.count()) as bar:	1✔
194	for record in bar:	1✔
195	record.deduplication.subset = choice(sets)	1✔
196	record.save()	1✔
197	n += 1	1✔
198	click.secho(f"{n} records has been assigned to {len(sets)} sets.")	1✔

rero / rero-ils / 30269105825

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous