NaturalHistoryMuseum / data-importer / #30

Committed 02 Sep 2025 11:35AM UTC coverage: 77.202% (-2.0%) from 79.159%

Build # #30

Build Type

push

coveralls-python

Committed by

web-flow

Commit Message

merge: #61 from ginger/purge-non-members

Coverage Stats

20 of 70 new or added lines in 5 files covered. (28.57%)

1297 of 1680 relevant lines covered (77.2%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/dataimporter/cli/shell.py

from functools import partial
from typing import Dict, List, Tuple, Union

from dataimporter.cli.utils import console
from dataimporter.importer import DataImporter


def setup_env(importer: DataImporter) -> dict:
    """
    Returns a dict of variables to made available in the maintenance shell.

    :param importer: a DataImporter instance
    :return: a dict
    """
    return {
        'importer': importer,
        'console': console,
        # some convenience functions for printing data
        'pcd': partial(print_record_data, importer, 'ecatalogue'),
        'ptd': partial(print_record_data, importer, 'etaxonomy'),
        'pmd': partial(print_record_data, importer, 'emultimedia'),
        'pgd': partial(print_record_data, importer, 'gbif'),
        'cm': partial(check_membership, importer),
    }


def print_record_data(importer: DataImporter, name: str, record_id: Union[str, int]):
    """
    A convenience function for printing record data from the given data store.

    :param importer: a DataImporter instance
    :param name: the store to look up the record ID in
    :param record_id: a record ID (can be int or str, we deal with it)
    """
    store = importer.get_store(name)
    if store is None:
        console.print('store not found', style='red')
        return
    record = store.get_record(str(record_id))
    if record is None:
        console.print('record not found', style='red')
        return
    console.print(record.data)


def check_membership(importer: DataImporter, name: str, record_id: Union[str, int]):
    """
    A convenience function which checks if the given record ID is a member of the given
    view. If it isn't, the reason why is printed.

    :param importer: a DataImporter instance
    :param name: name of the view
    :param record_id: record ID (can be int or str, we deal with it)
    """
    view = importer.get_view(name)
    if view is None:
        console.print('view not found', style='red')
        return
    record = view.store.get_record(str(record_id))
    if record is None:
        console.print('record not found', style='red')
        return
    member_result = view.is_member(record)
    publish_result = view.is_publishable(record)
    if member_result and publish_result:
        console.print(f'{record_id} is a currently published member of {name}')
    elif member_result and not publish_result:
        console.print(
            f'{record_id} is a member of {name}, but is not published due to '
            f'{publish_result.reason}'
        )
    else:
        console.print(
            f"{record_id} is not a member of {name} due to '{member_result.reason}'"
        )


def find_unsuitable_records(
    importer: DataImporter, view_name: str, skip_deleted: bool = True
) -> Dict[str, List[Tuple[str, str]]]:
    """
    Finds and returns lists of records in a view that are no longer members or no longer
    match the publishing rules.Effectively a dry run of purge_unsuitable_records.

    :param importer: a DataImporter instance
    :param view_name: name of the view
    :param skip_deleted: whether to skip deleted records when iterating over mongo
        records (True, default), or return a report on these too (False)
    :return: a dict
    """
    non_publishable_records = []
    non_member_records = []
    view = importer.get_view(view_name)
    db = importer.get_database(view_name)
    total = 0
    deleted = 0
    missing_source = 0
    not_member = 0
    not_publishable = 0
    acceptable = 0
    if skip_deleted:
        filter_kwargs = {'filter': {'data._id': {'$exists': True}}}
    else:
        filter_kwargs = {}
    for mongo_record in db.iter_records(**filter_kwargs):
        total += 1
        if mongo_record.is_deleted:
            deleted += 1
            continue
        source_record = view.store.get_record(mongo_record.id)
        if not source_record:
            missing_source += 1
            continue
        is_member = view.is_member(source_record)
        if not is_member:
            not_member += 1
            continue
        is_publishable = view.is_publishable(source_record)
        if not is_publishable:
            not_publishable += 1
            non_publishable_records.append([source_record.id, is_publishable.reason])
            continue
        acceptable += 1
    console.print(
        f'total: {total} | deleted: {deleted} | missing: {missing_source} | not '
        f'members: {not_member} | not publishable: {not_publishable} | acceptable: '
        f'{acceptable}'
    )
    return {
        'not_publishable': non_publishable_records,
        'not_member': non_member_records,
    }

1	from functools import partial	×
NEW 2	from typing import Dict, List, Tuple, Union	×
3
4	from dataimporter.cli.utils import console	×
5	from dataimporter.importer import DataImporter	×
6
7
8	def setup_env(importer: DataImporter) -> dict:	×
9	"""
10	Returns a dict of variables to made available in the maintenance shell.
11
12	:param importer: a DataImporter instance
13	:return: a dict
14	"""
15	return {	×
16	'importer': importer,
17	'console': console,
18	# some convenience functions for printing data
19	'pcd': partial(print_record_data, importer, 'ecatalogue'),
20	'ptd': partial(print_record_data, importer, 'etaxonomy'),
21	'pmd': partial(print_record_data, importer, 'emultimedia'),
22	'pgd': partial(print_record_data, importer, 'gbif'),
23	'cm': partial(check_membership, importer),
24	}
25
26
27	def print_record_data(importer: DataImporter, name: str, record_id: Union[str, int]):	×
28	"""
29	A convenience function for printing record data from the given data store.
30
31	:param importer: a DataImporter instance
32	:param name: the store to look up the record ID in
33	:param record_id: a record ID (can be int or str, we deal with it)
34	"""
35	store = importer.get_store(name)	×
36	if store is None:	×
37	console.print('store not found', style='red')	×
38	return	×
39	record = store.get_record(str(record_id))	×
40	if record is None:	×
41	console.print('record not found', style='red')	×
42	return	×
43	console.print(record.data)	×
44
45
46	def check_membership(importer: DataImporter, name: str, record_id: Union[str, int]):	×
47	"""
48	A convenience function which checks if the given record ID is a member of the given
49	view. If it isn't, the reason why is printed.
50
51	:param importer: a DataImporter instance
52	:param name: name of the view
53	:param record_id: record ID (can be int or str, we deal with it)
54	"""
55	view = importer.get_view(name)	×
56	if view is None:	×
57	console.print('view not found', style='red')	×
58	return	×
59	record = view.store.get_record(str(record_id))	×
60	if record is None:	×
61	console.print('record not found', style='red')	×
62	return	×
63	member_result = view.is_member(record)	×
64	publish_result = view.is_publishable(record)	×
65	if member_result and publish_result:	×
NEW 66	console.print(f'{record_id} is a currently published member of {name}')	×
67	elif member_result and not publish_result:	×
NEW 68	console.print(	×
69	f'{record_id} is a member of {name}, but is not published due to '
70	f'{publish_result.reason}'
71	)
72	else:
NEW 73	console.print(	×
74	f"{record_id} is not a member of {name} due to '{member_result.reason}'"
75	)
76
77
NEW 78	def find_unsuitable_records(	×
79	importer: DataImporter, view_name: str, skip_deleted: bool = True
80	) -> Dict[str, List[Tuple[str, str]]]:
81	"""
82	Finds and returns lists of records in a view that are no longer members or no longer
83	match the publishing rules.Effectively a dry run of purge_unsuitable_records.
84
85	:param importer: a DataImporter instance
86	:param view_name: name of the view
87	:param skip_deleted: whether to skip deleted records when iterating over mongo
88	records (True, default), or return a report on these too (False)
89	:return: a dict
90	"""
NEW 91	non_publishable_records = []	×
NEW 92	non_member_records = []	×
NEW 93	view = importer.get_view(view_name)	×
NEW 94	db = importer.get_database(view_name)	×
NEW 95	total = 0	×
NEW 96	deleted = 0	×
NEW 97	missing_source = 0	×
NEW 98	not_member = 0	×
NEW 99	not_publishable = 0	×
NEW 100	acceptable = 0	×
NEW 101	if skip_deleted:	×
NEW 102	filter_kwargs = {'filter': {'data._id': {'$exists': True}}}	×
103	else:
NEW 104	filter_kwargs = {}	×
NEW 105	for mongo_record in db.iter_records(**filter_kwargs):	×
NEW 106	total += 1	×
NEW 107	if mongo_record.is_deleted:	×
NEW 108	deleted += 1	×
NEW 109	continue	×
NEW 110	source_record = view.store.get_record(mongo_record.id)	×
NEW 111	if not source_record:	×
NEW 112	missing_source += 1	×
NEW 113	continue	×
NEW 114	is_member = view.is_member(source_record)	×
NEW 115	if not is_member:	×
NEW 116	not_member += 1	×
NEW 117	continue	×
NEW 118	is_publishable = view.is_publishable(source_record)	×
NEW 119	if not is_publishable:	×
NEW 120	not_publishable += 1	×
NEW 121	non_publishable_records.append([source_record.id, is_publishable.reason])	×
NEW 122	continue	×
NEW 123	acceptable += 1	×
NEW 124	console.print(	×
125	f'total: {total} \| deleted: {deleted} \| missing: {missing_source} \| not '
126	f'members: {not_member} \| not publishable: {not_publishable} \| acceptable: '
127	f'{acceptable}'
128	)
NEW 129	return {	×
130	'not_publishable': non_publishable_records,
131	'not_member': non_member_records,
132	}

NaturalHistoryMuseum / data-importer / #30

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous