13682489429

Committed 05 Mar 2025 06:03PM UTC coverage: 33.8%. Remained the same

Build # 13682489429

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #117 from 4dn-dcic/ajs_upd_dependencies

Ajs upd dependencies

Run Details

2 of 2 new or added lines in 1 file covered. (100.0%)

22 existing lines in 14 files now uncovered.

1256 of 3716 relevant lines covered (33.8%)

2.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/scripts/load_items_json.py

#!/usr/bin/env python3

import sys
import argparse
import json
from datetime import datetime
from uuid import uuid4
from dcicutils.ff_utils import (
    get_authentication_with_server,
    post_metadata,
    get_metadata,
    patch_metadata,
)
from functions.script_utils import create_ff_arg_parser, authenticate
''' Will attempt to load data from a file into the database using the load_data endpoint if it can
    or post/patch_metadata if not
    The file can be a simple list of json items in which case you need to specify an item type
    with the --itype option (file created by generate_ontology is like this) or the file can
    specify a dictionary with item types as keys and list of jsons as values.

    If the --as_file option is used the json items must contain a uuid and the endpoint will
    attempt to read the file from the request - no ordering and if there are dependencies to
    linked items those items must either already be loaded or present in the file
    WARNING: currently only works locally or if file is uploaded as part of the app file system
'''

ORDER = [
    'user',
    'award',
    'lab',
    'static_section',
    'higlass_view_config',
    'page',
    'ontology',
    'ontology_term',
    'file_format',
    'badge',
    'organism',
    'genomic_region',
    'gene',
    'bio_feature',
    'target',
    'imaging_path',
    'publication',
    'publication_tracking',
    'document',
    'image',
    'vendor',
    'construct',
    'modification',
    'protocol',
    'sop_map',
    'biosample_cell_culture',
    'individual_human',
    'individual_mouse',
    'individual_fly',
    'individual_chicken',
    'biosource',
    'antibody',
    'enzyme',
    'treatment_rnai',
    'treatment_agent',
    'biosample',
    'quality_metric_fastqc',
    'quality_metric_bamqc',
    'quality_metric_pairsqc',
    'quality_metric_dedupqc_repliseq',
    'quality_metric_chipseq',
    'quality_metric_atacseq',
    'microscope_setting_d1',
    'microscope_setting_d2',
    'microscope_setting_a1',
    'microscope_setting_a2',
    'file_fastq',
    'file_processed',
    'file_reference',
    'file_calibration',
    'file_microscopy',
    'file_set',
    'file_set_calibration',
    'file_set_microscope_qc',
    'file_vistrack',
    'experiment_hi_c',
    'experiment_capture_c',
    'experiment_repliseq',
    'experiment_atacseq',
    'experiment_chiapet',
    'experiment_damid',
    'experiment_seq',
    'experiment_tsaseq',
    'experiment_mic',
    'experiment_set',
    'experiment_set_replicate',
    'data_release_update',
    'software',
    'analysis_step',
    'workflow',
    'workflow_mapping',
    'workflow_run_sbg',
    'workflow_run_awsem'
]


def get_args():  # pragma: no cover
    parser = argparse.ArgumentParser(
        description='Given a file of item jsons try to load into database',
        parents=[create_ff_arg_parser()],
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )

    parser.add_argument('infile',
                        help="the datafile containing json formatted items")
    parser.add_argument('--itypes',
                        nargs='*',
                        help="The item type(s) to load if not specified in the file by store key(s)")
    parser.add_argument('--id-field',
                        help="Field name to used as identifier for items (all item types in file)")
    parser.add_argument('--as-file',
                        default=False,
                        action='store_true',
                        help="Will attempt to load and process the file directly in the request. "
                             "This currently only works locally or if the file has been uploaded to "
                             "the apps file system")
    args = parser.parse_args()
    return args


def patch_jsons(auth, to_patch):
    for item in to_patch:
        uid = item.get('uuid')
        try:
            patch_metadata(item, uid, auth)
        except Exception as e:
            print(e)


def load_json(auth, itype, item_list, chunk_size=50):
    list_length = len(item_list)
    curr_pos = 0
    while curr_pos < list_length:
        slice_for = chunk_size if (chunk_size and chunk_size <= (list_length - curr_pos)) else list_length - curr_pos
        new_end = curr_pos + slice_for
        chunk = item_list[curr_pos: new_end]
        store = {itype: chunk}
        payload = {'store': store, 'overwrite': True}
        if 'localhost' in auth.get('server', ''):
            payload['config_uri'] = 'development.ini'
        try:
            post_metadata(payload, 'load_data', auth)
        except Exception as e:
            print("PROBLEM WITH POST")
            print(e)
        curr_pos = new_end


def load_file(auth, itype, filename):
    payload = {'in_file': filename, 'overwrite': True, 'itype': itype}
    if 'localhost' in auth.get('server', ''):
        payload['config_uri'] = 'development.ini'
    try:
        post_metadata(payload, 'load_data', auth)
    except Exception:
        raise


def get_item(val, auth):
    try:
        return get_metadata(val, auth).get('uuid')
    except:
        return None


def check_for_existing(item, itype, idfields, auth):
    uid = None
    for ifield in idfields:
        id2chk = item.get(ifield)
        if id2chk:
            if ifield == 'aliases':
                for a in id2chk:
                    uid = get_item(a, auth)
                    if uid:
                        return uid
            else:
                chkid = itype + '/' + id2chk
                uid = get_item(chkid, auth)
                if uid:
                    return uid
    return uid


def main():  # pragma: no cover
    start = datetime.now()
    print(str(start))
    args = get_args()
    auth = scu.authenticate(key=args.key, keyfile=args.keyfile, env=args.env)
    print('working on ', auth.get('server'))
    if args.as_file:
        if not args.dbupdate:
            print("DRY RUN - use --dbupdate to update the database")
        else:
            try:
                load_file(auth, args.itype, args.infile)
            except Exception as e:
                print(e)
    else:
        with open(args.infile) as ifile:
            item_store = json.loads(ifile.read())
            if not args.itype:
                if not isinstance(item_store, dict):
                    print("File is not in correct format")
                    sys.exit(1)
            else:
                if not isinstance(item_store, list):
                    print("File is not in correct format")
                    sys.exit(1)
                item_store = {args.itype: item_store}
            for itype, items in sorted(item_store.items(), key=lambda x: ORDER.index(x[0])):
                if not args.dbupdate:
                    print('DRY RUN - would try to load {} {} items'.format(len(items), itype))
                    continue
                if args.id_field:
                    identifiers = [args.id_field]
                else:
                    schema_path = 'profiles/' + itype + '.json'
                    schema_info = get_metadata(schema_path, auth)
                    identifiers = schema_info.get('identifyingProperties')
                # checking to see if an item exists
                # if no can use load_data endpoint
                # if yes do it the old fashioned way
                to_patch = []
                to_post = []
                for item in items:
                    uid = item.get('uuid')
                    if uid:
                        exists = get_item(uid, auth)
                        if exists:  # try a patch
                            to_patch.append(item)
                        else:
                            to_post.append(item)
                    else:
                        uid = check_for_existing(item, itype, identifiers, auth)
                        if uid:  # try a patch
                            item['uuid'] = uid
                            to_patch.append(item)
                        else:
                            uid = str(uuid4())
                            item['uuid'] = uid
                            to_post.append(item)
                if to_post:
                    load_json(auth, itype, to_post, chunk_size=1000)
                if to_patch:
                    patch_jsons(auth, to_patch)
    stop = datetime.now()
    print(str(stop))


if __name__ == '__main__':  # pragma: no cover
    main()

1	#!/usr/bin/env python3
2
3	import sys	×
4	import argparse	×
5	import json	×
6	from datetime import datetime	×
7	from uuid import uuid4	×
8	from dcicutils.ff_utils import (	×
9	get_authentication_with_server,
10	post_metadata,
11	get_metadata,
12	patch_metadata,
13	)
14	from functions.script_utils import create_ff_arg_parser, authenticate	×
UNCOV 15	''' Will attempt to load data from a file into the database using the load_data endpoint if it can
16	or post/patch_metadata if not
17	The file can be a simple list of json items in which case you need to specify an item type
18	with the --itype option (file created by generate_ontology is like this) or the file can
19	specify a dictionary with item types as keys and list of jsons as values.
20
21	If the --as_file option is used the json items must contain a uuid and the endpoint will
22	attempt to read the file from the request - no ordering and if there are dependencies to
23	linked items those items must either already be loaded or present in the file
24	WARNING: currently only works locally or if file is uploaded as part of the app file system
25	'''
26
27	ORDER = [	×
28	'user',
29	'award',
30	'lab',
31	'static_section',
32	'higlass_view_config',
33	'page',
34	'ontology',
35	'ontology_term',
36	'file_format',
37	'badge',
38	'organism',
39	'genomic_region',
40	'gene',
41	'bio_feature',
42	'target',
43	'imaging_path',
44	'publication',
45	'publication_tracking',
46	'document',
47	'image',
48	'vendor',
49	'construct',
50	'modification',
51	'protocol',
52	'sop_map',
53	'biosample_cell_culture',
54	'individual_human',
55	'individual_mouse',
56	'individual_fly',
57	'individual_chicken',
58	'biosource',
59	'antibody',
60	'enzyme',
61	'treatment_rnai',
62	'treatment_agent',
63	'biosample',
64	'quality_metric_fastqc',
65	'quality_metric_bamqc',
66	'quality_metric_pairsqc',
67	'quality_metric_dedupqc_repliseq',
68	'quality_metric_chipseq',
69	'quality_metric_atacseq',
70	'microscope_setting_d1',
71	'microscope_setting_d2',
72	'microscope_setting_a1',
73	'microscope_setting_a2',
74	'file_fastq',
75	'file_processed',
76	'file_reference',
77	'file_calibration',
78	'file_microscopy',
79	'file_set',
80	'file_set_calibration',
81	'file_set_microscope_qc',
82	'file_vistrack',
83	'experiment_hi_c',
84	'experiment_capture_c',
85	'experiment_repliseq',
86	'experiment_atacseq',
87	'experiment_chiapet',
88	'experiment_damid',
89	'experiment_seq',
90	'experiment_tsaseq',
91	'experiment_mic',
92	'experiment_set',
93	'experiment_set_replicate',
94	'data_release_update',
95	'software',
96	'analysis_step',
97	'workflow',
98	'workflow_mapping',
99	'workflow_run_sbg',
100	'workflow_run_awsem'
101	]
102
103
104	def get_args(): # pragma: no cover
105	parser = argparse.ArgumentParser(
106	description='Given a file of item jsons try to load into database',
107	parents=[create_ff_arg_parser()],
108	formatter_class=argparse.RawDescriptionHelpFormatter,
109	)
110
111	parser.add_argument('infile',
112	help="the datafile containing json formatted items")
113	parser.add_argument('--itypes',
114	nargs='*',
115	help="The item type(s) to load if not specified in the file by store key(s)")
116	parser.add_argument('--id-field',
117	help="Field name to used as identifier for items (all item types in file)")
118	parser.add_argument('--as-file',
119	default=False,
120	action='store_true',
121	help="Will attempt to load and process the file directly in the request. "
122	"This currently only works locally or if the file has been uploaded to "
123	"the apps file system")
124	args = parser.parse_args()
125	return args
126
127
128	def patch_jsons(auth, to_patch):	×
129	for item in to_patch:	×
130	uid = item.get('uuid')	×
131	try:	×
132	patch_metadata(item, uid, auth)	×
133	except Exception as e:	×
134	print(e)	×
135
136
137	def load_json(auth, itype, item_list, chunk_size=50):	×
138	list_length = len(item_list)	×
139	curr_pos = 0	×
140	while curr_pos < list_length:	×
141	slice_for = chunk_size if (chunk_size and chunk_size <= (list_length - curr_pos)) else list_length - curr_pos	×
142	new_end = curr_pos + slice_for	×
143	chunk = item_list[curr_pos: new_end]	×
144	store = {itype: chunk}	×
145	payload = {'store': store, 'overwrite': True}	×
146	if 'localhost' in auth.get('server', ''):	×
147	payload['config_uri'] = 'development.ini'	×
148	try:	×
149	post_metadata(payload, 'load_data', auth)	×
150	except Exception as e:	×
151	print("PROBLEM WITH POST")	×
152	print(e)	×
153	curr_pos = new_end	×
154
155
156	def load_file(auth, itype, filename):	×
157	payload = {'in_file': filename, 'overwrite': True, 'itype': itype}	×
158	if 'localhost' in auth.get('server', ''):	×
159	payload['config_uri'] = 'development.ini'	×
160	try:	×
161	post_metadata(payload, 'load_data', auth)	×
162	except Exception:	×
163	raise	×
164
165
166	def get_item(val, auth):	×
167	try:	×
168	return get_metadata(val, auth).get('uuid')	×
169	except:	×
170	return None	×
171
172
173	def check_for_existing(item, itype, idfields, auth):	×
174	uid = None	×
175	for ifield in idfields:	×
176	id2chk = item.get(ifield)	×
177	if id2chk:	×
178	if ifield == 'aliases':	×
179	for a in id2chk:	×
180	uid = get_item(a, auth)	×
181	if uid:	×
182	return uid	×
183	else:
184	chkid = itype + '/' + id2chk	×
185	uid = get_item(chkid, auth)	×
186	if uid:	×
187	return uid	×
188	return uid	×
189
190
191	def main(): # pragma: no cover
192	start = datetime.now()
193	print(str(start))
194	args = get_args()
195	auth = scu.authenticate(key=args.key, keyfile=args.keyfile, env=args.env)
196	print('working on ', auth.get('server'))
197	if args.as_file:
198	if not args.dbupdate:
199	print("DRY RUN - use --dbupdate to update the database")
200	else:
201	try:
202	load_file(auth, args.itype, args.infile)
203	except Exception as e:
204	print(e)
205	else:
206	with open(args.infile) as ifile:
207	item_store = json.loads(ifile.read())
208	if not args.itype:
209	if not isinstance(item_store, dict):
210	print("File is not in correct format")
211	sys.exit(1)
212	else:
213	if not isinstance(item_store, list):
214	print("File is not in correct format")
215	sys.exit(1)
216	item_store = {args.itype: item_store}
217	for itype, items in sorted(item_store.items(), key=lambda x: ORDER.index(x[0])):
218	if not args.dbupdate:
219	print('DRY RUN - would try to load {} {} items'.format(len(items), itype))
220	continue
221	if args.id_field:
222	identifiers = [args.id_field]
223	else:
224	schema_path = 'profiles/' + itype + '.json'
225	schema_info = get_metadata(schema_path, auth)
226	identifiers = schema_info.get('identifyingProperties')
227	# checking to see if an item exists
228	# if no can use load_data endpoint
229	# if yes do it the old fashioned way
230	to_patch = []
231	to_post = []
232	for item in items:
233	uid = item.get('uuid')
234	if uid:
235	exists = get_item(uid, auth)
236	if exists: # try a patch
237	to_patch.append(item)
238	else:
239	to_post.append(item)
240	else:
241	uid = check_for_existing(item, itype, identifiers, auth)
242	if uid: # try a patch
243	item['uuid'] = uid
244	to_patch.append(item)
245	else:
246	uid = str(uuid4())
247	item['uuid'] = uid
248	to_post.append(item)
249	if to_post:
250	load_json(auth, itype, to_post, chunk_size=1000)
251	if to_patch:
252	patch_jsons(auth, to_patch)
253	stop = datetime.now()
254	print(str(stop))
255
256
257	if __name__ == '__main__': # pragma: no cover
258	main()

4dn-dcic / dcicwrangling / 13682489429

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous