25816582271

Committed 13 May 2026 05:49PM UTC coverage: 64.619% (+5.5%) from 59.164%

Build # 25816582271

Build Type

Pull #594

github

Committed by

web-flow

Commit Message

Merge 5622f561a into 3215372af

Pull Request Pull Request #594: chore(deps): bump pandas to 3.0

Coverage Stats

1374 of 2268 branches covered (60.58%)

Branch coverage included in aggregate %.

8611 of 13184 relevant lines covered (65.31%)

0.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.59

/fuji_server/controllers/fair_object_controller.py

# SPDX-FileCopyrightText: 2020 PANGAEA (https://www.pangaea.de/)
#
# SPDX-License-Identifier: MIT

import datetime

import connexion

from fuji_server.controllers.fair_check import FAIRCheck
from fuji_server.helper.identifier_helper import IdentifierHelper
from fuji_server.helper.preprocessor import Preprocessor
from fuji_server.helper.results_exporter import FAIRResultsMapper
from fuji_server.models.fair_results import FAIRResults
from fuji_server.models.harvest_results_metadata import HarvestResultsMetadata


async def assess_by_id(body):
    """assess_by_id

    Evaluate FAIRness of a data object based on its identifier # noqa: E501

    :param body:
    :type body: dict | bytes

    :rtype: FAIRResults
    """

    allow_remote_logging = False
    # Request POST BODY has to be JSON
    if connexion.request.content_type == "application/json":
        # The client has to send this HTTP header (Allow-Remote-Logging:True) explicitely to enable remote logging
        # Useful for e.g. web clients..
        allow_remote_logging = connexion.request.headers.get("Allow-Remote-Logging")
        debug = True
        results = []
        # json_body = await connexion.request.json()
        # body = Body.from_dict(json_body)
        # clienturi = Body.from_dict(connexion.request
        identifier = body.get("object_identifier")
        debug = body.get("test_debug")
        metadata_service_endpoint = body.get("metadata_service_endpoint")
        oaipmh_endpoint = body.get("oaipmh_endpoint")
        metadata_service_type = body.get("metadata_service_type")
        usedatacite = body.get("use_datacite")
        usegithub = body.get("use_github")
        metric_version = body.get("metric_version")
        print("BODY METRIC", metric_version)
        auth_token = body.get("auth_token")
        auth_token_type = body.get("auth_token_type")
        logger = Preprocessor.logger
        # updating re3data
        Preprocessor.retrieve_datacite_re3repos()

        logger.info("Assessment target: " + identifier)
        print("Assessment target: ", identifier, flush=True)
        starttimestmp = datetime.datetime.now().replace(microsecond=0).isoformat() + "Z"
        ft = FAIRCheck(
            uid=identifier,
            test_debug=debug,
            metadata_service_url=metadata_service_endpoint,
            metadata_service_type=metadata_service_type,
            use_datacite=usedatacite,
            use_github=usegithub,
            oaipmh_endpoint=oaipmh_endpoint,
            metric_version=metric_version,
        )
        # dataset level authentication
        if auth_token:
            ft.set_auth_token(auth_token, auth_token_type)
        # set target for remote logging
        remote_log_host, remote_log_path = Preprocessor.remote_log_host, Preprocessor.remote_log_path
        # print(remote_log_host, remote_log_path)
        if remote_log_host and remote_log_path and allow_remote_logging:
            print("Remote logging enabled...")
            if ft.weblogger:
                ft.logger.addHandler(ft.weblogger)
        else:
            print("Remote logging disabled...")
            if ft.weblogger:
                ft.logger.removeHandler(ft.weblogger)
        print("F-UJI Version: ", ft.FUJI_VERSION)
        print("starting harvesting ")
        ft.harvest_all_metadata()
        ft.set_harvested_metadata()
        if ft.repeat_pid_check:
            ft.retrieve_metadata_external(ft.pid_url, repeat_mode=True)
            ft.set_harvested_metadata()
            ft.clean_metadata()
        ft.harvest_re3_data()
        ft.harvest_github()
        core_metadata_result = ft.check_minimal_metatadata()
        # print(ft.metadata_unmerged)
        # print('F-UJI checks: access level')
        access_level_result = ft.check_data_access_level()
        # print('F-UJI checks: license')
        license_result = ft.check_license()
        license_file_result = ft.check_license_file()
        # print('F-UJI checks: related')
        related_resources_result = ft.check_relatedresources()
        # print('F-UJI checks: searchable')
        check_searchable_result = ft.check_searchable()
        # print('F-UJI checks: data content')
        ft.harvest_all_data()
        uid_result, pid_result = ft.check_unique_persistent_metadata_identifier()
        # uid_data_result = ft.check_unique_content_identifier()
        # pid_data_result = ft.check_persistent_data_identifier()
        content_identifier_included_result = ft.check_data_identifier_included_in_metadata()
        retrievable_result = ft.check_metadata_data_retrievable()
        data_identifier_included_result = ft.check_data_content_metadata()

        metadata_identifier_included_result = ft.check_metadata_identifier_included_in_metadata()
        data_file_format_result = ft.check_data_file_format()
        upid_software_result = ft.check_unique_persistent_software_identifier()
        software_component_result = ft.check_software_component_identifier()
        version_identifier_result = ft.check_version_identifier()
        development_metadata_result = ft.check_development_metadata()
        open_api_result = ft.check_open_api()
        requirements_result = ft.check_requirements()
        test_cases_result = ft.check_test_cases()
        # print('F-UJI checks: data file format')
        community_standards_result = ft.check_community_metadatastandards()
        data_provenance_result = ft.check_data_provenance()
        code_provenance_result = ft.check_code_provenance()
        formal_metadata_result = ft.check_formal_metadata()
        # print('F-UJI checks: semantic vocab')
        semantic_vocab_result = ft.check_semantic_vocabulary()
        metadata_preserved_result = ft.check_metadata_preservation()
        standard_protocol_data_result = ft.check_standardised_protocol_data()
        standard_protocol_metadata_result = ft.check_standardised_protocol_metadata()
        standard_protocol_auth_result = ft.check_standardised_protocol_authentication()
        if uid_result:
            results.append(uid_result)
        if pid_result:
            results.append(pid_result)
        # if uid_data_result:
        #    results.append(uid_data_result)
        # if pid_data_result:
        #    results.append(pid_data_result)
        if retrievable_result:
            results.append(retrievable_result)
        if upid_software_result:
            results.append(upid_software_result)
        if software_component_result:
            results.append(software_component_result)
        if version_identifier_result:
            results.append(version_identifier_result)
        if development_metadata_result:
            results.append(development_metadata_result)
        if open_api_result:
            results.append(open_api_result)
        if requirements_result:
            results.append(requirements_result)
        if test_cases_result:
            results.append(test_cases_result)
        if core_metadata_result:
            results.append(core_metadata_result)
        if content_identifier_included_result:
            results.append(content_identifier_included_result)
        if check_searchable_result:
            results.append(check_searchable_result)
        if formal_metadata_result:
            results.append(formal_metadata_result)
        if semantic_vocab_result:
            results.append(semantic_vocab_result)
        if related_resources_result:
            results.append(related_resources_result)
        if data_identifier_included_result:
            results.append(data_identifier_included_result)
        if metadata_identifier_included_result:
            results.append(metadata_identifier_included_result)
        if license_result:
            results.append(license_result)
        if license_file_result:
            results.append(license_file_result)
        if access_level_result:
            results.append(access_level_result)
        if data_provenance_result:
            results.append(data_provenance_result)
        if code_provenance_result:
            results.append(code_provenance_result)
        if community_standards_result:
            results.append(community_standards_result)
        if data_file_format_result:
            results.append(data_file_format_result)
        if standard_protocol_data_result:
            results.append(standard_protocol_data_result)
        if standard_protocol_metadata_result:
            results.append(standard_protocol_metadata_result)
        if standard_protocol_auth_result:
            results.append(standard_protocol_auth_result)
        if metadata_preserved_result:
            results.append(metadata_preserved_result)
        debug_messages = ft.get_log_messages_dict()
        # ft.logger_message_stream.flush()
        summary = ft.get_assessment_summary(results)
        for res_k, res_v in enumerate(results):
            if ft.isDebug:
                debug_list = debug_messages.get(res_v["metric_identifier"])
                # debug_list= ft.msg_filter.getMessage(res_v['metric_identifier'])
                if debug_list is not None:
                    results[res_k]["test_debug"] = debug_messages.get(res_v["metric_identifier"])
                else:
                    results[res_k]["test_debug"] = ["INFO: No debug messages received"]
            else:
                results[res_k]["test_debug"] = ["INFO: Debugging disabled"]
                debug_messages = {}
        if len(ft.logger.handlers) > 1:
            ft.logger.handlers = [ft.logger.handlers[-1]]
        # endtimestmp = datetime.datetime.now().replace(microsecond=0).isoformat()
        endtimestmp = (
            datetime.datetime.now().replace(microsecond=0).isoformat() + "Z"
        )  # use timestamp format from RFC 3339 as specified in openapi3
        metric_spec = ft.metric_helper.metric_specification
        metric_version = ft.metric_helper.metric_version
        resolved_url = ft.landing_url
        if not resolved_url:
            resolved_url = "not defined"
        # metric_version = os.path.basename(Preprocessor.METRIC_YML_PATH)
        totalmetrics = len(results)
        request = body
        if ft.pid_url:
            idhelper = IdentifierHelper(ft.pid_url)
            request["normalized_object_identifier"] = idhelper.get_normalized_id()
        results.sort(key=lambda d: d["id"])  # sort results by metric ID
        #### metadata summary
        harvest_result = []
        for metadata in ft.metadata_unmerged:
            harvest_result.append(
                HarvestResultsMetadata(
                    metadata.get("offering_method"),
                    metadata.get("url"),
                    metadata.get("format"),
                    metadata.get("schema"),
                    metadata.get("namespaces"),
                    metadata.get("metadata"),
                )
            )
        ###
        final_response = FAIRResults(
            request=request,
            start_timestamp=starttimestmp,
            end_timestamp=endtimestmp,
            software_version=ft.FUJI_VERSION,
            test_id=ft.test_id,
            metric_version=metric_version,
            metric_specification=metric_spec,
            total_metrics=totalmetrics,
            results=results,
            summary=summary,
            resolved_url=resolved_url,
            harvested_metadata=harvest_result,
        )
        accept_header = connexion.request.headers.get("Accept")
        print("ACCEPT HEADER ", accept_header)
        # RDF
        rdf_mimes = FAIRResultsMapper.allowed_serialisations
        if connexion.request.headers.get("Accept") in rdf_mimes:
            rdf_mapper = FAIRResultsMapper(final_response)
            rdf = rdf_mapper.getQualityVocabularyRDF(connexion.request.headers.get("Accept"))
            print("RDF")
            return rdf, 200, {"content-type": connexion.request.headers.get("Accept")}
        # Standard JSON
        elif connexion.request.headers.get("Accept") in ["application/json", "*/*"]:
            return final_response, 200, {"content-type": "application/json"}
        else:
            return "", 400, {"content-type": "application/json"}
    else:
        return "", 400, {"content-type": "application/json"}

1	# SPDX-FileCopyrightText: 2020 PANGAEA (https://www.pangaea.de/)
2	#
3	# SPDX-License-Identifier: MIT
4
5	import datetime	1✔
6
7	import connexion	1✔
8
9	from fuji_server.controllers.fair_check import FAIRCheck	1✔
10	from fuji_server.helper.identifier_helper import IdentifierHelper	1✔
11	from fuji_server.helper.preprocessor import Preprocessor	1✔
12	from fuji_server.helper.results_exporter import FAIRResultsMapper	1✔
13	from fuji_server.models.fair_results import FAIRResults	1✔
14	from fuji_server.models.harvest_results_metadata import HarvestResultsMetadata	1✔
15
16
17	async def assess_by_id(body):	1✔
18	"""assess_by_id
19
20	Evaluate FAIRness of a data object based on its identifier # noqa: E501
21
22	:param body:
23	:type body: dict \| bytes
24
25	:rtype: FAIRResults
26	"""
27
28	allow_remote_logging = False	1✔
29	# Request POST BODY has to be JSON
30	if connexion.request.content_type == "application/json":	1✔
31	# The client has to send this HTTP header (Allow-Remote-Logging:True) explicitely to enable remote logging
32	# Useful for e.g. web clients..
33	allow_remote_logging = connexion.request.headers.get("Allow-Remote-Logging")	1✔
34	debug = True	1✔
35	results = []	1✔
36	# json_body = await connexion.request.json()
37	# body = Body.from_dict(json_body)
38	# clienturi = Body.from_dict(connexion.request
39	identifier = body.get("object_identifier")	1✔
40	debug = body.get("test_debug")	1✔
41	metadata_service_endpoint = body.get("metadata_service_endpoint")	1✔
42	oaipmh_endpoint = body.get("oaipmh_endpoint")	1✔
43	metadata_service_type = body.get("metadata_service_type")	1✔
44	usedatacite = body.get("use_datacite")	1✔
45	usegithub = body.get("use_github")	1✔
46	metric_version = body.get("metric_version")	1✔
47	print("BODY METRIC", metric_version)	1✔
48	auth_token = body.get("auth_token")	1✔
49	auth_token_type = body.get("auth_token_type")	1✔
50	logger = Preprocessor.logger	1✔
51	# updating re3data
52	Preprocessor.retrieve_datacite_re3repos()	1✔
53
54	logger.info("Assessment target: " + identifier)	1✔
55	print("Assessment target: ", identifier, flush=True)	1✔
56	starttimestmp = datetime.datetime.now().replace(microsecond=0).isoformat() + "Z"	1✔
57	ft = FAIRCheck(	1✔
58	uid=identifier,
59	test_debug=debug,
60	metadata_service_url=metadata_service_endpoint,
61	metadata_service_type=metadata_service_type,
62	use_datacite=usedatacite,
63	use_github=usegithub,
64	oaipmh_endpoint=oaipmh_endpoint,
65	metric_version=metric_version,
66	)
67	# dataset level authentication
68	if auth_token:	1✔
69	ft.set_auth_token(auth_token, auth_token_type)	×
70	# set target for remote logging
71	remote_log_host, remote_log_path = Preprocessor.remote_log_host, Preprocessor.remote_log_path	1✔
72	# print(remote_log_host, remote_log_path)
73	if remote_log_host and remote_log_path and allow_remote_logging:	1✔
74	print("Remote logging enabled...")	×
75	if ft.weblogger:	×
76	ft.logger.addHandler(ft.weblogger)	×
77	else:
78	print("Remote logging disabled...")	1✔
79	if ft.weblogger:	1✔
80	ft.logger.removeHandler(ft.weblogger)	×
81	print("F-UJI Version: ", ft.FUJI_VERSION)	1✔
82	print("starting harvesting ")	1✔
83	ft.harvest_all_metadata()	1✔
84	ft.set_harvested_metadata()	1✔
85	if ft.repeat_pid_check:	1✔
86	ft.retrieve_metadata_external(ft.pid_url, repeat_mode=True)	×
87	ft.set_harvested_metadata()	×
88	ft.clean_metadata()	×
89	ft.harvest_re3_data()	1✔
90	ft.harvest_github()	1✔
91	core_metadata_result = ft.check_minimal_metatadata()	1✔
92	# print(ft.metadata_unmerged)
93	# print('F-UJI checks: access level')
94	access_level_result = ft.check_data_access_level()	1✔
95	# print('F-UJI checks: license')
96	license_result = ft.check_license()	1✔
97	license_file_result = ft.check_license_file()	1✔
98	# print('F-UJI checks: related')
99	related_resources_result = ft.check_relatedresources()	1✔
100	# print('F-UJI checks: searchable')
101	check_searchable_result = ft.check_searchable()	1✔
102	# print('F-UJI checks: data content')
103	ft.harvest_all_data()	1✔
104	uid_result, pid_result = ft.check_unique_persistent_metadata_identifier()	1✔
105	# uid_data_result = ft.check_unique_content_identifier()
106	# pid_data_result = ft.check_persistent_data_identifier()
107	content_identifier_included_result = ft.check_data_identifier_included_in_metadata()	1✔
108	retrievable_result = ft.check_metadata_data_retrievable()	1✔
109	data_identifier_included_result = ft.check_data_content_metadata()	1✔
110
111	metadata_identifier_included_result = ft.check_metadata_identifier_included_in_metadata()	1✔
112	data_file_format_result = ft.check_data_file_format()	1✔
113	upid_software_result = ft.check_unique_persistent_software_identifier()	1✔
114	software_component_result = ft.check_software_component_identifier()	1✔
115	version_identifier_result = ft.check_version_identifier()	1✔
116	development_metadata_result = ft.check_development_metadata()	1✔
117	open_api_result = ft.check_open_api()	1✔
118	requirements_result = ft.check_requirements()	1✔
119	test_cases_result = ft.check_test_cases()	1✔
120	# print('F-UJI checks: data file format')
121	community_standards_result = ft.check_community_metadatastandards()	1✔
122	data_provenance_result = ft.check_data_provenance()	1✔
123	code_provenance_result = ft.check_code_provenance()	1✔
124	formal_metadata_result = ft.check_formal_metadata()	1✔
125	# print('F-UJI checks: semantic vocab')
126	semantic_vocab_result = ft.check_semantic_vocabulary()	1✔
127	metadata_preserved_result = ft.check_metadata_preservation()	1✔
128	standard_protocol_data_result = ft.check_standardised_protocol_data()	1✔
129	standard_protocol_metadata_result = ft.check_standardised_protocol_metadata()	1✔
130	standard_protocol_auth_result = ft.check_standardised_protocol_authentication()	1✔
131	if uid_result:	1✔
132	results.append(uid_result)	1✔
133	if pid_result:	1✔
134	results.append(pid_result)	1✔
135	# if uid_data_result:
136	# results.append(uid_data_result)
137	# if pid_data_result:
138	# results.append(pid_data_result)
139	if retrievable_result:	1✔
140	results.append(retrievable_result)	1✔
141	if upid_software_result:	1✔
142	results.append(upid_software_result)	×
143	if software_component_result:	1✔
144	results.append(software_component_result)	×
145	if version_identifier_result:	1✔
146	results.append(version_identifier_result)	×
147	if development_metadata_result:	1✔
148	results.append(development_metadata_result)	×
149	if open_api_result:	1✔
150	results.append(open_api_result)	×
151	if requirements_result:	1✔
152	results.append(requirements_result)	×
153	if test_cases_result:	1✔
154	results.append(test_cases_result)	×
155	if core_metadata_result:	1✔
156	results.append(core_metadata_result)	1✔
157	if content_identifier_included_result:	1✔
158	results.append(content_identifier_included_result)	1✔
159	if check_searchable_result:	1✔
160	results.append(check_searchable_result)	1✔
161	if formal_metadata_result:	1✔
162	results.append(formal_metadata_result)	1✔
163	if semantic_vocab_result:	1✔
164	results.append(semantic_vocab_result)	1✔
165	if related_resources_result:	1✔
166	results.append(related_resources_result)	1✔
167	if data_identifier_included_result:	1✔
168	results.append(data_identifier_included_result)	1✔
169	if metadata_identifier_included_result:	1✔
170	results.append(metadata_identifier_included_result)	×
171	if license_result:	1✔
172	results.append(license_result)	1✔
173	if license_file_result:	1✔
174	results.append(license_file_result)	×
175	if access_level_result:	1✔
176	results.append(access_level_result)	1✔
177	if data_provenance_result:	1✔
178	results.append(data_provenance_result)	1✔
179	if code_provenance_result:	1✔
180	results.append(code_provenance_result)	×
181	if community_standards_result:	1✔
182	results.append(community_standards_result)	1✔
183	if data_file_format_result:	1✔
184	results.append(data_file_format_result)	1✔
185	if standard_protocol_data_result:	1✔
186	results.append(standard_protocol_data_result)	×
187	if standard_protocol_metadata_result:	1✔
188	results.append(standard_protocol_metadata_result)	1✔
189	if standard_protocol_auth_result:	1✔
190	results.append(standard_protocol_auth_result)	1✔
191	if metadata_preserved_result:	1✔
192	results.append(metadata_preserved_result)	×
193	debug_messages = ft.get_log_messages_dict()	1✔
194	# ft.logger_message_stream.flush()
195	summary = ft.get_assessment_summary(results)	1✔
196	for res_k, res_v in enumerate(results):	1✔
197	if ft.isDebug:	1✔
198	debug_list = debug_messages.get(res_v["metric_identifier"])	1✔
199	# debug_list= ft.msg_filter.getMessage(res_v['metric_identifier'])
200	if debug_list is not None:	1✔
201	results[res_k]["test_debug"] = debug_messages.get(res_v["metric_identifier"])	1✔
202	else:
203	results[res_k]["test_debug"] = ["INFO: No debug messages received"]	×
204	else:
205	results[res_k]["test_debug"] = ["INFO: Debugging disabled"]	×
206	debug_messages = {}	×
207	if len(ft.logger.handlers) > 1:	1✔
208	ft.logger.handlers = [ft.logger.handlers[-1]]	×
209	# endtimestmp = datetime.datetime.now().replace(microsecond=0).isoformat()
210	endtimestmp = (	1✔
211	datetime.datetime.now().replace(microsecond=0).isoformat() + "Z"
212	) # use timestamp format from RFC 3339 as specified in openapi3
213	metric_spec = ft.metric_helper.metric_specification	1✔
214	metric_version = ft.metric_helper.metric_version	1✔
215	resolved_url = ft.landing_url	1✔
216	if not resolved_url:	1✔
217	resolved_url = "not defined"	×
218	# metric_version = os.path.basename(Preprocessor.METRIC_YML_PATH)
219	totalmetrics = len(results)	1✔
220	request = body	1✔
221	if ft.pid_url:	1✔
222	idhelper = IdentifierHelper(ft.pid_url)	1✔
223	request["normalized_object_identifier"] = idhelper.get_normalized_id()	1✔
224	results.sort(key=lambda d: d["id"]) # sort results by metric ID	1✔
225	#### metadata summary
226	harvest_result = []	1✔
227	for metadata in ft.metadata_unmerged:	1✔
228	harvest_result.append(	1✔
229	HarvestResultsMetadata(
230	metadata.get("offering_method"),
231	metadata.get("url"),
232	metadata.get("format"),
233	metadata.get("schema"),
234	metadata.get("namespaces"),
235	metadata.get("metadata"),
236	)
237	)
238	###
239	final_response = FAIRResults(	1✔
240	request=request,
241	start_timestamp=starttimestmp,
242	end_timestamp=endtimestmp,
243	software_version=ft.FUJI_VERSION,
244	test_id=ft.test_id,
245	metric_version=metric_version,
246	metric_specification=metric_spec,
247	total_metrics=totalmetrics,
248	results=results,
249	summary=summary,
250	resolved_url=resolved_url,
251	harvested_metadata=harvest_result,
252	)
253	accept_header = connexion.request.headers.get("Accept")	1✔
254	print("ACCEPT HEADER ", accept_header)	1✔
255	# RDF
256	rdf_mimes = FAIRResultsMapper.allowed_serialisations	1✔
257	if connexion.request.headers.get("Accept") in rdf_mimes:	1✔
258	rdf_mapper = FAIRResultsMapper(final_response)	×
259	rdf = rdf_mapper.getQualityVocabularyRDF(connexion.request.headers.get("Accept"))	×
260	print("RDF")	×
261	return rdf, 200, {"content-type": connexion.request.headers.get("Accept")}	×
262	# Standard JSON
263	elif connexion.request.headers.get("Accept") in ["application/json", "/"]:	1✔
264	return final_response, 200, {"content-type": "application/json"}	1✔
265	else:
266	return "", 400, {"content-type": "application/json"}	×
267	else:
268	return "", 400, {"content-type": "application/json"}	×

pangaea-data-publisher / fuji / 25816582271

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous