16725814017

Committed 04 Aug 2025 02:09PM UTC coverage: 86.311% (-3.3%) from 89.629%

Build # 16725814017

Build Type

push

github

Committed by

web-flow

Commit Message

Release v1.1.0

Run Details

350 of 449 new or added lines in 22 files covered. (77.95%)

9 existing lines in 2 files now uncovered.

2396 of 2776 relevant lines covered (86.31%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

56.36

/src/IO/sem/InputReader.py

import logging
import mimetypes
import os

from src.IO.MappingAbortionError import MappingAbortionError
from src.parser.ImageParser import ParserMode
from src.parser.ParserFactory import ParserFactory
from src.util import load_json, get_filetype_with_magica, robust_textfile_read


class InputReader:

    mapping = None
    parser_names = None

    def __init__(self, map_path, input_path):
        logging.info("Preparing parsers based on parsing map file and input.")
        self.mapping = load_json(map_path)

        if not os.path.exists(input_path):
            logging.error("Input file {} does not exist. Aborting".format(input_path))
            raise MappingAbortionError("Input file loading failed.")

        self.parser_names = self.get_applicable_parsers(input_path)

        if not self.parser_names:
            logging.error("No applicable parsers found for input {}".format(input_path))
            mimetype_set = list(set([v.expected_input_format() for v in ParserFactory.available_img_parsers.values()]))
            logging.info("Supported mimetypes: {}".format(mimetype_set))
            raise MappingAbortionError("Input file parsing aborted.")
        logging.info("Applicable parsers: {}".format(", ".join(self.parser_names)))


    @staticmethod
    def get_applicable_parsers(input_path, by_extension = False):
        """
        Filters the available image parsers to those applicable to the input file format.
        It tries to determine by extension, but can fallback to using magica.
        :param by_extension: set to True if guessing by extension should be used.
        :param input_path: file path to input
        :return: list of parser names that can handle the provided input format
        """
        applicable_types = [ip.expected_input_format() for ip in ParserFactory.available_img_parsers.values()]

        mt = None
        if by_extension:
            mt, _ = mimetypes.guess_type(input_path)
            logging.debug("Mimetypes file identification result: {}".format(mt))
        if not mt or mt == "application/unknown" or mt not in applicable_types: #fallback, especially if file extension is not available
            #Text files are tricky with magica, so try to read as such first
            mt = get_filetype_with_magica(input_path)
            logging.debug("Magika file identification result: {}".format(mt))
            if mt not in applicable_types:
                try:
                    robust_textfile_read(input_path)
                    mt = "text/plain"
                except:
                    logging.error("Could not determine mimetype for input {}".format(input_path))
                    raise MappingAbortionError

        logging.debug("Determined input type: {}".format(mt))

        available_parsers = []
        for k, p in ParserFactory.available_img_parsers.items():
            expected = p.expected_input_format()
            if expected == mt:
                available_parsers.append(k)
        return available_parsers

    def retrieve_image_info(self, input_path):
        """
        Applies the applicable list of parsers to the provided input. Stops on the first successful parsing result and returns it.
        Usually we do not expect more than one applicable parser to be available. If so, it would be advised to add more checks to keep the list of parsers at len 1.
        :param input_path: path to input file
        :return:
        """
        for parser in self.parser_names:
            logging.debug("Trying to parse image with {}".format(parser))
            imgp = ParserFactory.create_img_parser(parser, mode=ParserMode.SEM)

            result, raw = imgp.parse(input_path, self.mapping)
            if result and result.image_metadata:
                output_dict = result.image_metadata.to_schema_dict()
                return output_dict

1	import logging	1✔
2	import mimetypes	1✔
3	import os	1✔
4
5	from src.IO.MappingAbortionError import MappingAbortionError	1✔
6	from src.parser.ImageParser import ParserMode	1✔
7	from src.parser.ParserFactory import ParserFactory	1✔
8	from src.util import load_json, get_filetype_with_magica, robust_textfile_read	1✔
9
10
11	class InputReader:	1✔
12
13	mapping = None	1✔
14	parser_names = None	1✔
15
16	def __init__(self, map_path, input_path):	1✔
17	logging.info("Preparing parsers based on parsing map file and input.")	×
18	self.mapping = load_json(map_path)	×
19
20	if not os.path.exists(input_path):	×
21	logging.error("Input file {} does not exist. Aborting".format(input_path))	×
22	raise MappingAbortionError("Input file loading failed.")	×
23
24	self.parser_names = self.get_applicable_parsers(input_path)	×
25
26	if not self.parser_names:	×
27	logging.error("No applicable parsers found for input {}".format(input_path))	×
28	mimetype_set = list(set([v.expected_input_format() for v in ParserFactory.available_img_parsers.values()]))	×
29	logging.info("Supported mimetypes: {}".format(mimetype_set))	×
30	raise MappingAbortionError("Input file parsing aborted.")	×
31	logging.info("Applicable parsers: {}".format(", ".join(self.parser_names)))	×
32
33
34	@staticmethod	1✔
35	def get_applicable_parsers(input_path, by_extension = False):	1✔
36	"""
37	Filters the available image parsers to those applicable to the input file format.
38	It tries to determine by extension, but can fallback to using magica.
39	:param by_extension: set to True if guessing by extension should be used.
40	:param input_path: file path to input
41	:return: list of parser names that can handle the provided input format
42	"""
43	applicable_types = [ip.expected_input_format() for ip in ParserFactory.available_img_parsers.values()]	1✔
44
45	mt = None	1✔
46	if by_extension:	1✔
NEW 47	mt, _ = mimetypes.guess_type(input_path)	×
NEW 48	logging.debug("Mimetypes file identification result: {}".format(mt))	×
49	if not mt or mt == "application/unknown" or mt not in applicable_types: #fallback, especially if file extension is not available	1✔
50	#Text files are tricky with magica, so try to read as such first
51	mt = get_filetype_with_magica(input_path)	1✔
52	logging.debug("Magika file identification result: {}".format(mt))	1✔
53	if mt not in applicable_types:	1✔
54	try:	1✔
55	robust_textfile_read(input_path)	1✔
56	mt = "text/plain"	1✔
57	except:	×
58	logging.error("Could not determine mimetype for input {}".format(input_path))	×
59	raise MappingAbortionError	×
60
61	logging.debug("Determined input type: {}".format(mt))	1✔
62
63	available_parsers = []	1✔
64	for k, p in ParserFactory.available_img_parsers.items():	1✔
65	expected = p.expected_input_format()	1✔
66	if expected == mt:	1✔
67	available_parsers.append(k)	1✔
68	return available_parsers	1✔
69
70	def retrieve_image_info(self, input_path):	1✔
71	"""
72	Applies the applicable list of parsers to the provided input. Stops on the first successful parsing result and returns it.
73	Usually we do not expect more than one applicable parser to be available. If so, it would be advised to add more checks to keep the list of parsers at len 1.
74	:param input_path: path to input file
75	:return:
76	"""
77	for parser in self.parser_names:	×
78	logging.debug("Trying to parse image with {}".format(parser))	×
79	imgp = ParserFactory.create_img_parser(parser, mode=ParserMode.SEM)	×
80
81	result, raw = imgp.parse(input_path, self.mapping)	×
82	if result and result.image_metadata:	×
83	output_dict = result.image_metadata.to_schema_dict()	×
84	return output_dict	×

kit-data-manager / tomo_mapper / 16725814017

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous