8362804128

Committed 20 Mar 2024 04:52PM UTC coverage: 86.434% (-0.03%) from 86.459%

Build # 8362804128

Build Type

Pull #743

github

Committed by

phycodurus

Commit Message

add comments and documentation for improved developer ux

Pull Request Pull Request #743: 715 pan starrs forced photometry

Run Details

7 of 11 new or added lines in 3 files covered. (63.64%)

4 existing lines in 1 file now uncovered.

8678 of 10040 relevant lines covered (86.43%)

0.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.85

/tom_dataproducts/data_processor.py

import mimetypes

from django.conf import settings
from importlib import import_module

from tom_dataproducts.models import ReducedDatum


DEFAULT_DATA_PROCESSOR_CLASS = 'tom_dataproducts.data_processor.DataProcessor'


def run_data_processor(dp):
    """
    Reads the `data_product_type` from the dp parameter and imports the corresponding `DATA_PROCESSORS` specified in
    `settings.py`, then runs `process_data` and inserts the returned values into the database.

    :param dp: DataProduct which will be processed into a list
    :type dp: DataProduct

    :returns: QuerySet of `ReducedDatum` objects created by the `run_data_processor` call
    :rtype: `QuerySet` of `ReducedDatum`
    """

    try:
        processor_class = settings.DATA_PROCESSORS[dp.data_product_type]
    except Exception:
        processor_class = DEFAULT_DATA_PROCESSOR_CLASS

    try:
        mod_name, class_name = processor_class.rsplit('.', 1)
        mod = import_module(mod_name)
        clazz = getattr(mod, class_name)
    except (ImportError, AttributeError):
        raise ImportError('Could not import {}. Did you provide the correct path?'.format(processor_class))

    data_processor = clazz()
    data = data_processor.process_data(dp)
    data_type = data_processor.data_type_override() or dp.data_product_type

    reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=data_type,
                                   timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data]
    ReducedDatum.objects.bulk_create(reduced_datums)

    return ReducedDatum.objects.filter(data_product=dp)


class DataProcessor():

    FITS_MIMETYPES = ['image/fits', 'application/fits']
    PLAINTEXT_MIMETYPES = ['text/plain', 'text/csv']

    mimetypes.add_type('image/fits', '.fits')
    mimetypes.add_type('image/fits', '.fz')
    mimetypes.add_type('application/fits', '.fits')
    mimetypes.add_type('application/fits', '.fz')

    def process_data(self, data_product):
        """
        Routes a photometry processing call to a method specific to a file-format. This method is expected to be
        implemented by any subclasses.

        :param data_product: DataProduct which will be processed into a list
        :type data_product: DataProduct

        :returns: python list of 2-tuples, each with a timestamp and corresponding data
        :rtype: list of 2-tuples
        """
        return []

    def data_type_override(self):
        """
        Override for the ReducedDatum data type, if you want it to be different from the
        DataProduct data_type.
        """
        return ''

1	import mimetypes	1✔
2
3	from django.conf import settings	1✔
4	from importlib import import_module	1✔
5
6	from tom_dataproducts.models import ReducedDatum	1✔
7
8
9	DEFAULT_DATA_PROCESSOR_CLASS = 'tom_dataproducts.data_processor.DataProcessor'	1✔
10
11
12	def run_data_processor(dp):	1✔
13	"""
14	Reads the `data_product_type` from the dp parameter and imports the corresponding `DATA_PROCESSORS` specified in
15	`settings.py`, then runs `process_data` and inserts the returned values into the database.
16
17	:param dp: DataProduct which will be processed into a list
18	:type dp: DataProduct
19
20	:returns: QuerySet of `ReducedDatum` objects created by the `run_data_processor` call
21	:rtype: `QuerySet` of `ReducedDatum`
22	"""
23
24	try:	1✔
25	processor_class = settings.DATA_PROCESSORS[dp.data_product_type]	1✔
UNCOV 26	except Exception:	×
UNCOV 27	processor_class = DEFAULT_DATA_PROCESSOR_CLASS	×
28
29	try:	1✔
30	mod_name, class_name = processor_class.rsplit('.', 1)	1✔
31	mod = import_module(mod_name)	1✔
32	clazz = getattr(mod, class_name)	1✔
UNCOV 33	except (ImportError, AttributeError):	×
UNCOV 34	raise ImportError('Could not import {}. Did you provide the correct path?'.format(processor_class))	×
35
36	data_processor = clazz()	1✔
37	data = data_processor.process_data(dp)	1✔
38	data_type = data_processor.data_type_override() or dp.data_product_type	1✔
39
40	reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=data_type,	1✔
41	timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data]
42	ReducedDatum.objects.bulk_create(reduced_datums)	1✔
43
44	return ReducedDatum.objects.filter(data_product=dp)	1✔
45
46
47	class DataProcessor():	1✔
48
49	FITS_MIMETYPES = ['image/fits', 'application/fits']	1✔
50	PLAINTEXT_MIMETYPES = ['text/plain', 'text/csv']	1✔
51
52	mimetypes.add_type('image/fits', '.fits')	1✔
53	mimetypes.add_type('image/fits', '.fz')	1✔
54	mimetypes.add_type('application/fits', '.fits')	1✔
55	mimetypes.add_type('application/fits', '.fz')	1✔
56
57	def process_data(self, data_product):	1✔
58	"""
59	Routes a photometry processing call to a method specific to a file-format. This method is expected to be
60	implemented by any subclasses.
61
62	:param data_product: DataProduct which will be processed into a list
63	:type data_product: DataProduct
64
65	:returns: python list of 2-tuples, each with a timestamp and corresponding data
66	:rtype: list of 2-tuples
67	"""
68	return []	×
69
70	def data_type_override(self):	1✔
71	"""
72	Override for the ReducedDatum data type, if you want it to be different from the
73	DataProduct data_type.
74	"""
75	return ''	1✔

TOMToolkit / tom_base / 8362804128

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous