1683

pending completion

Build # 1683

Build Type

Pull #260

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/LCOGT/banzai/commit/<a class=hub.com/LCOGT/banzai/commit/5ad7a259178c64312398109935dc45dd21b3140a">5ad7a2591<a href="https://github.com/LCOGT/banzai/commit/5ad7a259178c64312398109935dc45dd21b3140a">">Merge </a><a class="double-link" href="https://github.com/LCOGT/banzai/commit/<a class="double-link" href="https://github.com/LCOGT/banzai/commit/3556e31171aed553ebef7080fa614565dcbeba20">3556e3117</a>">3556e3117</a><a href="https://github.com/LCOGT/banzai/commit/5ad7a259178c64312398109935dc45dd21b3140a"> into a5afa896f">a5afa896f</a>

Pull Request Pull Request #260: Default Context

Run Details

7 of 7 new or added lines in 3 files covered. (100.0%)

1815 of 2838 relevant lines covered (63.95%)

0.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.2

/banzai/main.py

""" main.py: Main driver script for banzai.

    Console entry points to run the BANZAI pipeline.

Author
    Curtis McCully (cmccully@lcogt.net)

October 2015
"""
import argparse
import logging

from kombu import Exchange, Connection, Queue
from kombu.mixins import ConsumerMixin

from types import ModuleType

from banzai import settings, dbs, logs, calibrations
from banzai.context import Context
from banzai.utils import date_utils, stage_utils, import_utils, image_utils, fits_utils
from banzai.celery import process_image, app, schedule_calibration_stacking
from celery.schedules import crontab
import celery
import celery.bin.beat

logger = logging.getLogger('banzai')


class RealtimeModeListener(ConsumerMixin):
    def __init__(self, runtime_context):
        self.runtime_context = runtime_context
        self.broker_url = runtime_context.broker_url

    def on_connection_error(self, exc, interval):
        logger.error("{0}. Retrying connection in {1} seconds...".format(exc, interval))
        self.connection = self.connection.clone()
        self.connection.ensure_connection(max_retries=10)

    def get_consumers(self, Consumer, channel):
        consumer = Consumer(queues=[self.queue], callbacks=[self.on_message])
        # Only fetch one thing off the queue at a time
        consumer.qos(prefetch_count=1)
        return [consumer]

    def on_message(self, body, message):
        process_image.apply_async(args=(body, vars(self.runtime_context)))
        message.ack()  # acknowledge to the sender we got this message (it can be popped)


def add_settings_to_context(args, settings):
    # Get all of the settings that are not builtins and store them in the context object
    for setting in dir(settings):
        if '__' != setting[:2] and not isinstance(getattr(settings, setting), ModuleType):
            setattr(args, setting, getattr(settings, setting))


def parse_args(settings, extra_console_arguments=None, parser_description='Process LCO data.',
               parse_system_args=True):
    """
    Create a context object from the given arguments
    :param settings: settings object/module that has defaults for the context object
    :param extra_console_arguments: Non default arguments to add to the command line
    :param parser_description: Help str printed to the console
    :param parse_system_args: Actually parse command line parameters. Setting this False is
        useful to create a context object with defaults to be used for testing.
    :return:
    """
    parser = argparse.ArgumentParser(description=parser_description)

    parser.add_argument("--processed-path", default='/archive/engineering',
                        help='Top level directory where the processed data will be stored')
    parser.add_argument("--log-level", default='info', choices=['debug', 'info', 'warning',
                                                                'critical', 'fatal', 'error'])
    parser.add_argument('--post-to-archive', dest='post_to_archive', action='store_true', default=False)
    parser.add_argument('--no-file-cache', dest='no_file_cache', action='store_true', default=False,
                        help='Turn off saving files to disk')
    parser.add_argument('--post-to-elasticsearch', dest='post_to_elasticsearch', action='store_true',
                        default=False)
    parser.add_argument('--fpack', dest='fpack', action='store_true', default=False,
                        help='Fpack the output files?')
    parser.add_argument('--override-missing-calibrations', dest='override_missing', action='store_true', default=False,
                        help='Continue processing a file even if a master calibration does not exist?')
    parser.add_argument('--rlevel', dest='reduction_level', default=91, type=int, help='Reduction level')
    parser.add_argument('--db-address', dest='db_address',
                        default='mysql://cmccully:password@localhost/test',
                        help='Database address: Should be in SQLAlchemy form')
    parser.add_argument('--elasticsearch-url', dest='elasticsearch_url',
                        default='http://elasticsearch.lco.gtn:9200')
    parser.add_argument('--es-index', dest='elasticsearch_qc_index', default='banzai_qc',
                        help='ElasticSearch index to use for QC results')
    parser.add_argument('--es-doc-type', dest='elasticsearch_doc_type', default='qc',
                        help='Elasticsearch document type for QC records')
    parser.add_argument('--no-bpm', dest='no_bpm', default=False, action='store_true',
                        help='Do not use a bad pixel mask to reduce data (BPM contains all zeros)')
    parser.add_argument('--use-only-older-calibrations', dest='use_only_older_calibrations', default=False,
                        action='store_true', help='Only use calibrations that were created before the start of the block')
    parser.add_argument('--preview-mode', dest='preview_mode', default=False, action='store_true',
                        help='Save the reductions to the preview directory')
    parser.add_argument('--max-tries', dest='max_tries', default=5,
                        help='Maximum number of times to try to process a frame')
    parser.add_argument('--broker-url', dest='broker_url',
                        help='URL for the FITS broker service.')

    if extra_console_arguments is None:
        extra_console_arguments = []
    for argument in extra_console_arguments:
        parser.add_argument(*argument['args'], **argument['kwargs'])

    if parse_system_args:
        args = parser.parse_args()
    else:
        args = parser.parse_args([])
    logs.set_log_level(args.log_level)

    add_settings_to_context(args, settings)
    return Context(args)


def reduce_single_frame():
    extra_console_arguments = [{'args': ['--filepath'],
                                'kwargs': {'dest': 'path', 'help': 'Full path to the file to process'}}]
    runtime_context = parse_args(settings, extra_console_arguments=extra_console_arguments)
    # Short circuit
    if not image_utils.image_can_be_processed(fits_utils.get_primary_header(runtime_context.path), runtime_context):
        logger.error('Image cannot be processed. Check to make sure the instrument '
                     'is in the database and that the OBSTYPE is recognized by BANZAI',
                     extra_tags={'filename': runtime_context.path})
        return
    try:
        stage_utils.run_pipeline_stages([{'path': runtime_context.path}], runtime_context)
    except Exception:
        logger.error(logs.format_exception(), extra_tags={'filepath': runtime_context.path})


def make_master_calibrations():
    extra_console_arguments = [{'args': ['--site'],
                                'kwargs': {'dest': 'site', 'help': 'Site code (e.g. ogg)', 'required': True}},
                               {'args': ['--camera'],
                                'kwargs': {'dest': 'camera', 'help': 'Camera (e.g. kb95)', 'required': True}},
                               {'args': ['--frame-type'],
                                'kwargs': {'dest': 'frame_type', 'help': 'Type of frames to process',
                                           'choices': ['bias', 'dark', 'skyflat'], 'required': True}},
                               {'args': ['--min-date'],
                                'kwargs': {'dest': 'min_date', 'required': True, 'type': date_utils.validate_date,
                                           'help': 'Earliest observation time of the individual calibration frames. '
                                                   'Must be in the format "YYYY-MM-DDThh:mm:ss".'}},
                               {'args': ['--max-date'],
                                'kwargs': {'dest': 'max_date', 'required': True, 'type': date_utils.validate_date,
                                           'help': 'Latest observation time of the individual calibration frames. '
                                                   'Must be in the format "YYYY-MM-DDThh:mm:ss".'}}]

    runtime_context = parse_args(settings, extra_console_arguments=extra_console_arguments)
    instrument = dbs.query_for_instrument(runtime_context.db_address, runtime_context.site, runtime_context.camera)
    calibrations.make_master_calibrations(instrument,  runtime_context.frame_type.upper(),
                                          runtime_context.min_date, runtime_context.max_date, runtime_context)


def start_stacking_scheduler():
    logger.info('Entered entrypoint to celery beat scheduling')
    runtime_context = parse_args(settings)
    for site, entry in runtime_context.SCHEDULE_STACKING_CRON_ENTRIES.items():
        app.add_periodic_task(crontab(minute=entry['minute'], hour=entry['hour']),
                              schedule_calibration_stacking.s(site=site, runtime_context=vars(runtime_context)))

    beat = celery.bin.beat.beat(app=app)
    logger.info('Starting celery beat')
    beat.run()


def run_realtime_pipeline():
    extra_console_arguments = [{'args': ['--n-processes'],
                                'kwargs': {'dest': 'n_processes', 'default': 12,
                                           'help': 'Number of listener processes to spawn.', 'type': int}},
                               {'args': ['--queue-name'],
                                'kwargs': {'dest': 'queue_name', 'default': 'banzai_pipeline',
                                           'help': 'Name of the queue to listen to from the fits exchange.'}}]

    runtime_context = parse_args(settings, extra_console_arguments=extra_console_arguments)
    start_listener(runtime_context)


def start_listener(runtime_context):
    # Need to keep the amqp logger level at least as high as INFO,
    # or else it send heartbeat check messages every second
    logging.getLogger('amqp').setLevel(max(logger.level, getattr(logging, 'INFO')))
    logger.info('Starting pipeline listener')

    fits_exchange = Exchange(runtime_context.FITS_EXCHANGE, type='fanout')
    listener = RealtimeModeListener(runtime_context)

    with Connection(runtime_context.broker_url) as connection:
        listener.connection = connection.clone()
        listener.queue = Queue(runtime_context.queue_name, fits_exchange)
        try:
            listener.run()
        except listener.connection.connection_errors:
            listener.connection = connection.clone()
            listener.ensure_connection(max_retries=10)
        except KeyboardInterrupt:
            logger.info('Shutting down pipeline listener.')


def mark_frame(mark_as):
    parser = argparse.ArgumentParser(description="Set the is_bad flag to mark the frame as {mark_as}"
                                                 "for a calibration frame in the database ".format(mark_as=mark_as))
    parser.add_argument('--filename', dest='filename', required=True,
                        help='Name of calibration file to be marked')
    parser.add_argument('--db-address', dest='db_address',
                        default='mysql://cmccully:password@localhost/test',
                        help='Database address: Should be in SQLAlchemy form')
    parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
                                                                 'critical', 'fatal', 'error'])

    args = parser.parse_args()
    logs.set_log_level(args.log_level)

    logger.info("Marking the frame {filename} as {mark_as}".format(filename=args.filename, mark_as=mark_as))
    dbs.mark_frame(args.filename, mark_as, db_address=args.db_address)
    logger.info("Finished")


def add_instrument():
    parser = argparse.ArgumentParser(description="Add a new instrument to the database")
    parser.add_argument("--site", help='Site code (e.g. ogg)', required=True)
    parser.add_argument("--camera", help='Camera (e.g. kb95)', required=True)
    parser.add_argument("--name", help='Instrument name (e.g kb05, nres03)', required=True)
    parser.add_argument("--camera-type", dest='camera_type',
                        help="Camera type (e.g. 1m0-SciCam-Sinistro)", required=True)
    parser.add_argument('--db-address', dest='db_address', default='sqlite:///test.db',
                        help='Database address: Should be in SQLAlchemy format')
    args = parser.parse_args()
    instrument = {'site': args.site,
                  'camera': args.camera,
                  'type': args.camera_type,
                  'name': args.name}
    dbs.add_instrument(instrument, args.db_address)


def mark_frame_as_good():
    mark_frame("good")


def mark_frame_as_bad():
    mark_frame("bad")


def update_db():
    parser = argparse.ArgumentParser(description="Query the configdb to ensure that the instruments table"
                                                 "has the most up-to-date information")

    parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
                                                                 'critical', 'fatal', 'error'])
    parser.add_argument('--configdb-address', dest='configdb_address',
                        default='http://configdb.lco.gtn/sites/',
                        help='URL of the configdb with instrument information')
    parser.add_argument('--db-address', dest='db_address',
                        default='mysql://cmccully:password@localhost/test',
                        help='Database address: Should be in SQLAlchemy form')
    args = parser.parse_args()
    logs.set_log_level(args.log_level)

    try:
        dbs.populate_instrument_tables(db_address=args.db_address, configdb_address=args.configdb_address)
    except Exception:
        logger.error('Could not populate instruments table: {error}'.format(error=logs.format_exception()))


def add_bpm():
    parser = argparse.ArgumentParser(description="Add a bad pixel mask to the db.")
    parser.add_argument('--filename', help='Full path to Bad Pixel Mask file')
    parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
                                                                 'critical', 'fatal', 'error'])
    parser.add_argument('--db-address', dest='db_address',
                        default='mysql://cmccully:password@localhost/test',
                        help='Database address: Should be in SQLAlchemy form')
    args = parser.parse_args()
    add_settings_to_context(args, settings)
    logs.set_log_level(args.log_level)
    frame_factory = import_utils.import_attribute(settings.FRAME_FACTORY)()
    bpm_image = frame_factory.open({'path': args.filename}, args)
    bpm_image.is_master = True
    dbs.save_calibration_info(args.filename, bpm_image, args.db_address)


def create_db():
    """
    Create the database structure.

    This only needs to be run once on initialization of the database.
    """
    parser = argparse.ArgumentParser("Create the database.\n\n"
                                     "This only needs to be run once on initialization of the database.")

    parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',
                                                                 'critical', 'fatal', 'error'])
    parser.add_argument('--db-address', dest='db_address',
                        default='sqlite3:///test.db',
                        help='Database address: Should be in SQLAlchemy form')
    args = parser.parse_args()
    logs.set_log_level(args.log_level)

    dbs.create_db(args.db_address)

1	""" main.py: Main driver script for banzai.
2
3	Console entry points to run the BANZAI pipeline.
4
5	Author
6	Curtis McCully (cmccully@lcogt.net)
7
8	October 2015
9	"""
10	import argparse	1✔
11	import logging	1✔
12
13	from kombu import Exchange, Connection, Queue	1✔
14	from kombu.mixins import ConsumerMixin	1✔
15
16	from types import ModuleType	1✔
17
18	from banzai import settings, dbs, logs, calibrations	1✔
19	from banzai.context import Context	1✔
20	from banzai.utils import date_utils, stage_utils, import_utils, image_utils, fits_utils	1✔
21	from banzai.celery import process_image, app, schedule_calibration_stacking	1✔
22	from celery.schedules import crontab	1✔
23	import celery	1✔
24	import celery.bin.beat	1✔
25
26	logger = logging.getLogger('banzai')	1✔
27
28
29	class RealtimeModeListener(ConsumerMixin):	1✔
30	def __init__(self, runtime_context):	1✔
31	self.runtime_context = runtime_context	×
32	self.broker_url = runtime_context.broker_url	×
33
34	def on_connection_error(self, exc, interval):	1✔
35	logger.error("{0}. Retrying connection in {1} seconds...".format(exc, interval))	×
36	self.connection = self.connection.clone()	×
37	self.connection.ensure_connection(max_retries=10)	×
38
39	def get_consumers(self, Consumer, channel):	1✔
40	consumer = Consumer(queues=[self.queue], callbacks=[self.on_message])	×
41	# Only fetch one thing off the queue at a time
42	consumer.qos(prefetch_count=1)	×
43	return [consumer]	×
44
45	def on_message(self, body, message):	1✔
46	process_image.apply_async(args=(body, vars(self.runtime_context)))	×
47	message.ack() # acknowledge to the sender we got this message (it can be popped)	×
48
49
50	def add_settings_to_context(args, settings):	1✔
51	# Get all of the settings that are not builtins and store them in the context object
52	for setting in dir(settings):	×
53	if '__' != setting[:2] and not isinstance(getattr(settings, setting), ModuleType):	×
54	setattr(args, setting, getattr(settings, setting))	×
55
56
57	def parse_args(settings, extra_console_arguments=None, parser_description='Process LCO data.',	1✔
58	parse_system_args=True):
59	"""
60	Create a context object from the given arguments
61	:param settings: settings object/module that has defaults for the context object
62	:param extra_console_arguments: Non default arguments to add to the command line
63	:param parser_description: Help str printed to the console
64	:param parse_system_args: Actually parse command line parameters. Setting this False is
65	useful to create a context object with defaults to be used for testing.
66	:return:
67	"""
68	parser = argparse.ArgumentParser(description=parser_description)	×
69
70	parser.add_argument("--processed-path", default='/archive/engineering',	×
71	help='Top level directory where the processed data will be stored')
72	parser.add_argument("--log-level", default='info', choices=['debug', 'info', 'warning',	×
73	'critical', 'fatal', 'error'])
74	parser.add_argument('--post-to-archive', dest='post_to_archive', action='store_true', default=False)	×
75	parser.add_argument('--no-file-cache', dest='no_file_cache', action='store_true', default=False,	×
76	help='Turn off saving files to disk')
77	parser.add_argument('--post-to-elasticsearch', dest='post_to_elasticsearch', action='store_true',	×
78	default=False)
79	parser.add_argument('--fpack', dest='fpack', action='store_true', default=False,	×
80	help='Fpack the output files?')
81	parser.add_argument('--override-missing-calibrations', dest='override_missing', action='store_true', default=False,	×
82	help='Continue processing a file even if a master calibration does not exist?')
83	parser.add_argument('--rlevel', dest='reduction_level', default=91, type=int, help='Reduction level')	×
84	parser.add_argument('--db-address', dest='db_address',	×
85	default='mysql://cmccully:password@localhost/test',
86	help='Database address: Should be in SQLAlchemy form')
87	parser.add_argument('--elasticsearch-url', dest='elasticsearch_url',	×
88	default='http://elasticsearch.lco.gtn:9200')
89	parser.add_argument('--es-index', dest='elasticsearch_qc_index', default='banzai_qc',	×
90	help='ElasticSearch index to use for QC results')
91	parser.add_argument('--es-doc-type', dest='elasticsearch_doc_type', default='qc',	×
92	help='Elasticsearch document type for QC records')
93	parser.add_argument('--no-bpm', dest='no_bpm', default=False, action='store_true',	×
94	help='Do not use a bad pixel mask to reduce data (BPM contains all zeros)')
95	parser.add_argument('--use-only-older-calibrations', dest='use_only_older_calibrations', default=False,	×
96	action='store_true', help='Only use calibrations that were created before the start of the block')
97	parser.add_argument('--preview-mode', dest='preview_mode', default=False, action='store_true',	×
98	help='Save the reductions to the preview directory')
99	parser.add_argument('--max-tries', dest='max_tries', default=5,	×
100	help='Maximum number of times to try to process a frame')
101	parser.add_argument('--broker-url', dest='broker_url',	×
102	help='URL for the FITS broker service.')
103
104	if extra_console_arguments is None:	×
105	extra_console_arguments = []	×
106	for argument in extra_console_arguments:	×
107	parser.add_argument(argument['args'], *argument['kwargs'])	×
108
109	if parse_system_args:	×
110	args = parser.parse_args()	×
111	else:
112	args = parser.parse_args([])	×
113	logs.set_log_level(args.log_level)	×
114
115	add_settings_to_context(args, settings)	×
116	return Context(args)	×
117
118
119	def reduce_single_frame():	1✔
120	extra_console_arguments = [{'args': ['--filepath'],	×
121	'kwargs': {'dest': 'path', 'help': 'Full path to the file to process'}}]
122	runtime_context = parse_args(settings, extra_console_arguments=extra_console_arguments)	×
123	# Short circuit
124	if not image_utils.image_can_be_processed(fits_utils.get_primary_header(runtime_context.path), runtime_context):	×
125	logger.error('Image cannot be processed. Check to make sure the instrument '	×
126	'is in the database and that the OBSTYPE is recognized by BANZAI',
127	extra_tags={'filename': runtime_context.path})
128	return	×
129	try:	×
130	stage_utils.run_pipeline_stages([{'path': runtime_context.path}], runtime_context)	×
131	except Exception:	×
132	logger.error(logs.format_exception(), extra_tags={'filepath': runtime_context.path})	×
133
134
135	def make_master_calibrations():	1✔
136	extra_console_arguments = [{'args': ['--site'],	×
137	'kwargs': {'dest': 'site', 'help': 'Site code (e.g. ogg)', 'required': True}},
138	{'args': ['--camera'],
139	'kwargs': {'dest': 'camera', 'help': 'Camera (e.g. kb95)', 'required': True}},
140	{'args': ['--frame-type'],
141	'kwargs': {'dest': 'frame_type', 'help': 'Type of frames to process',
142	'choices': ['bias', 'dark', 'skyflat'], 'required': True}},
143	{'args': ['--min-date'],
144	'kwargs': {'dest': 'min_date', 'required': True, 'type': date_utils.validate_date,
145	'help': 'Earliest observation time of the individual calibration frames. '
146	'Must be in the format "YYYY-MM-DDThh:mm:ss".'}},
147	{'args': ['--max-date'],
148	'kwargs': {'dest': 'max_date', 'required': True, 'type': date_utils.validate_date,
149	'help': 'Latest observation time of the individual calibration frames. '
150	'Must be in the format "YYYY-MM-DDThh:mm:ss".'}}]
151
152	runtime_context = parse_args(settings, extra_console_arguments=extra_console_arguments)	×
153	instrument = dbs.query_for_instrument(runtime_context.db_address, runtime_context.site, runtime_context.camera)	×
154	calibrations.make_master_calibrations(instrument, runtime_context.frame_type.upper(),	×
155	runtime_context.min_date, runtime_context.max_date, runtime_context)
156
157
158	def start_stacking_scheduler():	1✔
159	logger.info('Entered entrypoint to celery beat scheduling')	×
160	runtime_context = parse_args(settings)	×
161	for site, entry in runtime_context.SCHEDULE_STACKING_CRON_ENTRIES.items():	×
162	app.add_periodic_task(crontab(minute=entry['minute'], hour=entry['hour']),	×
163	schedule_calibration_stacking.s(site=site, runtime_context=vars(runtime_context)))
164
165	beat = celery.bin.beat.beat(app=app)	×
166	logger.info('Starting celery beat')	×
167	beat.run()	×
168
169
170	def run_realtime_pipeline():	1✔
171	extra_console_arguments = [{'args': ['--n-processes'],	×
172	'kwargs': {'dest': 'n_processes', 'default': 12,
173	'help': 'Number of listener processes to spawn.', 'type': int}},
174	{'args': ['--queue-name'],
175	'kwargs': {'dest': 'queue_name', 'default': 'banzai_pipeline',
176	'help': 'Name of the queue to listen to from the fits exchange.'}}]
177
178	runtime_context = parse_args(settings, extra_console_arguments=extra_console_arguments)	×
179	start_listener(runtime_context)	×
180
181
182	def start_listener(runtime_context):	1✔
183	# Need to keep the amqp logger level at least as high as INFO,
184	# or else it send heartbeat check messages every second
185	logging.getLogger('amqp').setLevel(max(logger.level, getattr(logging, 'INFO')))	×
186	logger.info('Starting pipeline listener')	×
187
188	fits_exchange = Exchange(runtime_context.FITS_EXCHANGE, type='fanout')	×
189	listener = RealtimeModeListener(runtime_context)	×
190
191	with Connection(runtime_context.broker_url) as connection:	×
192	listener.connection = connection.clone()	×
193	listener.queue = Queue(runtime_context.queue_name, fits_exchange)	×
194	try:	×
195	listener.run()	×
196	except listener.connection.connection_errors:	×
197	listener.connection = connection.clone()	×
198	listener.ensure_connection(max_retries=10)	×
199	except KeyboardInterrupt:	×
200	logger.info('Shutting down pipeline listener.')	×
201
202
203	def mark_frame(mark_as):	1✔
204	parser = argparse.ArgumentParser(description="Set the is_bad flag to mark the frame as {mark_as}"	×
205	"for a calibration frame in the database ".format(mark_as=mark_as))
206	parser.add_argument('--filename', dest='filename', required=True,	×
207	help='Name of calibration file to be marked')
208	parser.add_argument('--db-address', dest='db_address',	×
209	default='mysql://cmccully:password@localhost/test',
210	help='Database address: Should be in SQLAlchemy form')
211	parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',	×
212	'critical', 'fatal', 'error'])
213
214	args = parser.parse_args()	×
215	logs.set_log_level(args.log_level)	×
216
217	logger.info("Marking the frame {filename} as {mark_as}".format(filename=args.filename, mark_as=mark_as))	×
218	dbs.mark_frame(args.filename, mark_as, db_address=args.db_address)	×
219	logger.info("Finished")	×
220
221
222	def add_instrument():	1✔
223	parser = argparse.ArgumentParser(description="Add a new instrument to the database")	×
224	parser.add_argument("--site", help='Site code (e.g. ogg)', required=True)	×
225	parser.add_argument("--camera", help='Camera (e.g. kb95)', required=True)	×
226	parser.add_argument("--name", help='Instrument name (e.g kb05, nres03)', required=True)	×
227	parser.add_argument("--camera-type", dest='camera_type',	×
228	help="Camera type (e.g. 1m0-SciCam-Sinistro)", required=True)
229	parser.add_argument('--db-address', dest='db_address', default='sqlite:///test.db',	×
230	help='Database address: Should be in SQLAlchemy format')
231	args = parser.parse_args()	×
232	instrument = {'site': args.site,	×
233	'camera': args.camera,
234	'type': args.camera_type,
235	'name': args.name}
236	dbs.add_instrument(instrument, args.db_address)	×
237
238
239	def mark_frame_as_good():	1✔
240	mark_frame("good")	×
241
242
243	def mark_frame_as_bad():	1✔
244	mark_frame("bad")	×
245
246
247	def update_db():	1✔
248	parser = argparse.ArgumentParser(description="Query the configdb to ensure that the instruments table"	×
249	"has the most up-to-date information")
250
251	parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',	×
252	'critical', 'fatal', 'error'])
253	parser.add_argument('--configdb-address', dest='configdb_address',	×
254	default='http://configdb.lco.gtn/sites/',
255	help='URL of the configdb with instrument information')
256	parser.add_argument('--db-address', dest='db_address',	×
257	default='mysql://cmccully:password@localhost/test',
258	help='Database address: Should be in SQLAlchemy form')
259	args = parser.parse_args()	×
260	logs.set_log_level(args.log_level)	×
261
262	try:	×
263	dbs.populate_instrument_tables(db_address=args.db_address, configdb_address=args.configdb_address)	×
264	except Exception:	×
265	logger.error('Could not populate instruments table: {error}'.format(error=logs.format_exception()))	×
266
267
268	def add_bpm():	1✔
269	parser = argparse.ArgumentParser(description="Add a bad pixel mask to the db.")	×
270	parser.add_argument('--filename', help='Full path to Bad Pixel Mask file')	×
271	parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',	×
272	'critical', 'fatal', 'error'])
273	parser.add_argument('--db-address', dest='db_address',	×
274	default='mysql://cmccully:password@localhost/test',
275	help='Database address: Should be in SQLAlchemy form')
276	args = parser.parse_args()	×
277	add_settings_to_context(args, settings)	×
278	logs.set_log_level(args.log_level)	×
279	frame_factory = import_utils.import_attribute(settings.FRAME_FACTORY)()	×
280	bpm_image = frame_factory.open({'path': args.filename}, args)	×
281	bpm_image.is_master = True	×
282	dbs.save_calibration_info(args.filename, bpm_image, args.db_address)	×
283
284
285	def create_db():	1✔
286	"""
287	Create the database structure.
288
289	This only needs to be run once on initialization of the database.
290	"""
291	parser = argparse.ArgumentParser("Create the database.\n\n"	1✔
292	"This only needs to be run once on initialization of the database.")
293
294	parser.add_argument("--log-level", default='debug', choices=['debug', 'info', 'warning',	1✔
295	'critical', 'fatal', 'error'])
296	parser.add_argument('--db-address', dest='db_address',	1✔
297	default='sqlite3:///test.db',
298	help='Database address: Should be in SQLAlchemy form')
299	args = parser.parse_args()	1✔
300	logs.set_log_level(args.log_level)	1✔
301
302	dbs.create_db(args.db_address)	1✔

LCOGT / banzai / 1683

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous