LCOGT / banzai-nres / 593

pending completion

Build # 593

Build Type

Pull #34

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/LCOGT/banzai-nres/commit/<a class=hub.com/LCOGT/banzai-nres/commit/<a class="double-link" href="https://git"><a class=hub.com/LCOGT/banzai-nres/commit/<a class="double-link" href="https://git"><a class=hub.com/LCOGT/banzai-nres/commit/66492299e8b60e9f9cd0fd97bf60c6f15bf77de2">66492299e">&lt;a href=&quot;https://github.com/LCOGT/banzai-nres/commit/</a><a class="double-link" href="https://github.com/LCOGT/banzai-nres/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LCOGT/banzai-nres/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LCOGT/banzai-nres/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LCOGT/banzai-nres/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LCOGT/banzai-nres/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/LCOGT/banzai-nres/commit/66492299e8b60e9f9cd0fd97bf60c6f15bf77de2">66492299e</a><a href="https://github.com/LCOGT/banzai-nres/commit/66492299e8b60e9f9cd0fd97bf60c6f15bf77de2">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/LCOGT/banzai-nres/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/LCOGT/banzai-nres/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LCOGT/banzai-nres/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LCOGT/banzai-nres/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LCOGT/banzai-nres/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LCOGT/banzai-nres/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/LCOGT/banzai-nres/commit/66492299e8b60e9f9cd0fd97bf60c6f15bf77de2&quot;&gt;66492299e&lt;/a&gt;&lt;a href=&quot;https://github.com/LCOGT/banzai-nres/commit/66492299e8b60e9f9cd0fd97bf60c6f1... (continued)

Pull Request Pull Request #34: Add the ability to normalize out the continuum.

Run Details

125 of 125 new or added lines in 11 files covered. (100.0%)

86381 of 290528 relevant lines covered (29.73%)

0.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

32.52

/banzai_nres/tests/test_e2e.py

import pytest
from banzai.tests.utils import FakeResponse, get_min_and_max_dates
from banzai_nres import settings
import os
import mock
import numpy as np
from banzai.utils import file_utils
import time
from glob import glob
import pkg_resources
from banzai.celery import app, schedule_calibration_stacking
from banzai.dbs import get_session
from banzai import dbs
from types import ModuleType
from datetime import datetime
from dateutil.parser import parse
from astropy.io import fits
from astropy.table import Table
from banzai.utils import fits_utils

import logging

logger = logging.getLogger('banzai')

TEST_PACKAGE = 'banzai_nres.tests'

DATA_ROOT = os.path.join(os.sep, 'archive', 'engineering')

SITES = [os.path.basename(site_path) for site_path in glob(os.path.join(DATA_ROOT, '???'))]
INSTRUMENTS = [os.path.join(site, os.path.basename(instrument_path)) for site in SITES
               for instrument_path in glob(os.path.join(os.path.join(DATA_ROOT, site, '*')))]

DAYS_OBS = [os.path.join(instrument, os.path.basename(dayobs_path)) for instrument in INSTRUMENTS
            for dayobs_path in glob(os.path.join(DATA_ROOT, instrument, '201*'))]

TEST_PACKAGE = 'banzai_nres.tests'
CONFIGDB_FILENAME = pkg_resources.resource_filename('banzai_nres.tests', 'data/configdb_example.json')



def observation_portal_side_effect(*args, **kwargs):
    site = kwargs['params']['site']
    start = datetime.strftime(parse(kwargs['params']['start_after']).replace(tzinfo=None).date(), '%Y%m%d')
    filename = 'test_obs_portal_response_{site}_{start}.json'.format(site=site, start=start)
    filename = pkg_resources.resource_filename('banzai_nres.tests', f'data/{filename}')
    return FakeResponse(filename)


def get_instrument_ids(db_address, names):
    with get_session(db_address) as db_session:
        instruments = []
        for name in names:
            criteria = dbs.Instrument.name == name
            instruments.extend(db_session.query(dbs.Instrument).filter(criteria).all())
    return [instrument.id for instrument in instruments]


def celery_join():
    celery_inspector = app.control.inspect()
    log_counter = 0
    while True:
        time.sleep(1)
        log_counter += 1
        if log_counter % 5 == 0:
            logger.info('Processing: ' + '. ' * (log_counter // 5))
        queues = [celery_inspector.active(), celery_inspector.scheduled(), celery_inspector.reserved()]
        if any([queue is None or 'celery@banzai-celery-worker' not in queue for queue in queues]):
            logger.warning('No valid celery queues were detected, retrying...', extra_tags={'queues': queues})
            # Reset the celery connection
            celery_inspector = app.control.inspect()
            continue
        if all([len(queue['celery@banzai-celery-worker']) == 0 for queue in queues]):
            break


def run_reduce_individual_frames(raw_filenames):
    logger.info('Reducing individual frames for filenames: {filenames}'.format(filenames=raw_filenames))
    for day_obs in DAYS_OBS:
        raw_path = os.path.join(DATA_ROOT, day_obs, 'raw')
        for filename in glob(os.path.join(raw_path, raw_filenames)):
            file_utils.post_to_archive_queue(filename, os.getenv('FITS_BROKER'),
                                             exchange_name=os.getenv('FITS_EXCHANGE'))
    celery_join()
    logger.info('Finished reducing individual frames for filenames: {filenames}'.format(filenames=raw_filenames))


def stack_calibrations(frame_type):
    logger.info('Stacking calibrations for frame type: {frame_type}'.format(frame_type=frame_type))
    logger.info('Stacking calibrations for frame type: {frame_type}'.format(frame_type=frame_type))
    for day_obs in DAYS_OBS:
        site, camera, dayobs = day_obs.split('/')
        timezone = dbs.get_timezone(site, db_address=os.environ['DB_ADDRESS'])
        min_date, max_date = get_min_and_max_dates(timezone, dayobs=dayobs)
        runtime_context = dict(processed_path=DATA_ROOT, log_level='debug', post_to_archive=False,
                               post_to_elasticsearch=False, fpack=True, reduction_level=91,
                               db_address=os.environ['DB_ADDRESS'], elasticsearch_qc_index='banzai_qc',
                               elasticsearch_url='http://elasticsearch.lco.gtn:9200', elasticsearch_doc_type='qc',
                               no_bpm=False, ignore_schedulability=True, use_only_older_calibrations=False,
                               preview_mode=False, max_tries=5, broker_url=os.getenv('FITS_BROKER'),
                               no_file_cache=False)
        for setting in dir(settings):
            if '__' != setting[:2] and not isinstance(getattr(settings, setting), ModuleType):
                runtime_context[setting] = getattr(settings, setting)
        schedule_calibration_stacking(site, runtime_context, min_date=min_date, max_date=max_date,
                                      frame_types=[frame_type])
    celery_join()
    logger.info('Finished stacking calibrations for frame type: {frame_type}'.format(frame_type=frame_type))


def mark_frames_as_good(raw_filenames):
    logger.info('Marking frames as good for filenames: {filenames}'.format(filenames=raw_filenames))
    for day_obs in DAYS_OBS:
        for filename in glob(os.path.join(DATA_ROOT, day_obs, 'processed', raw_filenames)):
            dbs.mark_frame(os.path.basename(filename), "good", db_address=os.environ['DB_ADDRESS'])
    logger.info('Finished marking frames as good for filenames: {filenames}'.format(filenames=raw_filenames))


def get_expected_number_of_calibrations(raw_filenames, calibration_type):
    number_of_stacks_that_should_have_been_created = 0
    for day_obs in DAYS_OBS:
        raw_filenames_for_this_dayobs = glob(os.path.join(DATA_ROOT, day_obs, 'raw', raw_filenames))
        if calibration_type.lower() == 'lampflat' or calibration_type.lower() == 'double':
            # Group by fibers lit if we are stacking lampflats or doubles (arc frames)
            observed_fibers = []
            for raw_filename in raw_filenames_for_this_dayobs:
                cal_hdu = fits.open(raw_filename)
                observed_fibers.append(cal_hdu[1].header.get('OBJECTS'))
                cal_hdu.close()
            observed_fibers = set(observed_fibers)
            number_of_stacks_that_should_have_been_created += len(observed_fibers)
        else:
            # Just one calibration per night
            if len(raw_filenames_for_this_dayobs) > 0:
                number_of_stacks_that_should_have_been_created += 1
    return number_of_stacks_that_should_have_been_created


def check_if_individual_frames_exist(filenames):
    for day_obs in DAYS_OBS:
        raw_files = glob(os.path.join(DATA_ROOT, day_obs, 'raw', filenames))
        processed_files = glob(os.path.join(DATA_ROOT, day_obs, 'processed', filenames.replace('00', '91')))
        assert len(raw_files) == len(processed_files)


def run_check_if_stacked_calibrations_were_created(raw_filenames, calibration_type):
    created_stacked_calibrations = []
    number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_filenames, calibration_type)
    for day_obs in DAYS_OBS:
        created_stacked_calibrations += glob(os.path.join(DATA_ROOT, day_obs, 'processed',
                                                          '*' + calibration_type.lower() + '*.fits*'))
    assert number_of_stacks_that_should_have_been_created > 0
    assert len(created_stacked_calibrations) == number_of_stacks_that_should_have_been_created


def run_check_if_stacked_calibrations_have_extensions(calibration_type, extensions_to_check):
    created_stacked_calibrations = []
    for day_obs in DAYS_OBS:
        created_stacked_calibrations += glob(os.path.join(DATA_ROOT, day_obs, 'processed',
                                                          '*' + calibration_type.lower() + '*.fits*'))
    for cal in created_stacked_calibrations:
        hdulist = fits.open(cal)
        extnames = [hdulist[i].header.get('extname', None) for i in range(len(hdulist))]
        for ext in extensions_to_check:
            logger.info(f'checking if {ext} is in the saved extensions of {cal}')
            assert ext in extnames


def check_extracted_spectra(raw_filename, spec_extname, columns):
    created_images = []
    for day_obs in DAYS_OBS:
        created_images += glob(os.path.join(DATA_ROOT, day_obs, 'processed', raw_filename))
    for filename in created_images:
        with fits.open(filename) as f:
            hdu = fits_utils.unpack(f)
        spectrum = Table(hdu[spec_extname].data)
        for colname in columns:
            assert colname in spectrum.colnames
            assert not np.allclose(spectrum[colname], 0)
        assert 'RV' in hdu[0].header


def run_check_if_stacked_calibrations_are_in_db(raw_filenames, calibration_type):
    number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_filenames, calibration_type)
    with dbs.get_session(os.environ['DB_ADDRESS']) as db_session:
        calibrations_in_db = db_session.query(dbs.CalibrationImage).filter(dbs.CalibrationImage.type == calibration_type)
        calibrations_in_db = calibrations_in_db.filter(dbs.CalibrationImage.is_master).all()
    assert number_of_stacks_that_should_have_been_created > 0
    assert len(calibrations_in_db) == number_of_stacks_that_should_have_been_created


@pytest.mark.e2e
@pytest.fixture(scope='module')
@mock.patch('banzai.dbs.requests.get', return_value=FakeResponse(CONFIGDB_FILENAME))
def init(configdb):
    os.system(f'banzai_create_db --db-address={os.environ["DB_ADDRESS"]}')
    dbs.populate_instrument_tables(db_address=os.environ["DB_ADDRESS"], configdb_address='http://fakeconfigdb')
    os.system(f'banzai_add_site --site elp --latitude 30.67986944 --longitude -104.015175 --elevation 2027 --timezone -6 --db-address={os.environ["DB_ADDRESS"]}')
    os.system(f'banzai_add_site --site lsc --latitude -30.1673833333 --longitude -70.8047888889 --elevation 2198 --timezone -4 --db-address={os.environ["DB_ADDRESS"]}')
    os.system(f'banzai_add_instrument --site lsc --camera fl09 --name nres01 --instrument-type 1m0-NRES-SciCam --db-address={os.environ["DB_ADDRESS"]}')
    os.system(f'banzai_add_instrument --site elp --camera fl17 --name nres02 --instrument-type 1m0-NRES-SciCam --db-address={os.environ["DB_ADDRESS"]}')
    for instrument in INSTRUMENTS:
        for bpm_filename in glob(os.path.join(DATA_ROOT, instrument, 'bpm/*bpm*')):
            logger.info(f'adding bpm {bpm_filename} to the database')
            os.system(f'banzai_nres_add_bpm --filename {bpm_filename} --db-address={os.environ["DB_ADDRESS"]}')


@pytest.mark.e2e
@pytest.mark.master_bias
class TestMasterBiasCreation:
    @pytest.fixture(autouse=True)
    @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)
    def stack_bias_frames(self, mock_lake, init):
        run_reduce_individual_frames('*b00.fits*')
        mark_frames_as_good('*b91.fits*')
        stack_calibrations('bias')

    def test_if_stacked_bias_frame_was_created(self):
        check_if_individual_frames_exist('*b00*')
        run_check_if_stacked_calibrations_were_created('*b00.fits*', 'bias')
        run_check_if_stacked_calibrations_are_in_db('*b00.fits*', 'BIAS')


@pytest.mark.e2e
@pytest.mark.master_dark
class TestMasterDarkCreation:
    @pytest.fixture(autouse=True)
    @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)
    def stack_dark_frames(self, mock_lake):
        run_reduce_individual_frames('*d00.fits*')
        mark_frames_as_good('*d91.fits*')
        stack_calibrations('dark')

    def test_if_stacked_dark_frame_was_created(self):
        check_if_individual_frames_exist('*d00*')
        run_check_if_stacked_calibrations_were_created('*d00.fits*', 'dark')
        run_check_if_stacked_calibrations_are_in_db('*d00.fits*', 'DARK')


@pytest.mark.e2e
@pytest.mark.master_flat
class TestMasterFlatCreation:
    @pytest.fixture(autouse=True)
    @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)
    def stack_flat_frames(self, mock_lake):
        run_reduce_individual_frames('*w00.fits*')
        mark_frames_as_good('*w91.fits*')
        stack_calibrations('lampflat')

    def test_if_stacked_flat_frame_was_created(self):
        check_if_individual_frames_exist('*w00*')
        run_check_if_stacked_calibrations_were_created('*w00.fits*', 'lampflat')
        run_check_if_stacked_calibrations_have_extensions('lampflat', ['TRACES', 'PROFILE', 'BLAZE'])
        run_check_if_stacked_calibrations_are_in_db('*w00.fits*', 'LAMPFLAT')


@pytest.mark.e2e
@pytest.mark.master_arc
class TestMasterArcCreation:
    @pytest.fixture(autouse=True)
    @mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)
    def stack_arc_frames(self, mock_lake):
        run_reduce_individual_frames('*a00.fits*')
        mark_frames_as_good('*a91.fits*')
        stack_calibrations('double')

    def test_if_stacked_arc_frame_was_created(self):
        check_if_individual_frames_exist('*a00*')
        run_check_if_stacked_calibrations_were_created('*a00.fits*', 'double')
        run_check_if_stacked_calibrations_have_extensions('double', ['WAVELENGTH', 'FEATURES'])
        run_check_if_stacked_calibrations_are_in_db('*a00.fits*', 'DOUBLE')


@pytest.mark.e2e
@pytest.mark.science_frames
class TestScienceFrameProcessing:
    @pytest.fixture(autouse=True)
    def process_frames(self):
        run_reduce_individual_frames('*e00.fits*')

    def test_if_science_frames_were_created(self):
        check_if_individual_frames_exist('*e00.fits*')
        check_extracted_spectra('*e91.fits*', '1DSPEC', ['wavelength', 'flux', 'uncertainty'])

1	import pytest	1✔
2	from banzai.tests.utils import FakeResponse, get_min_and_max_dates	1✔
3	from banzai_nres import settings	1✔
4	import os	1✔
5	import mock	1✔
6	import numpy as np	1✔
7	from banzai.utils import file_utils	1✔
8	import time	1✔
9	from glob import glob	1✔
10	import pkg_resources	1✔
11	from banzai.celery import app, schedule_calibration_stacking	1✔
12	from banzai.dbs import get_session	1✔
13	from banzai import dbs	1✔
14	from types import ModuleType	1✔
15	from datetime import datetime	1✔
16	from dateutil.parser import parse	1✔
17	from astropy.io import fits	1✔
18	from astropy.table import Table	1✔
19	from banzai.utils import fits_utils	1✔
20
21	import logging	1✔
22
23	logger = logging.getLogger('banzai')	1✔
24
25	TEST_PACKAGE = 'banzai_nres.tests'	1✔
26
27	DATA_ROOT = os.path.join(os.sep, 'archive', 'engineering')	1✔
28
29	SITES = [os.path.basename(site_path) for site_path in glob(os.path.join(DATA_ROOT, '???'))]	1✔
30	INSTRUMENTS = [os.path.join(site, os.path.basename(instrument_path)) for site in SITES	1✔
31	for instrument_path in glob(os.path.join(os.path.join(DATA_ROOT, site, '*')))]
32
33	DAYS_OBS = [os.path.join(instrument, os.path.basename(dayobs_path)) for instrument in INSTRUMENTS	1✔
34	for dayobs_path in glob(os.path.join(DATA_ROOT, instrument, '201*'))]
35
36	TEST_PACKAGE = 'banzai_nres.tests'	1✔
37	CONFIGDB_FILENAME = pkg_resources.resource_filename('banzai_nres.tests', 'data/configdb_example.json')	1✔
38
39
40
41	def observation_portal_side_effect(args, *kwargs):	1✔
42	site = kwargs['params']['site']	×
43	start = datetime.strftime(parse(kwargs['params']['start_after']).replace(tzinfo=None).date(), '%Y%m%d')	×
44	filename = 'test_obs_portal_response_{site}_{start}.json'.format(site=site, start=start)	×
45	filename = pkg_resources.resource_filename('banzai_nres.tests', f'data/{filename}')	×
46	return FakeResponse(filename)	×
47
48
49	def get_instrument_ids(db_address, names):	1✔
50	with get_session(db_address) as db_session:	×
51	instruments = []	×
52	for name in names:	×
53	criteria = dbs.Instrument.name == name	×
54	instruments.extend(db_session.query(dbs.Instrument).filter(criteria).all())	×
55	return [instrument.id for instrument in instruments]	×
56
57
58	def celery_join():	1✔
59	celery_inspector = app.control.inspect()	×
60	log_counter = 0	×
61	while True:	×
62	time.sleep(1)	×
63	log_counter += 1	×
64	if log_counter % 5 == 0:	×
65	logger.info('Processing: ' + '. ' * (log_counter // 5))	×
66	queues = [celery_inspector.active(), celery_inspector.scheduled(), celery_inspector.reserved()]	×
67	if any([queue is None or 'celery@banzai-celery-worker' not in queue for queue in queues]):	×
68	logger.warning('No valid celery queues were detected, retrying...', extra_tags={'queues': queues})	×
69	# Reset the celery connection
70	celery_inspector = app.control.inspect()	×
71	continue	×
72	if all([len(queue['celery@banzai-celery-worker']) == 0 for queue in queues]):	×
73	break	×
74
75
76	def run_reduce_individual_frames(raw_filenames):	1✔
77	logger.info('Reducing individual frames for filenames: {filenames}'.format(filenames=raw_filenames))	×
78	for day_obs in DAYS_OBS:	×
79	raw_path = os.path.join(DATA_ROOT, day_obs, 'raw')	×
80	for filename in glob(os.path.join(raw_path, raw_filenames)):	×
81	file_utils.post_to_archive_queue(filename, os.getenv('FITS_BROKER'),	×
82	exchange_name=os.getenv('FITS_EXCHANGE'))
83	celery_join()	×
84	logger.info('Finished reducing individual frames for filenames: {filenames}'.format(filenames=raw_filenames))	×
85
86
87	def stack_calibrations(frame_type):	1✔
88	logger.info('Stacking calibrations for frame type: {frame_type}'.format(frame_type=frame_type))	×
89	logger.info('Stacking calibrations for frame type: {frame_type}'.format(frame_type=frame_type))	×
90	for day_obs in DAYS_OBS:	×
91	site, camera, dayobs = day_obs.split('/')	×
92	timezone = dbs.get_timezone(site, db_address=os.environ['DB_ADDRESS'])	×
93	min_date, max_date = get_min_and_max_dates(timezone, dayobs=dayobs)	×
94	runtime_context = dict(processed_path=DATA_ROOT, log_level='debug', post_to_archive=False,	×
95	post_to_elasticsearch=False, fpack=True, reduction_level=91,
96	db_address=os.environ['DB_ADDRESS'], elasticsearch_qc_index='banzai_qc',
97	elasticsearch_url='http://elasticsearch.lco.gtn:9200', elasticsearch_doc_type='qc',
98	no_bpm=False, ignore_schedulability=True, use_only_older_calibrations=False,
99	preview_mode=False, max_tries=5, broker_url=os.getenv('FITS_BROKER'),
100	no_file_cache=False)
101	for setting in dir(settings):	×
102	if '__' != setting[:2] and not isinstance(getattr(settings, setting), ModuleType):	×
103	runtime_context[setting] = getattr(settings, setting)	×
104	schedule_calibration_stacking(site, runtime_context, min_date=min_date, max_date=max_date,	×
105	frame_types=[frame_type])
106	celery_join()	×
107	logger.info('Finished stacking calibrations for frame type: {frame_type}'.format(frame_type=frame_type))	×
108
109
110	def mark_frames_as_good(raw_filenames):	1✔
111	logger.info('Marking frames as good for filenames: {filenames}'.format(filenames=raw_filenames))	×
112	for day_obs in DAYS_OBS:	×
113	for filename in glob(os.path.join(DATA_ROOT, day_obs, 'processed', raw_filenames)):	×
114	dbs.mark_frame(os.path.basename(filename), "good", db_address=os.environ['DB_ADDRESS'])	×
115	logger.info('Finished marking frames as good for filenames: {filenames}'.format(filenames=raw_filenames))	×
116
117
118	def get_expected_number_of_calibrations(raw_filenames, calibration_type):	1✔
119	number_of_stacks_that_should_have_been_created = 0	×
120	for day_obs in DAYS_OBS:	×
121	raw_filenames_for_this_dayobs = glob(os.path.join(DATA_ROOT, day_obs, 'raw', raw_filenames))	×
122	if calibration_type.lower() == 'lampflat' or calibration_type.lower() == 'double':	×
123	# Group by fibers lit if we are stacking lampflats or doubles (arc frames)
124	observed_fibers = []	×
125	for raw_filename in raw_filenames_for_this_dayobs:	×
126	cal_hdu = fits.open(raw_filename)	×
127	observed_fibers.append(cal_hdu[1].header.get('OBJECTS'))	×
128	cal_hdu.close()	×
129	observed_fibers = set(observed_fibers)	×
130	number_of_stacks_that_should_have_been_created += len(observed_fibers)	×
131	else:
132	# Just one calibration per night
133	if len(raw_filenames_for_this_dayobs) > 0:	×
134	number_of_stacks_that_should_have_been_created += 1	×
135	return number_of_stacks_that_should_have_been_created	×
136
137
138	def check_if_individual_frames_exist(filenames):	1✔
139	for day_obs in DAYS_OBS:	×
140	raw_files = glob(os.path.join(DATA_ROOT, day_obs, 'raw', filenames))	×
141	processed_files = glob(os.path.join(DATA_ROOT, day_obs, 'processed', filenames.replace('00', '91')))	×
142	assert len(raw_files) == len(processed_files)	×
143
144
145	def run_check_if_stacked_calibrations_were_created(raw_filenames, calibration_type):	1✔
146	created_stacked_calibrations = []	×
147	number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_filenames, calibration_type)	×
148	for day_obs in DAYS_OBS:	×
149	created_stacked_calibrations += glob(os.path.join(DATA_ROOT, day_obs, 'processed',	×
150	'' + calibration_type.lower() + '.fits*'))
151	assert number_of_stacks_that_should_have_been_created > 0	×
152	assert len(created_stacked_calibrations) == number_of_stacks_that_should_have_been_created	×
153
154
155	def run_check_if_stacked_calibrations_have_extensions(calibration_type, extensions_to_check):	1✔
156	created_stacked_calibrations = []	×
157	for day_obs in DAYS_OBS:	×
158	created_stacked_calibrations += glob(os.path.join(DATA_ROOT, day_obs, 'processed',	×
159	'' + calibration_type.lower() + '.fits*'))
160	for cal in created_stacked_calibrations:	×
161	hdulist = fits.open(cal)	×
162	extnames = [hdulist[i].header.get('extname', None) for i in range(len(hdulist))]	×
163	for ext in extensions_to_check:	×
164	logger.info(f'checking if {ext} is in the saved extensions of {cal}')	×
165	assert ext in extnames	×
166
167
168	def check_extracted_spectra(raw_filename, spec_extname, columns):	1✔
169	created_images = []	×
170	for day_obs in DAYS_OBS:	×
171	created_images += glob(os.path.join(DATA_ROOT, day_obs, 'processed', raw_filename))	×
172	for filename in created_images:	×
173	with fits.open(filename) as f:	×
174	hdu = fits_utils.unpack(f)	×
175	spectrum = Table(hdu[spec_extname].data)	×
176	for colname in columns:	×
177	assert colname in spectrum.colnames	×
178	assert not np.allclose(spectrum[colname], 0)	×
179	assert 'RV' in hdu[0].header	×
180
181
182	def run_check_if_stacked_calibrations_are_in_db(raw_filenames, calibration_type):	1✔
183	number_of_stacks_that_should_have_been_created = get_expected_number_of_calibrations(raw_filenames, calibration_type)	×
184	with dbs.get_session(os.environ['DB_ADDRESS']) as db_session:	×
185	calibrations_in_db = db_session.query(dbs.CalibrationImage).filter(dbs.CalibrationImage.type == calibration_type)	×
186	calibrations_in_db = calibrations_in_db.filter(dbs.CalibrationImage.is_master).all()	×
187	assert number_of_stacks_that_should_have_been_created > 0	×
188	assert len(calibrations_in_db) == number_of_stacks_that_should_have_been_created	×
189
190
191	@pytest.mark.e2e	1✔
192	@pytest.fixture(scope='module')	1✔
193	@mock.patch('banzai.dbs.requests.get', return_value=FakeResponse(CONFIGDB_FILENAME))	1✔
194	def init(configdb):
195	os.system(f'banzai_create_db --db-address={os.environ["DB_ADDRESS"]}')	×
196	dbs.populate_instrument_tables(db_address=os.environ["DB_ADDRESS"], configdb_address='http://fakeconfigdb')	×
197	os.system(f'banzai_add_site --site elp --latitude 30.67986944 --longitude -104.015175 --elevation 2027 --timezone -6 --db-address={os.environ["DB_ADDRESS"]}')	×
198	os.system(f'banzai_add_site --site lsc --latitude -30.1673833333 --longitude -70.8047888889 --elevation 2198 --timezone -4 --db-address={os.environ["DB_ADDRESS"]}')	×
199	os.system(f'banzai_add_instrument --site lsc --camera fl09 --name nres01 --instrument-type 1m0-NRES-SciCam --db-address={os.environ["DB_ADDRESS"]}')	×
200	os.system(f'banzai_add_instrument --site elp --camera fl17 --name nres02 --instrument-type 1m0-NRES-SciCam --db-address={os.environ["DB_ADDRESS"]}')	×
201	for instrument in INSTRUMENTS:	×
202	for bpm_filename in glob(os.path.join(DATA_ROOT, instrument, 'bpm/bpm')):	×
203	logger.info(f'adding bpm {bpm_filename} to the database')	×
204	os.system(f'banzai_nres_add_bpm --filename {bpm_filename} --db-address={os.environ["DB_ADDRESS"]}')	×
205
206
207	@pytest.mark.e2e	1✔
208	@pytest.mark.master_bias	1✔
209	class TestMasterBiasCreation:
210	@pytest.fixture(autouse=True)	1✔
211	@mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)	1✔
212	def stack_bias_frames(self, mock_lake, init):
213	run_reduce_individual_frames('b00.fits')	×
214	mark_frames_as_good('b91.fits')	×
215	stack_calibrations('bias')	×
216
217	def test_if_stacked_bias_frame_was_created(self):	1✔
218	check_if_individual_frames_exist('b00')	×
219	run_check_if_stacked_calibrations_were_created('b00.fits', 'bias')	×
220	run_check_if_stacked_calibrations_are_in_db('b00.fits', 'BIAS')	×
221
222
223	@pytest.mark.e2e	1✔
224	@pytest.mark.master_dark	1✔
225	class TestMasterDarkCreation:
226	@pytest.fixture(autouse=True)	1✔
227	@mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)	1✔
228	def stack_dark_frames(self, mock_lake):
229	run_reduce_individual_frames('d00.fits')	×
230	mark_frames_as_good('d91.fits')	×
231	stack_calibrations('dark')	×
232
233	def test_if_stacked_dark_frame_was_created(self):	1✔
234	check_if_individual_frames_exist('d00')	×
235	run_check_if_stacked_calibrations_were_created('d00.fits', 'dark')	×
236	run_check_if_stacked_calibrations_are_in_db('d00.fits', 'DARK')	×
237
238
239	@pytest.mark.e2e	1✔
240	@pytest.mark.master_flat	1✔
241	class TestMasterFlatCreation:
242	@pytest.fixture(autouse=True)	1✔
243	@mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)	1✔
244	def stack_flat_frames(self, mock_lake):
245	run_reduce_individual_frames('w00.fits')	×
246	mark_frames_as_good('w91.fits')	×
247	stack_calibrations('lampflat')	×
248
249	def test_if_stacked_flat_frame_was_created(self):	1✔
250	check_if_individual_frames_exist('w00')	×
251	run_check_if_stacked_calibrations_were_created('w00.fits', 'lampflat')	×
252	run_check_if_stacked_calibrations_have_extensions('lampflat', ['TRACES', 'PROFILE', 'BLAZE'])	×
253	run_check_if_stacked_calibrations_are_in_db('w00.fits', 'LAMPFLAT')	×
254
255
256	@pytest.mark.e2e	1✔
257	@pytest.mark.master_arc	1✔
258	class TestMasterArcCreation:
259	@pytest.fixture(autouse=True)	1✔
260	@mock.patch('banzai.utils.observation_utils.requests.get', side_effect=observation_portal_side_effect)	1✔
261	def stack_arc_frames(self, mock_lake):
262	run_reduce_individual_frames('a00.fits')	×
263	mark_frames_as_good('a91.fits')	×
264	stack_calibrations('double')	×
265
266	def test_if_stacked_arc_frame_was_created(self):	1✔
267	check_if_individual_frames_exist('a00')	×
268	run_check_if_stacked_calibrations_were_created('a00.fits', 'double')	×
269	run_check_if_stacked_calibrations_have_extensions('double', ['WAVELENGTH', 'FEATURES'])	×
270	run_check_if_stacked_calibrations_are_in_db('a00.fits', 'DOUBLE')	×
271
272
273	@pytest.mark.e2e	1✔
274	@pytest.mark.science_frames	1✔
275	class TestScienceFrameProcessing:
276	@pytest.fixture(autouse=True)	1✔
277	def process_frames(self):
278	run_reduce_individual_frames('e00.fits')	×
279
280	def test_if_science_frames_were_created(self):	1✔
281	check_if_individual_frames_exist('e00.fits')	×
282	check_extracted_spectra('e91.fits', '1DSPEC', ['wavelength', 'flux', 'uncertainty'])	×

LCOGT / banzai-nres / 593

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous