19150063725

Committed 06 Nov 2025 09:14PM UTC coverage: 37.719% (+0.7%) from 37.002%

Build # 19150063725

Build Type

Pull #2521

github

Committed by

web-flow

Commit Message

Merge b3bf9e090 into 6a90a0547

Pull Request Pull Request #2521: Add redshift QA scripts

Run Details

12990 of 34439 relevant lines covered (37.72%)

0.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/py/desispec/scripts/exposuretable.py

"""
desispec.scripts.exposuretable
==============================

"""
import os
import sys
import numpy as np
import re
from astropy.table import Table
from astropy.io import fits
## Import some helper functions, you can see their definitions by uncomenting the bash shell command
from desispec.io.util import parse_cameras, difference_camwords, validate_badamps
from desispec.workflow.exptable import summarize_exposure, default_obstypes_for_exptable, \
                                       instantiate_exposure_table, get_exposure_table_column_defs, \
                                       get_exposure_table_path, get_exposure_table_name, \
                                       night_to_month
from desispec.workflow.utils import define_variable_from_environment, listpath, pathjoin, get_printable_banner
from desispec.workflow.tableio import write_table



def create_exposure_tables(nights=None, night_range=None, path_to_data=None, exp_table_path=None, obstypes=None, \
                           exp_filetype='csv', cameras=None, bad_cameras=None, badamps=None,
                           verbose=False, no_specprod=False, overwrite_files=False):
    """
    Generates processing tables for the nights requested. Requires exposure tables to exist on disk.

    Args:
        nights: str, int, or comma separated list. The night(s) to generate procesing tables for.
        night_range: str, comma separated pair of nights in form YYYYMMDD,YYYYMMDD for first_night,last_night
                          specifying the beginning and end of a range of nights to be generated.
                          last_night should be inclusive.
        path_to_data: str. The path to the raw data and request*.json and manifest* files.
        exp_table_path: str. Full path to where to exposure tables should be saved, WITHOUT the monthly directory included.
        obstypes: str or comma separated list of strings. The exposure OBSTYPE's that you want to include in the exposure table.
        exp_filetype: str. The file extension (without the '.') of the exposure tables.
        verbose: boolean. Whether to give verbose output information or not. True prints more information.
        no_specprod: boolean. Create exposure table in repository location rather than the SPECPROD location
        overwrite_files: boolean. Whether to overwrite processing tables if they exist. True overwrites.
        cameras: str. Explicitly define the cameras for which you want to reduce the data. Should be a comma separated
                      list. Only numbers assumes you want to reduce r, b, and z for that camera. Otherwise specify
                      separately [brz][0-9].
        bad_cameras: str. Explicitly define the cameras that you don't want to reduce the data. Should be a comma
                          separated list. Only numbers assumes you want to reduce r, b, and z for that camera.
                          Otherwise specify separately [brz][0-9].
        badamps: str. Define amplifiers that you know to be bad and should not be processed. Should be a list separated
                      by comma or semicolon. Saved list will converted to semicolons. Each entry should be of the
                      form {camera}{spectrograph}{amp}, i.e. [brz][0-9][A-D].
    Returns: Nothing
    """
    if nights is None and night_range is None:
        raise ValueError("Must specify either nights or night_range")
    elif nights is not None and night_range is not None:
        raise ValueError("Must only specify either nights or night_range, not both")

    if nights is None or nights=='all':
        nights = list()
        for n in listpath(os.getenv('DESI_SPECTRO_DATA')):
            #- nights are 20YYMMDD
            if re.match(r'^20\d{6}$', n):
                nights.append(n)
    else:
        nights = [ int(val.strip()) for val in nights.split(",") ]

    nights = np.array(nights)

    if night_range is not None:
        if ',' not in night_range:
            raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")
        nightpair = night_range.split(',')
        if len(nightpair) != 2 or not nightpair[0].isnumeric() or not nightpair[1].isnumeric():
            raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")
        first_night, last_night = nightpair
        nights = nights[np.where(int(first_night)<=nights.astype(int))[0]]
        nights = nights[np.where(int(last_night)>=nights.astype(int))[0]]

    if obstypes is not None:
        obstypes = [ val.strip('\t ') for val in obstypes.split(",") ]
    else:
        obstypes = default_obstypes_for_exptable()

    print("Nights: ", nights)
    print("Obs types: ", obstypes)

    ## Deal with cameras and amps, if given
    camword = cameras
    if camword != '':
        camword = parse_cameras(camword)
    badcamword = bad_cameras
    if badcamword != '':
        badcamword = parse_cameras(badcamword)

    ## Warn people if changing camword
    finalcamword = 'a0123456789'
    if camword is not None and badcamword is None:
        badcamword = difference_camwords(finalcamword,camword)
        finalcamword = camword
    elif camword is not None and badcamword is not None:
        finalcamword = difference_camwords(camword, badcamword)
        badcamword = difference_camwords('a0123456789', finalcamword)
    elif badcamword is not None:
        finalcamword = difference_camwords(finalcamword,badcamword)
    else:
        badcamword = ''

    if badcamword != '':
        ## Inform the user what will be done with it.
        print(f"Modifying camword of data to be processed with badcamword: {badcamword}. " + \
              f"Camword to be processed: {finalcamword}")

    ## Make sure badamps is formatted properly
    if badamps is None:
        badamps = ''
    else:
        badamps = validate_badamps(badamps)

    ## Define where to find the data
    if path_to_data is None:
        path_to_data = define_variable_from_environment(env_name='DESI_SPECTRO_DATA',
                                                        var_descr="The data path")

    ## Define where to save the data
    usespecprod = (not no_specprod)
    if exp_table_path is None:
        exp_table_path = get_exposure_table_path(night=None,usespecprod=usespecprod)

    ## Make the save directory exists
    os.makedirs(exp_table_path, exist_ok=True)

    ## Loop over nights
    colnames, coltypes, coldefaults = get_exposure_table_column_defs(return_default_values=True)
    nights_with_data = listpath(path_to_data)
    for night in nights:
        if str(night) not in nights_with_data:
            print(f'Night: {night} not in data directory {path_to_data}. Skipping')
            continue

        print(get_printable_banner(input_str=night))

        ## Create an astropy exposure table for the night
        nightly_tab = instantiate_exposure_table()

        ## Loop through all exposures on disk
        for exp in listpath(path_to_data,str(night)):
            rowdict = summarize_exposure(path_to_data, night=night, exp=exp, obstypes=obstypes, \
                                         colnames=colnames, coldefaults=coldefaults, verbosely=verbose)
            if rowdict is not None and type(rowdict) is not str:
                rowdict['BADCAMWORD'] = badcamword
                rowdict['BADAMPS'] = badamps

                ## Add the dictionary of column values as a new row
                nightly_tab.add_row(rowdict)
            if verbose:
                print("Rowdict:\n",rowdict,"\n\n")

        if len(nightly_tab) > 0:
            month = night_to_month(night)
            exptab_path = pathjoin(exp_table_path,month)
            os.makedirs(exptab_path,exist_ok=True)
            exptab_name = get_exposure_table_name(night, extension=exp_filetype)
            exptab_name = pathjoin(exptab_path, exptab_name)
            write_table(nightly_tab, exptab_name, overwrite=overwrite_files)
        else:
            print('No rows to write to a file.')

        print("Exposure table generations complete")
        ## Flush the outputs
        sys.stdout.flush()
        sys.stderr.flush()

1	"""
2	desispec.scripts.exposuretable
3	==============================
4
5	"""
6	import os	×
7	import sys	×
8	import numpy as np	×
9	import re	×
10	from astropy.table import Table	×
11	from astropy.io import fits	×
12	## Import some helper functions, you can see their definitions by uncomenting the bash shell command
13	from desispec.io.util import parse_cameras, difference_camwords, validate_badamps	×
14	from desispec.workflow.exptable import summarize_exposure, default_obstypes_for_exptable, \	×
15	instantiate_exposure_table, get_exposure_table_column_defs, \
16	get_exposure_table_path, get_exposure_table_name, \
17	night_to_month
18	from desispec.workflow.utils import define_variable_from_environment, listpath, pathjoin, get_printable_banner	×
19	from desispec.workflow.tableio import write_table	×
20
21
22
23	def create_exposure_tables(nights=None, night_range=None, path_to_data=None, exp_table_path=None, obstypes=None, \	×
24	exp_filetype='csv', cameras=None, bad_cameras=None, badamps=None,
25	verbose=False, no_specprod=False, overwrite_files=False):
26	"""
27	Generates processing tables for the nights requested. Requires exposure tables to exist on disk.
28
29	Args:
30	nights: str, int, or comma separated list. The night(s) to generate procesing tables for.
31	night_range: str, comma separated pair of nights in form YYYYMMDD,YYYYMMDD for first_night,last_night
32	specifying the beginning and end of a range of nights to be generated.
33	last_night should be inclusive.
34	path_to_data: str. The path to the raw data and request.json and manifest files.
35	exp_table_path: str. Full path to where to exposure tables should be saved, WITHOUT the monthly directory included.
36	obstypes: str or comma separated list of strings. The exposure OBSTYPE's that you want to include in the exposure table.
37	exp_filetype: str. The file extension (without the '.') of the exposure tables.
38	verbose: boolean. Whether to give verbose output information or not. True prints more information.
39	no_specprod: boolean. Create exposure table in repository location rather than the SPECPROD location
40	overwrite_files: boolean. Whether to overwrite processing tables if they exist. True overwrites.
41	cameras: str. Explicitly define the cameras for which you want to reduce the data. Should be a comma separated
42	list. Only numbers assumes you want to reduce r, b, and z for that camera. Otherwise specify
43	separately [brz][0-9].
44	bad_cameras: str. Explicitly define the cameras that you don't want to reduce the data. Should be a comma
45	separated list. Only numbers assumes you want to reduce r, b, and z for that camera.
46	Otherwise specify separately [brz][0-9].
47	badamps: str. Define amplifiers that you know to be bad and should not be processed. Should be a list separated
48	by comma or semicolon. Saved list will converted to semicolons. Each entry should be of the
49	form {camera}{spectrograph}{amp}, i.e. [brz][0-9][A-D].
50	Returns: Nothing
51	"""
52	if nights is None and night_range is None:	×
53	raise ValueError("Must specify either nights or night_range")	×
54	elif nights is not None and night_range is not None:	×
55	raise ValueError("Must only specify either nights or night_range, not both")	×
56
57	if nights is None or nights=='all':	×
58	nights = list()	×
59	for n in listpath(os.getenv('DESI_SPECTRO_DATA')):	×
60	#- nights are 20YYMMDD
61	if re.match(r'^20\d{6}$', n):	×
62	nights.append(n)	×
63	else:
64	nights = [ int(val.strip()) for val in nights.split(",") ]	×
65
66	nights = np.array(nights)	×
67
68	if night_range is not None:	×
69	if ',' not in night_range:	×
70	raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")	×
71	nightpair = night_range.split(',')	×
72	if len(nightpair) != 2 or not nightpair[0].isnumeric() or not nightpair[1].isnumeric():	×
73	raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")	×
74	first_night, last_night = nightpair	×
75	nights = nights[np.where(int(first_night)<=nights.astype(int))[0]]	×
76	nights = nights[np.where(int(last_night)>=nights.astype(int))[0]]	×
77
78	if obstypes is not None:	×
79	obstypes = [ val.strip('\t ') for val in obstypes.split(",") ]	×
80	else:
81	obstypes = default_obstypes_for_exptable()	×
82
83	print("Nights: ", nights)	×
84	print("Obs types: ", obstypes)	×
85
86	## Deal with cameras and amps, if given
87	camword = cameras	×
88	if camword != '':	×
89	camword = parse_cameras(camword)	×
90	badcamword = bad_cameras	×
91	if badcamword != '':	×
92	badcamword = parse_cameras(badcamword)	×
93
94	## Warn people if changing camword
95	finalcamword = 'a0123456789'	×
96	if camword is not None and badcamword is None:	×
97	badcamword = difference_camwords(finalcamword,camword)	×
98	finalcamword = camword	×
99	elif camword is not None and badcamword is not None:	×
100	finalcamword = difference_camwords(camword, badcamword)	×
101	badcamword = difference_camwords('a0123456789', finalcamword)	×
102	elif badcamword is not None:	×
103	finalcamword = difference_camwords(finalcamword,badcamword)	×
104	else:
105	badcamword = ''	×
106
107	if badcamword != '':	×
108	## Inform the user what will be done with it.
109	print(f"Modifying camword of data to be processed with badcamword: {badcamword}. " + \	×
110	f"Camword to be processed: {finalcamword}")
111
112	## Make sure badamps is formatted properly
113	if badamps is None:	×
114	badamps = ''	×
115	else:
116	badamps = validate_badamps(badamps)	×
117
118	## Define where to find the data
119	if path_to_data is None:	×
120	path_to_data = define_variable_from_environment(env_name='DESI_SPECTRO_DATA',	×
121	var_descr="The data path")
122
123	## Define where to save the data
124	usespecprod = (not no_specprod)	×
125	if exp_table_path is None:	×
126	exp_table_path = get_exposure_table_path(night=None,usespecprod=usespecprod)	×
127
128	## Make the save directory exists
129	os.makedirs(exp_table_path, exist_ok=True)	×
130
131	## Loop over nights
132	colnames, coltypes, coldefaults = get_exposure_table_column_defs(return_default_values=True)	×
133	nights_with_data = listpath(path_to_data)	×
134	for night in nights:	×
135	if str(night) not in nights_with_data:	×
136	print(f'Night: {night} not in data directory {path_to_data}. Skipping')	×
137	continue	×
138
139	print(get_printable_banner(input_str=night))	×
140
141	## Create an astropy exposure table for the night
142	nightly_tab = instantiate_exposure_table()	×
143
144	## Loop through all exposures on disk
145	for exp in listpath(path_to_data,str(night)):	×
146	rowdict = summarize_exposure(path_to_data, night=night, exp=exp, obstypes=obstypes, \	×
147	colnames=colnames, coldefaults=coldefaults, verbosely=verbose)
148	if rowdict is not None and type(rowdict) is not str:	×
149	rowdict['BADCAMWORD'] = badcamword	×
150	rowdict['BADAMPS'] = badamps	×
151
152	## Add the dictionary of column values as a new row
153	nightly_tab.add_row(rowdict)	×
154	if verbose:	×
155	print("Rowdict:\n",rowdict,"\n\n")	×
156
157	if len(nightly_tab) > 0:	×
158	month = night_to_month(night)	×
159	exptab_path = pathjoin(exp_table_path,month)	×
160	os.makedirs(exptab_path,exist_ok=True)	×
161	exptab_name = get_exposure_table_name(night, extension=exp_filetype)	×
162	exptab_name = pathjoin(exptab_path, exptab_name)	×
163	write_table(nightly_tab, exptab_name, overwrite=overwrite_files)	×
164	else:
165	print('No rows to write to a file.')	×
166
167	print("Exposure table generations complete")	×
168	## Flush the outputs
169	sys.stdout.flush()	×
170	sys.stderr.flush()	×

desihub / desispec / 19150063725

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous