15788802466

Committed 20 Jun 2025 10:21PM UTC coverage: 37.029% (-2.1%) from 39.083%

Build # 15788802466

Build Type

Pull #2502

github

Committed by

weaverba137

Commit Message

fix more deprecation warnings

Pull Request Pull Request #2502: [WIP] NumPy 2 compatibility

Run Details

7 of 15 new or added lines in 11 files covered. (46.67%)

692 existing lines in 4 files now uncovered.

12422 of 33547 relevant lines covered (37.03%)

0.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/py/desispec/scripts/processingtable.py

"""
desispec.scripts.processingtable
================================

"""
import os
import sys
import numpy as np
import re
from astropy.table import Table, vstack
from astropy.io import fits
from desispec.io.util import parse_cameras, difference_camwords, validate_badamps
## Import some helper functions, you can see their definitions by uncomenting the bash shell command
from desispec.workflow.exptable import get_exposure_table_path, get_exposure_table_name, \
                                        night_to_month

from desispec.workflow.utils import define_variable_from_environment, listpath, pathjoin, get_printable_banner
from desispec.workflow.proctable import default_obstypes_for_proctable, get_processing_table_path, exptable_to_proctable, \
                                        get_processing_table_name
from desispec.workflow.tableio import load_table, write_table


def create_processing_tables(nights=None, night_range=None, exp_table_path=None, proc_table_path=None,
                             obstypes=None, overwrite_files=False, verbose=False, no_specprod_exptab=False,
                             exp_filetype='csv', prod_filetype='csv', joinsymb='|'):
    """
    Generates processing tables for the nights requested. Requires exposure tables to exist on disk.

    Args:
        nights: str, int, or comma separated list. The night(s) to generate procesing tables for.
        night_range: str, comma separated pair of nights in form YYYYMMDD,YYYYMMDD for first_night,last_night
                          specifying the beginning and end of a range of nights to be generated.
                          last_night should be inclusive.
        exp_table_path: str. Full path to where to exposure tables are stored, WITHOUT the monthly directory included.
        proc_table_path: str. Full path to where to processing tables to be written.
        obstypes: str or comma separated list of strings. The exposure OBSTYPE's that you want to include in the processing table.
        overwrite_files: boolean. Whether to overwrite processing tables if they exist. True overwrites.
        verbose: boolean. Whether to give verbose output information or not. True prints more information.
        no_specprod_exptab: boolean. Read exposure table in repository location rather than the SPECPROD location.
        exp_filetype: str. The file extension (without the '.') of the exposure tables.
        prod_filetype: str. The file extension (without the '.') of the processing tables.
        joinsymb: str. Symbol to use to indicate the separation of array values when converting to and from strings for
                       saving to csv. Default is highly advised and is '|'. Using a comma will break many things.

    Returns: Nothing

    Notes:
        Requires exposure tables to exist on disk. Either in the default location or at the location specified
        using the function arguments.
    """
    if nights is None and night_range is None:
        raise ValueError("Must specify either nights or night_range")
    elif nights is not None and night_range is not None:
        raise ValueError("Must only specify either nights or night_range, not both")

    if nights is None or nights == 'all':
        nights = list()
        for n in listpath(os.getenv('DESI_SPECTRO_DATA')):
            # - nights are 20YYMMDD
            if re.match(r'^20\d{6}$', n):
                nights.append(n)
    else:
        nights = [int(val.strip()) for val in nights.split(",")]

    nights = np.array(nights)

    if night_range is not None:
        if ',' not in night_range:
            raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")
        nightpair = night_range.split(',')
        if len(nightpair) != 2 or not nightpair[0].isnumeric() or not nightpair[1].isnumeric():
            raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")
        first_night, last_night = nightpair
        nights = nights[np.where(int(first_night) <= nights.astype(int))[0]]
        nights = nights[np.where(int(last_night) >= nights.astype(int))[0]]

    if obstypes is not None:
        obstypes = [ val.strip('\t ') for val in obstypes.split(",") ]
    else:
        obstypes = default_obstypes_for_proctable()

    ## Define where to find the data
    if exp_table_path is None:
        usespecprod = (not no_specprod_exptab)
        exp_table_path = get_exposure_table_path(night=None,usespecprod=usespecprod)

    ## Define where to save the data
    if proc_table_path is None:
        proc_table_path = get_processing_table_path()

    if type(nights) is str and nights == 'all':
        exptables = []
        for month in listpath(exp_table_path):
            exptables += listpath(exp_table_path, month)

        nights = np.unique(
            [file.split('_')[2].split('.')[0] for file in sorted(exptables) if '.' + exp_filetype in file]).astype(int)

    if verbose:
        print(f'Nights: {nights}')

    ## Make the save directory exists
    os.makedirs(exp_table_path, exist_ok=True)

    ## Make the save directory if it doesn't exist
    if not os.path.isdir(proc_table_path):
        print(f'Creating directory: {proc_table_path}')
        os.makedirs(proc_table_path)

    ## Create an astropy table for each night. Define the columns and datatypes, but leave each with 0 rows
    combined_table = Table()

    ## Loop over nights
    for night in nights:
        if verbose:
            print(get_printable_banner(input_str=night))
        else:
            print(f'Processing {night}')
        exptab_name = get_exposure_table_name(night=night, extension=exp_filetype)
        month = night_to_month(night)
        exptable = load_table(pathjoin(exp_table_path,month,exptab_name), process_mixins=False)

        if night == nights[0]:
            combined_table = exptable.copy()
        else:
            combined_table = vstack([combined_table, exptable])

    processing_table, unprocessed_table = exptable_to_proctable(combined_table, obstypes=obstypes)#,joinsymb=joinsymb)

    ## Save the tables
    proc_name = get_processing_table_name(extension=prod_filetype)
    unproc_name = proc_name.replace('processing', 'unprocessed')
    for tab, name in zip([processing_table, unprocessed_table], [proc_name, unproc_name]):
        if len(tab) > 0:
            pathname = pathjoin(proc_table_path, name)
            write_table(tab, pathname, overwrite=overwrite_files)
            print(f'Wrote file: {name}')

    print("Processing table generations complete")
    ## Flush the outputs
    sys.stdout.flush()
    sys.stderr.flush()

1	"""
2	desispec.scripts.processingtable
3	================================
4
5	"""
6	import os	×
7	import sys	×
8	import numpy as np	×
9	import re	×
10	from astropy.table import Table, vstack	×
11	from astropy.io import fits	×
12	from desispec.io.util import parse_cameras, difference_camwords, validate_badamps	×
13	## Import some helper functions, you can see their definitions by uncomenting the bash shell command
14	from desispec.workflow.exptable import get_exposure_table_path, get_exposure_table_name, \	×
15	night_to_month
16
17	from desispec.workflow.utils import define_variable_from_environment, listpath, pathjoin, get_printable_banner	×
18	from desispec.workflow.proctable import default_obstypes_for_proctable, get_processing_table_path, exptable_to_proctable, \	×
19	get_processing_table_name
20	from desispec.workflow.tableio import load_table, write_table	×
21
22
23	def create_processing_tables(nights=None, night_range=None, exp_table_path=None, proc_table_path=None,	×
24	obstypes=None, overwrite_files=False, verbose=False, no_specprod_exptab=False,
25	exp_filetype='csv', prod_filetype='csv', joinsymb='\|'):
26	"""
27	Generates processing tables for the nights requested. Requires exposure tables to exist on disk.
28
29	Args:
30	nights: str, int, or comma separated list. The night(s) to generate procesing tables for.
31	night_range: str, comma separated pair of nights in form YYYYMMDD,YYYYMMDD for first_night,last_night
32	specifying the beginning and end of a range of nights to be generated.
33	last_night should be inclusive.
34	exp_table_path: str. Full path to where to exposure tables are stored, WITHOUT the monthly directory included.
35	proc_table_path: str. Full path to where to processing tables to be written.
36	obstypes: str or comma separated list of strings. The exposure OBSTYPE's that you want to include in the processing table.
37	overwrite_files: boolean. Whether to overwrite processing tables if they exist. True overwrites.
38	verbose: boolean. Whether to give verbose output information or not. True prints more information.
39	no_specprod_exptab: boolean. Read exposure table in repository location rather than the SPECPROD location.
40	exp_filetype: str. The file extension (without the '.') of the exposure tables.
41	prod_filetype: str. The file extension (without the '.') of the processing tables.
42	joinsymb: str. Symbol to use to indicate the separation of array values when converting to and from strings for
43	saving to csv. Default is highly advised and is '\|'. Using a comma will break many things.
44
45	Returns: Nothing
46
47	Notes:
48	Requires exposure tables to exist on disk. Either in the default location or at the location specified
49	using the function arguments.
50	"""
51	if nights is None and night_range is None:	×
52	raise ValueError("Must specify either nights or night_range")	×
53	elif nights is not None and night_range is not None:	×
54	raise ValueError("Must only specify either nights or night_range, not both")	×
55
56	if nights is None or nights == 'all':	×
57	nights = list()	×
58	for n in listpath(os.getenv('DESI_SPECTRO_DATA')):	×
59	# - nights are 20YYMMDD
NEW 60	if re.match(r'^20\d{6}$', n):	×
61	nights.append(n)	×
62	else:
63	nights = [int(val.strip()) for val in nights.split(",")]	×
64
65	nights = np.array(nights)	×
66
67	if night_range is not None:	×
68	if ',' not in night_range:	×
69	raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")	×
70	nightpair = night_range.split(',')	×
71	if len(nightpair) != 2 or not nightpair[0].isnumeric() or not nightpair[1].isnumeric():	×
72	raise ValueError("night_range must be a comma separated pair of nights in form YYYYMMDD,YYYYMMDD")	×
73	first_night, last_night = nightpair	×
74	nights = nights[np.where(int(first_night) <= nights.astype(int))[0]]	×
75	nights = nights[np.where(int(last_night) >= nights.astype(int))[0]]	×
76
77	if obstypes is not None:	×
78	obstypes = [ val.strip('\t ') for val in obstypes.split(",") ]	×
79	else:
80	obstypes = default_obstypes_for_proctable()	×
81
82	## Define where to find the data
83	if exp_table_path is None:	×
84	usespecprod = (not no_specprod_exptab)	×
85	exp_table_path = get_exposure_table_path(night=None,usespecprod=usespecprod)	×
86
87	## Define where to save the data
88	if proc_table_path is None:	×
89	proc_table_path = get_processing_table_path()	×
90
91	if type(nights) is str and nights == 'all':	×
92	exptables = []	×
93	for month in listpath(exp_table_path):	×
94	exptables += listpath(exp_table_path, month)	×
95
96	nights = np.unique(	×
97	[file.split('_')[2].split('.')[0] for file in sorted(exptables) if '.' + exp_filetype in file]).astype(int)
98
99	if verbose:	×
100	print(f'Nights: {nights}')	×
101
102	## Make the save directory exists
103	os.makedirs(exp_table_path, exist_ok=True)	×
104
105	## Make the save directory if it doesn't exist
106	if not os.path.isdir(proc_table_path):	×
107	print(f'Creating directory: {proc_table_path}')	×
108	os.makedirs(proc_table_path)	×
109
110	## Create an astropy table for each night. Define the columns and datatypes, but leave each with 0 rows
111	combined_table = Table()	×
112
113	## Loop over nights
114	for night in nights:	×
115	if verbose:	×
116	print(get_printable_banner(input_str=night))	×
117	else:
118	print(f'Processing {night}')	×
119	exptab_name = get_exposure_table_name(night=night, extension=exp_filetype)	×
120	month = night_to_month(night)	×
121	exptable = load_table(pathjoin(exp_table_path,month,exptab_name), process_mixins=False)	×
122
123	if night == nights[0]:	×
124	combined_table = exptable.copy()	×
125	else:
126	combined_table = vstack([combined_table, exptable])	×
127
128	processing_table, unprocessed_table = exptable_to_proctable(combined_table, obstypes=obstypes)#,joinsymb=joinsymb)	×
129
130	## Save the tables
131	proc_name = get_processing_table_name(extension=prod_filetype)	×
132	unproc_name = proc_name.replace('processing', 'unprocessed')	×
133	for tab, name in zip([processing_table, unprocessed_table], [proc_name, unproc_name]):	×
134	if len(tab) > 0:	×
135	pathname = pathjoin(proc_table_path, name)	×
136	write_table(tab, pathname, overwrite=overwrite_files)	×
137	print(f'Wrote file: {name}')	×
138
139	print("Processing table generations complete")	×
140	## Flush the outputs
141	sys.stdout.flush()	×
142	sys.stderr.flush()	×

desihub / desispec / 15788802466

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous