15789610751

Committed 20 Jun 2025 11:37PM UTC coverage: 37.029% (-2.1%) from 39.083%

Build # 15789610751

Build Type

Pull #2502

github

Committed by

weaverba137

Commit Message

update change log

Pull Request Pull Request #2502: [WIP] NumPy 2 compatibility

Run Details

7 of 15 new or added lines in 11 files covered. (46.67%)

692 existing lines in 4 files now uncovered.

12422 of 33547 relevant lines covered (37.03%)

0.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/py/desispec/scripts/reformat_proctables.py

"""
desispec.scripts.reformat_proctables
====================================

"""
import argparse
import os
import glob
import sys
import numpy as np
import re
import time
from astropy.table import Table

from desispec.io.meta import findfile
from desispec.workflow.proctable import get_processing_table_column_defs
from desispec.workflow.utils import define_variable_from_environment, listpath, \
                                    pathjoin
from desispec.workflow.tableio import write_table, load_table
from desispec.scripts.exposuretable import create_exposure_tables


def get_parser():
    """
    Creates an arguments parser for the desi_reformat_processing_tables script
    """
    parser = argparse.ArgumentParser(usage = "{prog} [options]")
    parser.add_argument("-n", "--nights", type=str,  default=None, help="nights as comma separated string")
    parser.add_argument("--night-range", type=str, default=None, help="comma separated pair of nights in form YYYYMMDD,YYYYMMDD"+\
                                                                      "for first_night,last_night specifying the beginning"+\
                                                                      "and end of a range of nights to be generated. "+\
                                                                      "last_night should be inclusive.")
    parser.add_argument("--orig-filetype", type=str, default='csv', help="format type for original exposure tables")
    parser.add_argument("--out-filetype", type=str, default='csv', help="format type for output exposure tables")
    parser.add_argument("--dry-run", action="store_true",
                        help="Perform a dry run, printing the changes that would be made and the final output table "+
                             "but not overwriting the actual files on disk.")
    return parser

def reformat_processing_tables(nights=None, night_range=None, orig_filetype='csv',
                           out_filetype='csv', dry_run=False):
    """
    Generates updated processing tables for the nights requested. Requires
    a current processing table to exist on disk.

    Args:
        nights: str, int, or comma separated list. The night(s) to generate
                                                   processing tables for.
        night_range: str. comma separated pair of nights in form
                          YYYYMMDD,YYYYMMDD for first_night,last_night
                          specifying the beginning and end of a range of
                          nights to be generated. first_night and last_night are
                          inclusive.
        orig_filetype: str. The file extension (without the '.') of the processing
                            tables.
        out_filetype: str. The file extension for the outputted processing tables
                           (without the '.').

    Returns:
        Nothing
    """
    # log = get_logger()
    ## Make sure user specified what nights to run on
    if nights is None and night_range is None:
        raise ValueError("Must specify either nights or night_range."
                         +" To process all nights give nights=all")

    ## Get all nights in 2020's with data
    proctab_template = findfile('proctable', night=99999999)
    proctab_template = proctab_template.replace('99999999', '202[0-9][01][0-9][0-3][0-9]')
    proctab_template = proctab_template.replace('.csv', f'.{orig_filetype}')
    nights_with_proctables = list()
    for ptabfn in glob.glob(proctab_template):
        ## nights are 20YYMMDD
        matches = re.findall(r'20\d{6}', os.path.basename(ptabfn))
        if len(matches) == 1:
            n = int(matches[0])
            nights_with_proctables.append(n)
        else:
            print(f"Couldn't parse a night from proctable file: {ptabfn}")

    ## If unpecified or given "all", set nights to all nights with data
    check_night = False
    if nights is None or nights == 'all':
        nights = nights_with_proctables
        ## No need to check nights since derived from disk
    else:
        nights = [int(val.strip()) for val in nights.split(",")]
        ## If nights are specified, make sure we check that there is actually data
        check_night = True
    nights = np.sort(nights)

    ## If user specified a night range, cut nights to that range of dates
    if night_range is not None:
        if ',' not in night_range:
            raise ValueError("night_range must be a comma separated pair of "
                             + "nights in form YYYYMMDD,YYYYMMDD")
        nightpair = night_range.split(',')
        if len(nightpair) != 2 or not nightpair[0].isnumeric() \
                or not nightpair[1].isnumeric():
            raise ValueError("night_range must be a comma separated pair of "
                             + "nights in form YYYYMMDD,YYYYMMDD")
        first_night, last_night = nightpair
        nights = nights[np.where(int(first_night) <= nights.astype(int))[0]]
        nights = nights[np.where(int(last_night) >= nights.astype(int))[0]]

    ## Get current set of expected columns
    ptab_cols, ptab_dtypes, ptab_defs = get_processing_table_column_defs(return_default_values=True)
    ptab_cols, ptab_dtypes = np.array(ptab_cols), np.array(ptab_dtypes)

    ## Tell user the final list of nights and starting looping over them
    print("Nights: ", nights)
    for night in nights:
        if check_night and night not in nights_with_proctables:
            print(f"Night {night} doesn't have a processing table: Skipping.")
            continue

        ## If the processing table doesn't exist, skip, since we are updating
        ## not generating.
        orig_pathname = findfile('proctable', night=night).replace('.csv', f'.{orig_filetype}')
        if not os.path.exists(orig_pathname):
            print(f'Could not find processing table for night={night} at:'
                  + f' {orig_pathname}. Skipping this night.')
            continue

        ## Load the old and new tables to compare
        origtable = load_table(orig_pathname, tabletype='proctab')
        curr_colnames = np.array(list(origtable.colnames))
        expected_cols = np.isin(curr_colnames, ptab_cols)
        found_cols = np.isin(ptab_cols, curr_colnames)

        ## If everything is present, don't try to do anything
        if np.all(expected_cols) and np.all(found_cols):
            print(f"{orig_pathname} has all of the expected columns, not updating this table.")
            continue

        unexpected = list(curr_colnames[~expected_cols])
        missing = list(ptab_cols[~found_cols])
        print(f"Found the following unexpected columns: {unexpected}")
        print(f"Found the following missing columns: {missing}")

        ## Solving the only cases I'm currently aware of
        if 'CAMWORD' in unexpected and 'PROCCAMWORD' in missing:
            print(f"CAMWORD listed instead of PROCCAMWORD. Updating that.")
            origtable.rename_column('CAMWORD', 'PROCCAMWORD')
            unexpected.remove('CAWORD')
            missing.remove('PROCCAMWORD')

        if len(unexpected) > 0:
            print(f"WARNING: Script detected unexpected columns. Only handle "
                  + f"the case where 'CAMWORD' is defined instead of PROCCAMWORD. "
                  + f"The following unexpected columns will be dropped without "
                  + f"using the information they contain: {unexpected}.")
            for colname in unexpected:
                origtable.remove_column(colname)

        ## Add any missing columns
        for colname in missing:
            if colname not in ['BADAMPS', 'LASTSTEP', 'EXPFLAG']:
                print(f"WARNING: Script didn't expect {colname} to be missing. "
                      + f"Replacing with default values, but this may have "
                      + f"downstream consequences.")
            colindex = np.where(ptab_cols==colname)[0][0]
            newdat = [ptab_defs[colindex]] * len(origtable)
            newcol = Table.Column(name=colname, data=newdat, dtype=ptab_dtypes[colindex])
            origtable.add_column(newcol)

        ## Finally, reorder to the current column ordering
        origtable = origtable[list(ptab_cols)]

        ## If just testing, print the table and a cell-by-cell equality test
        ## for the scalar columns
        ## If not testing, move the original table to an archived filename
        ## and save the updated table to the official exptable pathname
        if dry_run:
            print("\n\nOutput file would have been:")
            origtable.pprint_all()
        else:
            ftime = time.strftime("%Y%m%d_%Hh%Mm")
            replaced_pathname = orig_pathname.replace(f".{orig_filetype}",
                                                      f".replaced-{ftime}.{orig_filetype}")
            print(f"Moving original file from {orig_pathname} to {replaced_pathname}")
            os.rename(orig_pathname,replaced_pathname)
            time.sleep(0.1)
            out_pathname = orig_pathname.replace(f".{orig_filetype}", f".{out_filetype}")
            write_table(origtable, out_pathname)
            print(f"Updated file saved to {out_pathname}. Original archived as {replaced_pathname}")

            print("\n\n")

        ## Flush the outputs
        sys.stdout.flush()
        sys.stderr.flush()
    print("Processing table regenerations complete")

1	"""
2	desispec.scripts.reformat_proctables
3	====================================
4
5	"""
6	import argparse	×
7	import os	×
8	import glob	×
9	import sys	×
10	import numpy as np	×
11	import re	×
12	import time	×
13	from astropy.table import Table	×
14
15	from desispec.io.meta import findfile	×
16	from desispec.workflow.proctable import get_processing_table_column_defs	×
17	from desispec.workflow.utils import define_variable_from_environment, listpath, \	×
18	pathjoin
19	from desispec.workflow.tableio import write_table, load_table	×
20	from desispec.scripts.exposuretable import create_exposure_tables	×
21
22
23	def get_parser():	×
24	"""
25	Creates an arguments parser for the desi_reformat_processing_tables script
26	"""
27	parser = argparse.ArgumentParser(usage = "{prog} [options]")	×
28	parser.add_argument("-n", "--nights", type=str, default=None, help="nights as comma separated string")	×
29	parser.add_argument("--night-range", type=str, default=None, help="comma separated pair of nights in form YYYYMMDD,YYYYMMDD"+\	×
30	"for first_night,last_night specifying the beginning"+\
31	"and end of a range of nights to be generated. "+\
32	"last_night should be inclusive.")
33	parser.add_argument("--orig-filetype", type=str, default='csv', help="format type for original exposure tables")	×
34	parser.add_argument("--out-filetype", type=str, default='csv', help="format type for output exposure tables")	×
35	parser.add_argument("--dry-run", action="store_true",	×
36	help="Perform a dry run, printing the changes that would be made and the final output table "+
37	"but not overwriting the actual files on disk.")
38	return parser	×
39
40	def reformat_processing_tables(nights=None, night_range=None, orig_filetype='csv',	×
41	out_filetype='csv', dry_run=False):
42	"""
43	Generates updated processing tables for the nights requested. Requires
44	a current processing table to exist on disk.
45
46	Args:
47	nights: str, int, or comma separated list. The night(s) to generate
48	processing tables for.
49	night_range: str. comma separated pair of nights in form
50	YYYYMMDD,YYYYMMDD for first_night,last_night
51	specifying the beginning and end of a range of
52	nights to be generated. first_night and last_night are
53	inclusive.
54	orig_filetype: str. The file extension (without the '.') of the processing
55	tables.
56	out_filetype: str. The file extension for the outputted processing tables
57	(without the '.').
58
59	Returns:
60	Nothing
61	"""
62	# log = get_logger()
63	## Make sure user specified what nights to run on
64	if nights is None and night_range is None:	×
65	raise ValueError("Must specify either nights or night_range."	×
66	+" To process all nights give nights=all")
67
68	## Get all nights in 2020's with data
69	proctab_template = findfile('proctable', night=99999999)	×
70	proctab_template = proctab_template.replace('99999999', '202[0-9][01][0-9][0-3][0-9]')	×
71	proctab_template = proctab_template.replace('.csv', f'.{orig_filetype}')	×
72	nights_with_proctables = list()	×
73	for ptabfn in glob.glob(proctab_template):	×
74	## nights are 20YYMMDD
NEW 75	matches = re.findall(r'20\d{6}', os.path.basename(ptabfn))	×
76	if len(matches) == 1:	×
77	n = int(matches[0])	×
78	nights_with_proctables.append(n)	×
79	else:
80	print(f"Couldn't parse a night from proctable file: {ptabfn}")	×
81
82	## If unpecified or given "all", set nights to all nights with data
83	check_night = False	×
84	if nights is None or nights == 'all':	×
85	nights = nights_with_proctables	×
86	## No need to check nights since derived from disk
87	else:
88	nights = [int(val.strip()) for val in nights.split(",")]	×
89	## If nights are specified, make sure we check that there is actually data
90	check_night = True	×
91	nights = np.sort(nights)	×
92
93	## If user specified a night range, cut nights to that range of dates
94	if night_range is not None:	×
95	if ',' not in night_range:	×
96	raise ValueError("night_range must be a comma separated pair of "	×
97	+ "nights in form YYYYMMDD,YYYYMMDD")
98	nightpair = night_range.split(',')	×
99	if len(nightpair) != 2 or not nightpair[0].isnumeric() \	×
100	or not nightpair[1].isnumeric():
101	raise ValueError("night_range must be a comma separated pair of "	×
102	+ "nights in form YYYYMMDD,YYYYMMDD")
103	first_night, last_night = nightpair	×
104	nights = nights[np.where(int(first_night) <= nights.astype(int))[0]]	×
105	nights = nights[np.where(int(last_night) >= nights.astype(int))[0]]	×
106
107	## Get current set of expected columns
108	ptab_cols, ptab_dtypes, ptab_defs = get_processing_table_column_defs(return_default_values=True)	×
109	ptab_cols, ptab_dtypes = np.array(ptab_cols), np.array(ptab_dtypes)	×
110
111	## Tell user the final list of nights and starting looping over them
112	print("Nights: ", nights)	×
113	for night in nights:	×
114	if check_night and night not in nights_with_proctables:	×
115	print(f"Night {night} doesn't have a processing table: Skipping.")	×
116	continue	×
117
118	## If the processing table doesn't exist, skip, since we are updating
119	## not generating.
120	orig_pathname = findfile('proctable', night=night).replace('.csv', f'.{orig_filetype}')	×
121	if not os.path.exists(orig_pathname):	×
122	print(f'Could not find processing table for night={night} at:'	×
123	+ f' {orig_pathname}. Skipping this night.')
124	continue	×
125
126	## Load the old and new tables to compare
127	origtable = load_table(orig_pathname, tabletype='proctab')	×
128	curr_colnames = np.array(list(origtable.colnames))	×
129	expected_cols = np.isin(curr_colnames, ptab_cols)	×
130	found_cols = np.isin(ptab_cols, curr_colnames)	×
131
132	## If everything is present, don't try to do anything
133	if np.all(expected_cols) and np.all(found_cols):	×
134	print(f"{orig_pathname} has all of the expected columns, not updating this table.")	×
135	continue	×
136
137	unexpected = list(curr_colnames[~expected_cols])	×
138	missing = list(ptab_cols[~found_cols])	×
139	print(f"Found the following unexpected columns: {unexpected}")	×
140	print(f"Found the following missing columns: {missing}")	×
141
142	## Solving the only cases I'm currently aware of
143	if 'CAMWORD' in unexpected and 'PROCCAMWORD' in missing:	×
144	print(f"CAMWORD listed instead of PROCCAMWORD. Updating that.")	×
145	origtable.rename_column('CAMWORD', 'PROCCAMWORD')	×
146	unexpected.remove('CAWORD')	×
147	missing.remove('PROCCAMWORD')	×
148
149	if len(unexpected) > 0:	×
150	print(f"WARNING: Script detected unexpected columns. Only handle "	×
151	+ f"the case where 'CAMWORD' is defined instead of PROCCAMWORD. "
152	+ f"The following unexpected columns will be dropped without "
153	+ f"using the information they contain: {unexpected}.")
154	for colname in unexpected:	×
155	origtable.remove_column(colname)	×
156
157	## Add any missing columns
158	for colname in missing:	×
159	if colname not in ['BADAMPS', 'LASTSTEP', 'EXPFLAG']:	×
160	print(f"WARNING: Script didn't expect {colname} to be missing. "	×
161	+ f"Replacing with default values, but this may have "
162	+ f"downstream consequences.")
163	colindex = np.where(ptab_cols==colname)[0][0]	×
164	newdat = [ptab_defs[colindex]] * len(origtable)	×
165	newcol = Table.Column(name=colname, data=newdat, dtype=ptab_dtypes[colindex])	×
166	origtable.add_column(newcol)	×
167
168	## Finally, reorder to the current column ordering
169	origtable = origtable[list(ptab_cols)]	×
170
171	## If just testing, print the table and a cell-by-cell equality test
172	## for the scalar columns
173	## If not testing, move the original table to an archived filename
174	## and save the updated table to the official exptable pathname
175	if dry_run:	×
176	print("\n\nOutput file would have been:")	×
177	origtable.pprint_all()	×
178	else:
179	ftime = time.strftime("%Y%m%d_%Hh%Mm")	×
180	replaced_pathname = orig_pathname.replace(f".{orig_filetype}",	×
181	f".replaced-{ftime}.{orig_filetype}")
182	print(f"Moving original file from {orig_pathname} to {replaced_pathname}")	×
183	os.rename(orig_pathname,replaced_pathname)	×
184	time.sleep(0.1)	×
185	out_pathname = orig_pathname.replace(f".{orig_filetype}", f".{out_filetype}")	×
186	write_table(origtable, out_pathname)	×
187	print(f"Updated file saved to {out_pathname}. Original archived as {replaced_pathname}")	×
188
189	print("\n\n")	×
190
191	## Flush the outputs
192	sys.stdout.flush()	×
193	sys.stderr.flush()	×
194	print("Processing table regenerations complete")	×

desihub / desispec / 15789610751

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous