19150063725

Committed 06 Nov 2025 09:14PM UTC coverage: 37.719% (+0.7%) from 37.002%

Build # 19150063725

Build Type

Pull #2521

github

Committed by

web-flow

Commit Message

Merge b3bf9e090 into 6a90a0547

Pull Request Pull Request #2521: Add redshift QA scripts

Run Details

12990 of 34439 relevant lines covered (37.72%)

0.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.79

/py/desispec/scripts/compute_dark.py

#!/usr/bin/env python

import os
import argparse
import numpy as np

import fitsio
import astropy.io.fits as pyfits
from astropy.table import Table,vstack

from desiutil.log import get_logger

from desispec.ccdcalib import compute_dark_file
from desispec.util import parse_nights, get_night_range
from desispec.io.util import get_speclog,erow_to_goodcamword,decode_camword
from desispec.io import findfile
from desispec.workflow.tableio import load_table


def compute_dark_baseparser():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                                     description="Compute a master dark",
                                     epilog='''
                                     Input is a list of raw dark images, possibly with various exposure times.
                                     Raw images are preprocessed without dark,mask correction.
                                     However gains are applied so the output is in electrons/sec.
                                     We first compute a masked median of the preprocessed images divided by their exposure time.
                                     Then mask outlier pixels, and then compute the dark with optimal weights (propto. exptime).
                                     We use for this the keyword EXPREQ in the raw image primary header, or EXPTIME if the former is absent.''')

    parser.add_argument('-n','--nights', type=str, default = None, required=False,
                        help='YEARMMDD nights to use (comma separated or with : to define a range. integers that do not correspond to valid dates are ignored)')
    parser.add_argument('-r', '--reference-night', type=int, default = None, required=False,
                        help='YEARMMDD reference night defining the hardware state for this dark frame (default is most recent, option cannot be set at the same time as reference-expid)')
    parser.add_argument('-b', '--before', type=int, default=30, required=False,
                        help = 'Number of nights before reference-night to include')
    parser.add_argument('-a', '--after', type=int, default=15, required=False,
                        help = 'Number of nights after reference-night to include')
    parser.add_argument('--reference-expid', type=int, default = None, required=False,
                        help='reference expid defining the hardware state for this dark frame (default is most recent, option cannot be set at the same time as reference-night)')
    parser.add_argument('--first-expid', type=int, default = None, required=False,
                        help='First EXPID to include (to use in combination with --nights option)')
    parser.add_argument('--last-expid', type=int, default = None, required=False,
                        help='Last EXPID to include (to use in combination with --nights option)')
    parser.add_argument('--min-exptime', type=float, default = 500,
                        help='minimal exposure time to consider')
    parser.add_argument('--max-exptime', type=float, default = None,
                        help='maximal exposure time to consider')
    parser.add_argument('--max-temperature-diff', type=float, default = 4. ,
                        help='maximal difference of CCD temperature to consider')    
    parser.add_argument('--bias', type = str, default = None, required=False,
                         help = 'specify a bias image calibration file (standard preprocessing calibration is turned off)')
    parser.add_argument('--nocosmic', action = 'store_true',
                        help = 'do not perform comic ray subtraction (much slower, but more accurate because median can leave traces)')
    parser.add_argument('--min-hours-since-vccd-on', type=float, default=4., required=False,
                        help='Minimum time (in hours) since voltages were turned on')
    parser.add_argument('--specprod', type=str, default=None, required=False,
                        help='Specify specprod to use nightly bias files and the exposure tables. Default is $SPECPROD if it is defined, otherwise will use the bias in DESI_SPECTRO_CALIB.')
    parser.add_argument('--save-preproc', action='store_true', help='save intermediate preproc files')
    parser.add_argument('--preproc-dark-dir', type=str, default=None, required=False,
                        help='Specify alternate specprod directory where preprocessed dark frame images are saved. Default is same input specprod')
    parser.add_argument('--dry-run', action='store_true', help="If dry_run, print which images would be used, but don't compute dark.")
    parser.add_argument('--min-dark-exposures', type=int, default=4, required=False,
                        help='Minimum number of dark exposures to use. Default is 4. If less than this number of exposures are valid, ' \
                        'the script will raise an error and exit.')
    parser.add_argument('--max-dark-exposures', type=int, default=50, required=False,
                        help='Maximum number of dark exposures to use. Default is 50. If more than this number of exposures are found, ' \
                        'the script will downselect to the closest exposures in time up to this limit.')
    parser.add_argument('--skip-camera-check', action='store_true', help="If True, doesn't check if camera exists for an exposure ahead of time.")
    parser.add_argument('--dont-search-filesystem', action='store_true', help="If True, doesn't search filesystem for exposures.")
    return parser


def compute_dark_parser():
    """
    Takes the "base" argparser that is also used by desi_compute_dark_night and add specific args
    """
    parser = compute_dark_baseparser()
    parser.add_argument('-i','--images', type = str, default = None, required = False, nargs="*",
                        help = 'path of raws image fits files (or use --nights or --reference-night)')
    parser.add_argument('-c','--camera',type = str, required = True,
                        help = 'header HDU (int or string)')
    parser.add_argument('-o','--outfile', type = str, default = None, required = True,
                        help = 'output median image filename')
    return parser


def parse(options=None):
    # parse the command line arguments
    parser = compute_dark_parser()
    
    #- uses sys.argv if options=None
    args = parser.parse_args(options)

    return args

def get_stacked_dark_exposure_table(args):
    """
    Get the exposure table for the dark exposures to be used.
    If --nights is specified, it will return the exposures for those nights.
    If --reference-night is specified, it will return the exposures around that night.
    If --images is specified, it will return the exposures corresponding to those images.
    """
    log  = get_logger()
    # check all required environment variables, then return error if any are missing
    envok = True
    for k in ["DESI_SPECTRO_DATA","DESI_SPECTRO_REDUX","SPECPROD"] :
        if k not in os.environ :
            envok = False
            log.error(f"args.nights/reference_night specified but variable {k} is not set so we cannot find the exposures.")
            if k=="SPECPROD" :
                log.error("consider using argument --specprod.")

    if not envok:
        return None

    if args.nights is None:
        nights = get_night_range(args.reference_night, args.before, args.after)
    else:
        nights = parse_nights(args.nights)

    log.info(f"Will look for dark exposures in nights: {nights}")
    tables = []
    missing_nights = []
    for night in nights :
        filename = findfile("exposure_table",night=night, readonly=True)
        if os.path.isfile(filename) :
            tmp_table=load_table(filename, suppress_logging=True)
            if len(tmp_table)==0 : continue

            # keep only valid exposures
            keep = (tmp_table['LASTSTEP'] != 'ignore')
            tmp_table = tmp_table[keep]
            if len(tmp_table)==0 : continue

            # only keep useful rows to avoid issues with columns
            table = tmp_table['NIGHT', 'EXPID', 'MJD-OBS', 'OBSTYPE', 'EXPTIME', 'CAMWORD', 'BADCAMWORD', 'BADAMPS']
            tables.append(table)
        else :
            log.warning(f"No exposure table for {night}")
            nightdir=os.path.join(os.environ["DESI_SPECTRO_DATA"],str(night))
            if not os.path.isdir(nightdir) :
                log.warning(f"No data directory {nightdir}")
                continue
            missing_nights.append(night)

    if len(missing_nights)>0 and args.dont_search_filesystem:
        log.info(f"Found nights without exposure tables ({missing_nights}) "
                 + "but told not to search for missing nights, so continuing without them.")
    elif len(missing_nights)>0:
        log.info(f"Computing speclog for nights without exposure tables ({missing_nights})")
        table = get_speclog(missing_nights)
        table.rename_column('MJD', 'MJD-OBS')
        if len(table)>0 :
            table = table[["NIGHT","EXPID","MJD-OBS","OBSTYPE",
                           "EXPTIME", "CAMWORD", "BADCAMWORD", "BADAMPS"]]
            for i in range(len(table)) :
                table["OBSTYPE"][i]=table["OBSTYPE"][i].lower()
            tables.append(table)
    if len(tables)>0 :
        exptable=vstack(tables)
    else :
        log.error(f"empty list of dark exposures")
        return None

    valid=(exptable["OBSTYPE"]=="dark")
    log.info(f"{np.sum(valid)} dark exposures")
    valid &= (exptable["EXPTIME"]>=args.min_exptime)
    log.info(f"{np.sum(valid)} dark exposures with EXPTIME>={args.min_exptime}")
    if args.first_expid is not None :
        valid &= (exptable["EXPID"]>=args.first_expid)
        log.info(f"{np.sum(valid)} dark exposures with EXPID>={args.first_expid}")
    if args.last_expid is not None :
        valid &= (exptable["EXPID"]<=args.last_expid)
        log.info(f"{np.sum(valid)} dark exposures with EXPID<={args.last_expid}")

    # trim to valid exposures
    exptable = exptable[valid]
    exptable.sort('EXPID')
    log.info(f"{len(exptable)} valid dark exposures found.")

    # assemble corresponding images
    args.images = []
    file_exists = np.ones(len(exptable), dtype=bool)
    if not args.dont_search_filesystem:
        for e in range(len(exptable)):
            filename = findfile("raw",night=exptable["NIGHT"][e],expid=exptable["EXPID"][e])
            if not os.path.exists(filename):
                # "Missing" files can occur due to a mismatch between the NIGHT header keyword
                # and the directory in which the file is found, e.g. 20250620/00298589/desi-00298589.fits.fz
                # has header NIGHT=20250619, but also FLAVOR=science instead of FLAVOR=dark
                file_exists[e] = False
                log.error(f'Skipping missing file {filename}')

    if not np.all(file_exists):
        exptable = exptable[file_exists]
        log.info(f"{len(exptable)} exposures will be used to build the {args.camera} dark")
        #log.info(exptable)

    return exptable


def main(args=None, exptable=None):

    if not isinstance(args, argparse.Namespace):
        args = parse(args)

    log  = get_logger()

    # Methods to specify what images to use
    #   1. --images
    #   2. --nights [--first-expid, --last-expid, [--reference-night | --reference-expid]]
    #   3. --reference-night [--before, --after, [--first-expid, --last-expid]]

    if args.specprod is not None :
        os.environ["SPECPROD"] = args.specprod

    # check consistency of input options
    if args.nights is not None and args.images is not None :
        log.critical("Cannot specify both --nights and --image arguments")
        return 1

    if args.nights is None and args.images is None and args.reference_night is None:
        log.critical("Need to specify input using --images, --nights, or --reference-night")
        return 1

    if args.reference_night is not None and args.reference_expid is not None :
        log.error("Cannot use --reference-night and --reference-expid at the same time.")
        return 1

    ## If we don't have the minimum number, exit now
    if args.images is not None and len(arg,images) < args.min_dark_exposures:
        log.critical(f"Number of possible dark exposures {len(args.images)} "
                     + f"< required dark exposures ({args.min_dark_exposures}).")
        return 1
   
    # first find the exposures if they are not given in input
    if args.images is None:
        if exptable is None:
            # if no images are given, we need to find the exposures
            exptable = get_stacked_dark_exposure_table(args)
        
        if not args.skip_camera_check:
            log.info(f"Subselecting exposures in exposure table in which {args.camera}.")
            keep = np.repeat(True,len(exptable))
            for i,entry in enumerate(exptable) :
                keep[i] &= ( args.camera in decode_camword(erow_to_goodcamword(entry, suppress_logging=True, exclude_badamps=True)) )
            exptable = exptable[keep]

        if exptable is None:
            log.critical(f"No dark exposures found for {args.camera}")
            return 1
        elif len(exptable) < args.min_dark_exposures:
            log.critical(f"Number of possible dark exposures {len(exptable)} "
                         + f"< required dark exposures ({args.min_dark_exposures}).")
            return 1

        # assemble corresponding images
        args.images, mjds = [], []
        for row in exptable:
            filename = findfile("raw",night=row["NIGHT"],expid=row["EXPID"], readonly=True)
            if os.path.exists(filename):
                args.images.append(filename)
                mjds.append(row["MJD-OBS"])
            else:
                log.critical(f'Raw data {filename} is missing. This should not happen. Exiting.')
                return 1
        args.images = np.array(args.images)
        mjds = np.array(mjds)

    # find the most recent exposure with the camera and read its header
    # unless reference_expid or reference_night is set
    reference_header = None
    reference_header_possible_filenames = []
    if args.reference_expid is not None :
        selection=np.where(exptable["EXPID"]==args.reference_expid)[0]
        if selection.size == 0 :
            log.critical(f"Reference expid {args.reference_expid} is not in the list of input darks")
            return 1
        reference_header_possible_filenames.append(args.images[selection[0]])
    elif args.reference_night is not None :
        selection=np.where(exptable["NIGHT"]==args.reference_night)[0]
        if selection.size == 0 :
            log.warning(f"No dark during reference night {args.reference_night} in input list. "
                      + "Looking for a science exposure instead.")
            etab = load_table(tabletype='exposure_table',
                              tablename=findfile('exposure_table', night=args.reference_night))
            boolsel = (etab['OBSTYPE'] == 'science') & (etab['LASTSTEP'] != 'ignore')
            if boolsel.sum() > 0:
                for scierow in etab[boolsel][::-1]:
                    if args.camera in decode_camword(erow_to_goodcamword(scierow, suppress_logging=True, exclude_badamps=False)):
                        reference_header_possible_filenames.append(findfile('raw', night=args.reference_night, expid=scierow['EXPID']))
            if len(reference_header_possible_filenames) == 0:
                log.critical(f"No dark during reference night {args.reference_night} in input list "
                            + "AND no valid science exposures. Cannot proceed. Exiting.")
                return 1
        else:
            indices=np.argsort(exptable["EXPID"][selection])[::-1]
            for index in indices :
                reference_header_possible_filenames.append(args.images[selection[index]])
    else :
        if exptable is not None:
            indices=np.argsort(exptable["EXPID"])[::-1]
            for index in indices :
                reference_header_possible_filenames.append(args.images[index])
        else:
            reference_header_possible_filenames.extend(args.images)

    reference_header = None
    for filename in reference_header_possible_filenames :
        try:
            reference_header = fitsio.read_header(filename, ext=args.camera)
        except OSError:
            log.warning(f'No camera {args.camera} in {filename}')
            continue
        if reference_header is not None:
            break
    if reference_header is None :
        log.critical(f"No exposure has the camera {args.camera}.")
        return 1

    assert args.camera.upper() == reference_header['CAMERA'].upper()

    if exptable is not None and len(args.images) > args.max_dark_exposures:
        log.warning(f"More than {args.max_dark_exposures} dark exposures found, sorting by time " 
                    + f"difference in MJD-OBS to reference header "
                    + f"MJD-OBS {reference_header['MJD-OBS']}")
        sel = np.argsort(np.abs(mjds-float(reference_header['MJD-OBS'])))
        args.images = args.images[sel]

    if args.dry_run:
        image_str = ' '.join(args.images)
        log.info(f'Input images: {image_str}')
        log.info('--dry-run mode, exiting before generating dark')
        return 0

    compute_dark_file(args.images, args.outfile, camera=args.camera, bias=args.bias,
                      nocosmic=args.nocosmic,
                      min_vccdsec=(args.min_hours_since_vccd_on * 3600.),
                      max_temperature_diff=args.max_temperature_diff,
                      reference_header=reference_header,
                      save_preproc=args.save_preproc,
                      preproc_dark_dir=args.preproc_dark_dir,
                      min_dark_exposures=args.min_dark_exposures,
                      max_dark_exposures=args.max_dark_exposures)
    return 0

1	#!/usr/bin/env python
2
3	import os	1✔
4	import argparse	1✔
5	import numpy as np	1✔
6
7	import fitsio	1✔
8	import astropy.io.fits as pyfits	1✔
9	from astropy.table import Table,vstack	1✔
10
11	from desiutil.log import get_logger	1✔
12
13	from desispec.ccdcalib import compute_dark_file	1✔
14	from desispec.util import parse_nights, get_night_range	1✔
15	from desispec.io.util import get_speclog,erow_to_goodcamword,decode_camword	1✔
16	from desispec.io import findfile	1✔
17	from desispec.workflow.tableio import load_table	1✔
18
19
20	def compute_dark_baseparser():	1✔
21	parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,	1✔
22	description="Compute a master dark",
23	epilog='''
24	Input is a list of raw dark images, possibly with various exposure times.
25	Raw images are preprocessed without dark,mask correction.
26	However gains are applied so the output is in electrons/sec.
27	We first compute a masked median of the preprocessed images divided by their exposure time.
28	Then mask outlier pixels, and then compute the dark with optimal weights (propto. exptime).
29	We use for this the keyword EXPREQ in the raw image primary header, or EXPTIME if the former is absent.''')
30
31	parser.add_argument('-n','--nights', type=str, default = None, required=False,	1✔
32	help='YEARMMDD nights to use (comma separated or with : to define a range. integers that do not correspond to valid dates are ignored)')
33	parser.add_argument('-r', '--reference-night', type=int, default = None, required=False,	1✔
34	help='YEARMMDD reference night defining the hardware state for this dark frame (default is most recent, option cannot be set at the same time as reference-expid)')
35	parser.add_argument('-b', '--before', type=int, default=30, required=False,	1✔
36	help = 'Number of nights before reference-night to include')
37	parser.add_argument('-a', '--after', type=int, default=15, required=False,	1✔
38	help = 'Number of nights after reference-night to include')
39	parser.add_argument('--reference-expid', type=int, default = None, required=False,	1✔
40	help='reference expid defining the hardware state for this dark frame (default is most recent, option cannot be set at the same time as reference-night)')
41	parser.add_argument('--first-expid', type=int, default = None, required=False,	1✔
42	help='First EXPID to include (to use in combination with --nights option)')
43	parser.add_argument('--last-expid', type=int, default = None, required=False,	1✔
44	help='Last EXPID to include (to use in combination with --nights option)')
45	parser.add_argument('--min-exptime', type=float, default = 500,	1✔
46	help='minimal exposure time to consider')
47	parser.add_argument('--max-exptime', type=float, default = None,	1✔
48	help='maximal exposure time to consider')
49	parser.add_argument('--max-temperature-diff', type=float, default = 4. ,	1✔
50	help='maximal difference of CCD temperature to consider')
51	parser.add_argument('--bias', type = str, default = None, required=False,	1✔
52	help = 'specify a bias image calibration file (standard preprocessing calibration is turned off)')
53	parser.add_argument('--nocosmic', action = 'store_true',	1✔
54	help = 'do not perform comic ray subtraction (much slower, but more accurate because median can leave traces)')
55	parser.add_argument('--min-hours-since-vccd-on', type=float, default=4., required=False,	1✔
56	help='Minimum time (in hours) since voltages were turned on')
57	parser.add_argument('--specprod', type=str, default=None, required=False,	1✔
58	help='Specify specprod to use nightly bias files and the exposure tables. Default is $SPECPROD if it is defined, otherwise will use the bias in DESI_SPECTRO_CALIB.')
59	parser.add_argument('--save-preproc', action='store_true', help='save intermediate preproc files')	1✔
60	parser.add_argument('--preproc-dark-dir', type=str, default=None, required=False,	1✔
61	help='Specify alternate specprod directory where preprocessed dark frame images are saved. Default is same input specprod')
62	parser.add_argument('--dry-run', action='store_true', help="If dry_run, print which images would be used, but don't compute dark.")	1✔
63	parser.add_argument('--min-dark-exposures', type=int, default=4, required=False,	1✔
64	help='Minimum number of dark exposures to use. Default is 4. If less than this number of exposures are valid, ' \
65	'the script will raise an error and exit.')
66	parser.add_argument('--max-dark-exposures', type=int, default=50, required=False,	1✔
67	help='Maximum number of dark exposures to use. Default is 50. If more than this number of exposures are found, ' \
68	'the script will downselect to the closest exposures in time up to this limit.')
69	parser.add_argument('--skip-camera-check', action='store_true', help="If True, doesn't check if camera exists for an exposure ahead of time.")	1✔
70	parser.add_argument('--dont-search-filesystem', action='store_true', help="If True, doesn't search filesystem for exposures.")	1✔
71	return parser	1✔
72
73
74	def compute_dark_parser():	1✔
75	"""
76	Takes the "base" argparser that is also used by desi_compute_dark_night and add specific args
77	"""
78	parser = compute_dark_baseparser()	1✔
79	parser.add_argument('-i','--images', type = str, default = None, required = False, nargs="*",	1✔
80	help = 'path of raws image fits files (or use --nights or --reference-night)')
81	parser.add_argument('-c','--camera',type = str, required = True,	1✔
82	help = 'header HDU (int or string)')
83	parser.add_argument('-o','--outfile', type = str, default = None, required = True,	1✔
84	help = 'output median image filename')
85	return parser	1✔
86
87
88	def parse(options=None):	1✔
89	# parse the command line arguments
90	parser = compute_dark_parser()	×
91
92	#- uses sys.argv if options=None
93	args = parser.parse_args(options)	×
94
95	return args	×
96
97	def get_stacked_dark_exposure_table(args):	1✔
98	"""
99	Get the exposure table for the dark exposures to be used.
100	If --nights is specified, it will return the exposures for those nights.
101	If --reference-night is specified, it will return the exposures around that night.
102	If --images is specified, it will return the exposures corresponding to those images.
103	"""
104	log = get_logger()	1✔
105	# check all required environment variables, then return error if any are missing
106	envok = True	1✔
107	for k in ["DESI_SPECTRO_DATA","DESI_SPECTRO_REDUX","SPECPROD"] :	1✔
108	if k not in os.environ :	1✔
109	envok = False	×
110	log.error(f"args.nights/reference_night specified but variable {k} is not set so we cannot find the exposures.")	×
111	if k=="SPECPROD" :	×
112	log.error("consider using argument --specprod.")	×
113
114	if not envok:	1✔
115	return None	×
116
117	if args.nights is None:	1✔
118	nights = get_night_range(args.reference_night, args.before, args.after)	1✔
119	else:
120	nights = parse_nights(args.nights)	×
121
122	log.info(f"Will look for dark exposures in nights: {nights}")	1✔
123	tables = []	1✔
124	missing_nights = []	1✔
125	for night in nights :	1✔
126	filename = findfile("exposure_table",night=night, readonly=True)	1✔
127	if os.path.isfile(filename) :	1✔
128	tmp_table=load_table(filename, suppress_logging=True)	1✔
129	if len(tmp_table)==0 : continue	1✔
130
131	# keep only valid exposures
132	keep = (tmp_table['LASTSTEP'] != 'ignore')	1✔
133	tmp_table = tmp_table[keep]	1✔
134	if len(tmp_table)==0 : continue	1✔
135
136	# only keep useful rows to avoid issues with columns
137	table = tmp_table['NIGHT', 'EXPID', 'MJD-OBS', 'OBSTYPE', 'EXPTIME', 'CAMWORD', 'BADCAMWORD', 'BADAMPS']	1✔
138	tables.append(table)	1✔
139	else :
140	log.warning(f"No exposure table for {night}")	1✔
141	nightdir=os.path.join(os.environ["DESI_SPECTRO_DATA"],str(night))	1✔
142	if not os.path.isdir(nightdir) :	1✔
143	log.warning(f"No data directory {nightdir}")	1✔
144	continue	1✔
145	missing_nights.append(night)	1✔
146
147	if len(missing_nights)>0 and args.dont_search_filesystem:	1✔
148	log.info(f"Found nights without exposure tables ({missing_nights}) "	1✔
149	+ "but told not to search for missing nights, so continuing without them.")
150	elif len(missing_nights)>0:	1✔
151	log.info(f"Computing speclog for nights without exposure tables ({missing_nights})")	×
152	table = get_speclog(missing_nights)	×
153	table.rename_column('MJD', 'MJD-OBS')	×
154	if len(table)>0 :	×
155	table = table[["NIGHT","EXPID","MJD-OBS","OBSTYPE",	×
156	"EXPTIME", "CAMWORD", "BADCAMWORD", "BADAMPS"]]
157	for i in range(len(table)) :	×
158	table["OBSTYPE"][i]=table["OBSTYPE"][i].lower()	×
159	tables.append(table)	×
160	if len(tables)>0 :	1✔
161	exptable=vstack(tables)	1✔
162	else :
163	log.error(f"empty list of dark exposures")	×
164	return None	×
165
166	valid=(exptable["OBSTYPE"]=="dark")	1✔
167	log.info(f"{np.sum(valid)} dark exposures")	1✔
168	valid &= (exptable["EXPTIME"]>=args.min_exptime)	1✔
169	log.info(f"{np.sum(valid)} dark exposures with EXPTIME>={args.min_exptime}")	1✔
170	if args.first_expid is not None :	1✔
171	valid &= (exptable["EXPID"]>=args.first_expid)	×
172	log.info(f"{np.sum(valid)} dark exposures with EXPID>={args.first_expid}")	×
173	if args.last_expid is not None :	1✔
174	valid &= (exptable["EXPID"]<=args.last_expid)	×
175	log.info(f"{np.sum(valid)} dark exposures with EXPID<={args.last_expid}")	×
176
177	# trim to valid exposures
178	exptable = exptable[valid]	1✔
179	exptable.sort('EXPID')	1✔
180	log.info(f"{len(exptable)} valid dark exposures found.")	1✔
181
182	# assemble corresponding images
183	args.images = []	1✔
184	file_exists = np.ones(len(exptable), dtype=bool)	1✔
185	if not args.dont_search_filesystem:	1✔
186	for e in range(len(exptable)):	×
187	filename = findfile("raw",night=exptable["NIGHT"][e],expid=exptable["EXPID"][e])	×
188	if not os.path.exists(filename):	×
189	# "Missing" files can occur due to a mismatch between the NIGHT header keyword
190	# and the directory in which the file is found, e.g. 20250620/00298589/desi-00298589.fits.fz
191	# has header NIGHT=20250619, but also FLAVOR=science instead of FLAVOR=dark
192	file_exists[e] = False	×
193	log.error(f'Skipping missing file {filename}')	×
194
195	if not np.all(file_exists):	1✔
196	exptable = exptable[file_exists]	×
197	log.info(f"{len(exptable)} exposures will be used to build the {args.camera} dark")	×
198	#log.info(exptable)
199
200	return exptable	1✔
201
202
203	def main(args=None, exptable=None):
204
205	if not isinstance(args, argparse.Namespace):
206	args = parse(args)
207
208	log = get_logger()
209
210	# Methods to specify what images to use
211	# 1. --images
212	# 2. --nights [--first-expid, --last-expid, [--reference-night \| --reference-expid]]
213	# 3. --reference-night [--before, --after, [--first-expid, --last-expid]]
214
215	if args.specprod is not None :
216	os.environ["SPECPROD"] = args.specprod
217
218	# check consistency of input options
219	if args.nights is not None and args.images is not None :
220	log.critical("Cannot specify both --nights and --image arguments")
221	return 1
222
223	if args.nights is None and args.images is None and args.reference_night is None:
224	log.critical("Need to specify input using --images, --nights, or --reference-night")
225	return 1
226
227	if args.reference_night is not None and args.reference_expid is not None :
228	log.error("Cannot use --reference-night and --reference-expid at the same time.")
229	return 1
230
231	## If we don't have the minimum number, exit now
232	if args.images is not None and len(arg,images) < args.min_dark_exposures:
233	log.critical(f"Number of possible dark exposures {len(args.images)} "
234	+ f"< required dark exposures ({args.min_dark_exposures}).")
235	return 1
236
237	# first find the exposures if they are not given in input
238	if args.images is None:
239	if exptable is None:
240	# if no images are given, we need to find the exposures
241	exptable = get_stacked_dark_exposure_table(args)
242
243	if not args.skip_camera_check:
244	log.info(f"Subselecting exposures in exposure table in which {args.camera}.")
245	keep = np.repeat(True,len(exptable))
246	for i,entry in enumerate(exptable) :
247	keep[i] &= ( args.camera in decode_camword(erow_to_goodcamword(entry, suppress_logging=True, exclude_badamps=True)) )
248	exptable = exptable[keep]
249
250	if exptable is None:
251	log.critical(f"No dark exposures found for {args.camera}")
252	return 1
253	elif len(exptable) < args.min_dark_exposures:
254	log.critical(f"Number of possible dark exposures {len(exptable)} "
255	+ f"< required dark exposures ({args.min_dark_exposures}).")
256	return 1
257
258	# assemble corresponding images
259	args.images, mjds = [], []
260	for row in exptable:
261	filename = findfile("raw",night=row["NIGHT"],expid=row["EXPID"], readonly=True)
262	if os.path.exists(filename):
263	args.images.append(filename)
264	mjds.append(row["MJD-OBS"])
265	else:
266	log.critical(f'Raw data {filename} is missing. This should not happen. Exiting.')
267	return 1
268	args.images = np.array(args.images)
269	mjds = np.array(mjds)
270
271	# find the most recent exposure with the camera and read its header
272	# unless reference_expid or reference_night is set
273	reference_header = None
274	reference_header_possible_filenames = []
275	if args.reference_expid is not None :
276	selection=np.where(exptable["EXPID"]==args.reference_expid)[0]
277	if selection.size == 0 :
278	log.critical(f"Reference expid {args.reference_expid} is not in the list of input darks")
279	return 1
280	reference_header_possible_filenames.append(args.images[selection[0]])
281	elif args.reference_night is not None :
282	selection=np.where(exptable["NIGHT"]==args.reference_night)[0]
283	if selection.size == 0 :
284	log.warning(f"No dark during reference night {args.reference_night} in input list. "
285	+ "Looking for a science exposure instead.")
286	etab = load_table(tabletype='exposure_table',
287	tablename=findfile('exposure_table', night=args.reference_night))
288	boolsel = (etab['OBSTYPE'] == 'science') & (etab['LASTSTEP'] != 'ignore')
289	if boolsel.sum() > 0:
290	for scierow in etab[boolsel][::-1]:
291	if args.camera in decode_camword(erow_to_goodcamword(scierow, suppress_logging=True, exclude_badamps=False)):
292	reference_header_possible_filenames.append(findfile('raw', night=args.reference_night, expid=scierow['EXPID']))
293	if len(reference_header_possible_filenames) == 0:
294	log.critical(f"No dark during reference night {args.reference_night} in input list "
295	+ "AND no valid science exposures. Cannot proceed. Exiting.")
296	return 1
297	else:
298	indices=np.argsort(exptable["EXPID"][selection])[::-1]
299	for index in indices :
300	reference_header_possible_filenames.append(args.images[selection[index]])
301	else :
302	if exptable is not None:
303	indices=np.argsort(exptable["EXPID"])[::-1]
304	for index in indices :
305	reference_header_possible_filenames.append(args.images[index])
306	else:
307	reference_header_possible_filenames.extend(args.images)
308
309	reference_header = None
310	for filename in reference_header_possible_filenames :
311	try:
312	reference_header = fitsio.read_header(filename, ext=args.camera)
313	except OSError:
314	log.warning(f'No camera {args.camera} in {filename}')
315	continue
316	if reference_header is not None:
317	break
318	if reference_header is None :
319	log.critical(f"No exposure has the camera {args.camera}.")
320	return 1
321
322	assert args.camera.upper() == reference_header['CAMERA'].upper()
323
324	if exptable is not None and len(args.images) > args.max_dark_exposures:
325	log.warning(f"More than {args.max_dark_exposures} dark exposures found, sorting by time "
326	+ f"difference in MJD-OBS to reference header "
327	+ f"MJD-OBS {reference_header['MJD-OBS']}")
328	sel = np.argsort(np.abs(mjds-float(reference_header['MJD-OBS'])))
329	args.images = args.images[sel]
330
331	if args.dry_run:
332	image_str = ' '.join(args.images)
333	log.info(f'Input images: {image_str}')
334	log.info('--dry-run mode, exiting before generating dark')
335	return 0
336
337	compute_dark_file(args.images, args.outfile, camera=args.camera, bias=args.bias,
338	nocosmic=args.nocosmic,
339	min_vccdsec=(args.min_hours_since_vccd_on * 3600.),
340	max_temperature_diff=args.max_temperature_diff,
341	reference_header=reference_header,
342	save_preproc=args.save_preproc,
343	preproc_dark_dir=args.preproc_dark_dir,
344	min_dark_exposures=args.min_dark_exposures,
345	max_dark_exposures=args.max_dark_exposures)
346	return 0

desihub / desispec / 19150063725

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous