• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

desihub / desispec / 6268877977

22 Sep 2023 01:13AM UTC coverage: 24.319% (-0.004%) from 24.323%
6268877977

Pull #2122

github-actions

moustakas
Merge branch 'main' into validate-sv
Pull Request #2122: update validredshifts.validate() to work on SV observations

12 of 12 new or added lines in 1 file covered. (100.0%)

10789 of 44364 relevant lines covered (24.32%)

0.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/py/desispec/validredshifts.py
1
"""
2
desispec.validredshifts
3
=======================
4

5
"""
6
# Example usage:
7
# redrock_path = '/global/cfs/cdirs/desi/spectro/redux/guadalupe/tiles/cumulative/1392/20210517/redrock-5-1392-thru20210517.fits'
8
# cat = validate(redrock_path, return_target_columns=True)
9

10
import os, warnings
×
11
import numpy as np
×
12
from astropy.table import Table, hstack, join
×
13
import fitsio
×
14

15

16
def get_good_fiberstatus(cat, isqso=False):
×
17
    '''
18
    Validate the fiber status for a redrock catalog.
19

20
    Args:
21
        cat: redrock catalog (e.g., in astropy table format)
22

23
    Options:
24
        isqso: bool (default False), if True, use the specific QSO criteria
25

26
    Returns:
27
        good_fiberstatus: boolean array
28
    '''
29

30
    if not isqso:
×
31
        good_fiberstatus = cat['COADD_FIBERSTATUS']==0
×
32
    else:
33
        # good_fiberstatus = (cat['COADD_FIBERSTATUS']==0) | (cat['COADD_FIBERSTATUS']==8388608) | (cat['COADD_FIBERSTATUS']==16777216)
34
        from desispec.maskbits import fibermask
×
35
        good_fiberstatus = (cat['COADD_FIBERSTATUS']==0) | (cat['COADD_FIBERSTATUS']==fibermask['BADAMPR']) | (cat['COADD_FIBERSTATUS']==fibermask['BADAMPZ'])
×
36
    good_fiberstatus &= cat['ZWARN'] & 2**9 == 0  # NO DATA flag
×
37
    return good_fiberstatus
×
38

39

40
def validate(redrock_path, fiberstatus_cut=True, return_target_columns=False, extra_columns=None, emline_path=None, ignore_emline=False):
×
41
    '''
42
    Validate the redshift quality with tracer-dependent criteria for redrock+afterburner results.
43

44
    Args:
45
        redrock_path: str, path of redrock FITS file
46

47
    Options:
48
        fiberstatus_cut: bool (default True), if True, impose requirements on COADD_FIBERSTATUS and ZWARN
49
        return_target_columns: bool (default False), if True, include columns that indicate if the object belongs to each class of DESI targets
50
        extra_columns: list of str (default None), additional columns to include in the output
51
        emline_path: str (default None), specify the location of the emline file; by default the emline file is in the same directory as the redrock file
52
        ignore_emline: bool (default False), if True, ignore the emline file and do not validate the ELG redshift
53

54
    Returns:
55
        cat: astropy table with basic columns such as TARGETID and boolean columns (e.g., GOOD_BGS)
56
             that indicate if each object meets the redshift quality criteria of specific tracers
57
    '''
58

59
    output_columns = ['GOOD_BGS', 'GOOD_LRG', 'GOOD_ELG', 'GOOD_QSO']
×
60
    if return_target_columns:
×
61
        output_columns += ['LRG', 'ELG', 'QSO', 'ELG_LOP', 'ELG_HIP', 'ELG_VLO', 'BGS_ANY', 'BGS_FAINT', 'BGS_BRIGHT']
×
62

63
    if ignore_emline:
×
64
        output_columns.remove('GOOD_ELG')
×
65

66
    if extra_columns is None:
×
67
        extra_columns = ['TARGETID', 'Z', 'ZWARN', 'COADD_FIBERSTATUS']
×
68
    output_columns = output_columns + list(np.array(extra_columns)[~np.in1d(extra_columns, output_columns)])
×
69

70
    ############################ Load data ############################
71

72
    columns_redshifts = ['TARGETID', 'CHI2', 'Z', 'ZERR', 'ZWARN', 'SPECTYPE', 'DELTACHI2']
×
73
    columns_fibermap = ['TARGETID', 'COADD_FIBERSTATUS', 'TARGET_RA', 'TARGET_DEC']
×
74
    columns_emline = ['TARGETID', 'OII_FLUX', 'OII_FLUX_IVAR']
×
75
    columns_qso_mgii = ['TARGETID', 'IS_QSO_MGII']
×
76
    columns_qso_qn = ['TARGETID', 'Z_NEW', 'ZERR_NEW', 'IS_QSO_QN_NEW_RR', 'C_LYA', 'C_CIV', 'C_CIII', 'C_MgII', 'C_Hbeta', 'C_Halpha']
×
77

78
    dir_path = os.path.dirname(redrock_path)
×
79
    qso_mgii_path = os.path.join(dir_path, os.path.basename(redrock_path).replace('redrock-', 'qso_mgii-'))
×
80
    qso_qn_path = os.path.join(dir_path, os.path.basename(redrock_path).replace('redrock-', 'qso_qn-'))
×
81
    if emline_path is None:
×
82
        emline_path = os.path.join(dir_path, os.path.basename(redrock_path).replace('redrock-', 'emline-'))
×
83

84
    tmp_redshifts = Table(fitsio.read(redrock_path, ext='REDSHIFTS', columns=columns_redshifts))
×
85

86
    # read the full fibermap until we determine the targeting columns
87
    from desitarget.targets import main_cmx_or_sv
×
88
    tmp_fibermap = fitsio.read(redrock_path, ext='FIBERMAP')
×
89
    surv_target, surv_mask, surv = main_cmx_or_sv(tmp_fibermap)
×
90
    if surv.lower() == 'cmx':
×
91
        raise NotImplementedError('Determining valid redshifts for commissioning targets is not supported.')
92

93
    desi_target_col, bgs_target_col, _ = surv_target
×
94
    desi_mask, bgs_mask, _ = surv_mask
×
95
    tmp_fibermap = Table(tmp_fibermap[columns_fibermap + surv_target])
×
96

97
    tmp_qso_mgii = Table(fitsio.read(qso_mgii_path, columns=(columns_qso_mgii)))
×
98
    tmp_qso_qn = Table(fitsio.read(qso_qn_path, columns=(columns_qso_qn)))
×
99
    if not ignore_emline:
×
100
        tmp_emline = Table(fitsio.read(emline_path, columns=(columns_emline)))
×
101

102
    # Sanity check
103
    tid = tmp_redshifts['TARGETID'].copy()
×
104
    if not (np.all(tid==tmp_fibermap['TARGETID']) \
×
105
            # and np.all(tid==tmp_emline['TARGETID']) \
106
            and np.all(tid==tmp_qso_mgii['TARGETID']) \
107
            and np.all(tid==tmp_qso_qn['TARGETID'])):
108
        raise ValueError
×
109
    if (not ignore_emline) and (not np.all(tid==tmp_emline['TARGETID'])):
×
110
        raise ValueError
×
111

112
    tmp_fibermap.remove_column('TARGETID')
×
113
    tmp_qso_mgii.remove_column('TARGETID')
×
114
    tmp_qso_qn.remove_column('TARGETID')
×
115
    if not ignore_emline:
×
116
        tmp_emline.remove_column('TARGETID')
×
117

118
    if not ignore_emline:
×
119
        cat = hstack([tmp_redshifts, tmp_fibermap, tmp_qso_mgii, tmp_qso_qn, tmp_emline], join_type='inner')
×
120
    else:
121
        cat = hstack([tmp_redshifts, tmp_fibermap, tmp_qso_mgii, tmp_qso_qn], join_type='inner')
×
122

123
    if return_target_columns:
×
124
        for name in ['LRG', 'ELG', 'QSO', 'ELG_LOP', 'ELG_HIP', 'ELG_VLO', 'BGS_ANY', 'BGS_FAINT', 'BGS_BRIGHT']:
×
125
            if name in ['BGS_FAINT', 'BGS_BRIGHT']:
×
126
                cat[name] = cat[bgs_target_col] & bgs_mask[name] > 0
×
127
            else:
128
                if name in desi_mask.names(): # not all bits were used in SV (e.g., ELG_LOP)
×
129
                    cat[name] = cat[desi_target_col] & desi_mask[name] > 0
×
130
                else:
131
                    cat[name] = np.zeros(len(cat), bool)
×
132
        # # Bitmask definitions: https://github.com/desihub/desitarget/blob/master/py/desitarget/data/targetmask.yaml
133
        # cat['LRG'] = cat['DESI_TARGET'] & 2**0 > 0
134
        # cat['ELG'] = cat['DESI_TARGET'] & 2**1 > 0
135
        # cat['QSO'] = cat['DESI_TARGET'] & 2**2 > 0
136
        # cat['ELG_LOP'] = cat['DESI_TARGET'] & 2**5 > 0
137
        # cat['ELG_HIP'] = cat['DESI_TARGET'] & 2**6 > 0
138
        # cat['ELG_VLO'] = cat['DESI_TARGET'] & 2**7 > 0
139
        # cat['BGS_ANY'] = cat['DESI_TARGET'] & 2**60 > 0
140
        # cat['BGS_FAINT'] = cat['BGS_TARGET'] & 2**0 > 0
141
        # cat['BGS_BRIGHT'] = cat['BGS_TARGET'] & 2**1 > 0
142

143
    cat = actually_validate(cat, fiberstatus_cut=fiberstatus_cut, ignore_emline=ignore_emline)
×
144
    cat = cat[output_columns]
×
145

146
    return cat
×
147

148

149
def actually_validate(cat, fiberstatus_cut=True, ignore_emline=False):
×
150
    '''
151
    Apply redshift quality criteria
152

153
    Args:
154
        cat: astropy table with the necessary columns for redshift quality determination
155

156
    Options:
157
        fiberstatus_cut: bool (default True), if True, impose requirements on COADD_FIBERSTATUS and ZWARN
158
        return_target_columns: bool (default False), if True, include columns that indicate if the object belongs to each class of DESI targets
159
        extra_columns: list of str (default None), additional columns to include in the output
160
        emline_path: str (default None), specify the location of the emline file; by default the emline file is in the same directory as the redrock file
161
        ignore_emline: bool (default False), if True, ignore the emline file and do not validate the ELG redshift
162

163
    Returns:
164
        cat: astropy table with boolean columns (e.g., GOOD_BGS) added
165
    '''
166

167
    # BGS
168
    cat['GOOD_BGS'] = cat['ZWARN']==0
×
169
    cat['GOOD_BGS'] &= cat['DELTACHI2']>40
×
170
    if fiberstatus_cut:
×
171
        cat['GOOD_BGS'] &= get_good_fiberstatus(cat)
×
172

173
    # LRG
174
    cat['GOOD_LRG'] = cat['ZWARN']==0
×
175
    cat['GOOD_LRG'] &= cat['Z']<1.5
×
176
    cat['GOOD_LRG'] &= cat['DELTACHI2']>15
×
177
    if fiberstatus_cut:
×
178
        cat['GOOD_LRG'] &= get_good_fiberstatus(cat)
×
179

180
    # ELG
181
    if not ignore_emline:
×
182
        with warnings.catch_warnings():
×
183
            warnings.simplefilter('ignore')
×
184
            cat['GOOD_ELG'] = (cat['OII_FLUX']>0) & (cat['OII_FLUX_IVAR']>0)
×
185
            cat['GOOD_ELG'] &= np.log10(cat['OII_FLUX'] * np.sqrt(cat['OII_FLUX_IVAR'])) > 0.9 - 0.2 * np.log10(cat['DELTACHI2'])
×
186
        if fiberstatus_cut:
×
187
            cat['GOOD_ELG'] &= get_good_fiberstatus(cat)
×
188

189
    # QSO - adopted from the code from Edmond
190
    # https://github.com/echaussidon/LSS/blob/8ca53f4c38cfa29722ee6958687e188cc894ed2b/py/LSS/qso_cat_utils.py#L282
191
    cat['IS_QSO_QN'] = np.max(np.array([cat[name] for name in ['C_LYA', 'C_CIV', 'C_CIII', 'C_MgII', 'C_Hbeta', 'C_Halpha']]), axis=0) > 0.95
×
192
    cat['IS_QSO_QN_NEW_RR'] &= cat['IS_QSO_QN']
×
193
    cat['QSO_MASKBITS'] = np.zeros(len(cat), dtype=int)
×
194
    cat['QSO_MASKBITS'][cat['SPECTYPE']=='QSO'] += 2**1
×
195
    cat['QSO_MASKBITS'][cat['IS_QSO_MGII']] += 2**2
×
196
    cat['QSO_MASKBITS'][cat['IS_QSO_QN']] += 2**3
×
197
    cat['QSO_MASKBITS'][cat['IS_QSO_QN_NEW_RR']] += 2**4
×
198
    cat['Z'][cat['IS_QSO_QN_NEW_RR']] = cat['Z_NEW'][cat['IS_QSO_QN_NEW_RR']].copy()
×
199
    cat['ZERR'][cat['IS_QSO_QN_NEW_RR']] = cat['ZERR_NEW'][cat['IS_QSO_QN_NEW_RR']].copy()
×
200
    # Correct bump at z~3.7
201
    sel_pb_redshift = (((cat['Z'] > 3.65) & (cat['Z'] < 3.9)) | ((cat['Z'] > 5.15) & (cat['Z'] < 5.35))) & ((cat['C_LYA'] < 0.95) | (cat['C_CIV'] < 0.95))
×
202
    cat['QSO_MASKBITS'][sel_pb_redshift] = 0
×
203
    cat['GOOD_QSO'] = cat['QSO_MASKBITS']>0
×
204
    if fiberstatus_cut:
×
205
        cat['GOOD_QSO'] &= get_good_fiberstatus(cat, isqso=True)
×
206

207
    return cat
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc