• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

desihub / desispec / 11924426179

20 Nov 2024 12:39AM UTC coverage: 30.072% (-0.09%) from 30.16%
11924426179

Pull #2411

github

segasai
reformat
Pull Request #2411: save the trace shift offsets in the psf file

1 of 31 new or added lines in 3 files covered. (3.23%)

1675 existing lines in 20 files now uncovered.

14637 of 48673 relevant lines covered (30.07%)

0.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/py/desispec/workflow/proc_dashboard_funcs.py
1
"""
2
desispec.workflow.proc_dashboard_funcs
3
======================================
4

5
"""
6
import os,glob
×
7
import json
×
8
import sys
×
9
import re
×
10
import time,datetime
×
11
import numpy as np
×
12
from os import listdir
×
13
from astropy.table import Table
×
14
from astropy.io import fits
×
15

16
########################
17
### Helper Functions ###
18
########################
UNCOV
19
from desispec.io import rawdata_root, specprod_root
×
20
from desispec.io.util import camword_to_spectros, decode_camword, \
×
21
    difference_camwords, create_camword, parse_badamps
UNCOV
22
from desispec.workflow.exptable import get_exposure_table_column_types, \
×
23
    default_obstypes_for_exptable, get_exposure_table_column_defaults, \
24
    get_exposure_table_pathname
25
from desispec.workflow.proctable import get_processing_table_pathname
×
26
from desispec.workflow.tableio import load_table
×
27

28

UNCOV
29
def get_output_dir(desi_spectro_redux, specprod, output_dir, makedir=True):
×
30
    if 'DESI_SPECTRO_DATA' not in os.environ.keys():
×
31
        os.environ['DESI_SPECTRO_DATA'] = '/global/cfs/cdirs/desi/spectro/data/'
×
32

UNCOV
33
    if specprod is None:
×
34
        if 'SPECPROD' not in os.environ.keys():
×
35
            os.environ['SPECPROD'] = 'daily'
×
36
        specprod = os.environ['SPECPROD']
×
37
    else:
UNCOV
38
        os.environ['SPECPROD'] = specprod
×
39

UNCOV
40
    if desi_spectro_redux is None:
×
41
        if 'DESI_SPECTRO_REDUX' not in os.environ.keys():  # these are not set by default in cronjob mode.
×
42
            os.environ['DESI_SPECTRO_REDUX'] = \
×
43
                '/global/cfs/cdirs/desi/spectro/redux/'
UNCOV
44
        desi_spectro_redux = os.environ['DESI_SPECTRO_REDUX']
×
45
    else:
UNCOV
46
        os.environ['DESI_SPECTRO_REDUX'] = desi_spectro_redux
×
47

48
    ## Verify the production directory exists
UNCOV
49
    prod_dir = os.path.join(desi_spectro_redux, specprod)
×
50
    if not os.path.exists(prod_dir):
×
51
        raise ValueError(
×
52
            f"Path {prod_dir} doesn't exist for production directory.")
53

54
    ## Define output_dir if not defined
UNCOV
55
    if output_dir is None:
×
56
        if 'DESI_DASHBOARD' not in os.environ.keys():
×
57
            os.environ['DESI_DASHBOARD'] = os.path.join(prod_dir,
×
58
                                                        'run', 'dashboard')
UNCOV
59
        output_dir = os.environ["DESI_DASHBOARD"]
×
60
    else:
UNCOV
61
        os.environ['DESI_DASHBOARD'] = output_dir
×
62

63
    ## Ensure we have directories to output to
UNCOV
64
    if makedir:
×
65
        os.makedirs(output_dir, exist_ok=True)
×
66

UNCOV
67
    return output_dir, prod_dir
×
68

UNCOV
69
def get_nights_dict(nights_arg, start_night, end_night, prod_dir):
×
70
    if nights_arg is None or nights_arg == 'all' \
×
71
            or (',' not in nights_arg and int(nights_arg) < 20000000):
UNCOV
72
        nights = list()
×
UNCOV
73
        for n in listdir(
×
74
                os.path.join(prod_dir, 'run', 'scripts', 'night')):
75
            # - nights are 20YYMMDD
UNCOV
76
            if re.match('^20\d{6}$', n):
×
UNCOV
77
                nights.append(n)
×
78
    else:
UNCOV
79
        nights = [nigh.strip(' \t') for nigh in nights_arg.split(',')]
×
80

81
    # tonight=what_night_is_it()   # Disabled per Anthony's request
82
    # if str(tonight) not in nights:
83
    #    nights.append(str(tonight))
UNCOV
84
    nights.sort(reverse=True)
×
85

86
    nights = np.array(nights).astype(int)
×
87

88
    if start_night is not None:
×
89
        nights = nights[np.where(int(start_night) <= nights)[0]]
×
UNCOV
90
    if end_night is not None:
×
UNCOV
91
        nights = nights[np.where(int(end_night) >= nights)[0]]
×
92

93
    if nights_arg is not None and nights_arg.isnumeric() and len(
×
94
            nights) >= int(nights_arg):
95
        if end_night is None or start_night is not None:
×
UNCOV
96
            print(f"Only showing the most recent {int(nights_arg)} days")
×
UNCOV
97
            nights = nights[:int(nights_arg)]
×
98
        else:
UNCOV
99
            nights = nights[-1 * int(nights_arg):]
×
100

UNCOV
101
    nights_dict = dict()
×
102
    for night in nights:
×
UNCOV
103
        month = str(night)[:6]
×
104
        if month not in nights_dict.keys():
×
105
            nights_dict[month] = [night]
×
106
        else:
107
            nights_dict[month].append(night)
×
108

109
    return nights_dict, nights
×
110

111
def get_tables(night, check_on_disk=False, exptab_colnames=None):
×
112
    if exptab_colnames is None:
×
113
        exptab_colnames = ['EXPID', 'FA_SURV', 'FAPRGRM', 'CAMWORD', 'BADCAMWORD',
×
114
                           'BADAMPS', 'EXPTIME', 'OBSTYPE', 'TILEID', 'COMMENTS',
115
                           'LASTSTEP']
116

117
    file_exptable = get_exposure_table_pathname(night)
×
UNCOV
118
    file_processing = get_processing_table_pathname(specprod=None,
×
119
                                                    prodmod=night)
120
    # procpath,procname = os.path.split(file_processing)
121
    # file_unprocessed = os.path.join(procpath,procname.replace('processing','unprocessed'))
UNCOV
122
    edefs = get_exposure_table_column_defaults(asdict=True)
×
UNCOV
123
    for col in exptab_colnames:
×
UNCOV
124
        if col not in edefs.keys():
×
UNCOV
125
            ValueError(f"requested dashboard exposure table column {col} not" +
×
126
                       f" in the exposure table columns: {edefs.keys()}.")
127

UNCOV
128
    try:  # Try reading tables first. Switch to counting files if failed.
×
UNCOV
129
        d_exp = load_table(file_exptable, tabletype='exptable')
×
UNCOV
130
        for col in exptab_colnames:
×
UNCOV
131
            if col not in d_exp.colnames:
×
UNCOV
132
                d_exp[col] = edefs[col]
×
UNCOV
133
    except:
×
UNCOV
134
        print(
×
135
            f'WARNING: Error reading exptable for {night}. Changing check_on_disk to True and scanning files on disk.')
UNCOV
136
        etypes = get_exposure_table_column_types(asdict=True)
×
UNCOV
137
        exptab_dtypes = [etypes[col] for col in exptab_colnames]
×
UNCOV
138
        d_exp = Table(names=exptab_colnames, dtype=exptab_dtypes)
×
UNCOV
139
        check_on_disk = True
×
140

UNCOV
141
    unaccounted_for_expids, unaccounted_for_tileids = [], []
×
UNCOV
142
    if check_on_disk:
×
UNCOV
143
        rawdatatemplate = os.path.join(rawdata_root(), night, '{zexpid}',
×
144
                                       'desi-{zexpid}.fits.fz')
UNCOV
145
        rawdata_fileglob = rawdatatemplate.format(zexpid='*')
×
UNCOV
146
        known_exposures = set(list(d_exp['EXPID']))
×
147
        newexpids = list(find_new_exps(rawdata_fileglob, known_exposures))
×
UNCOV
148
        newexpids.sort(reverse=True)
×
149
        default_obstypes = default_obstypes_for_exptable()
×
150
        for expid in newexpids:
×
151
            zfild_expid = str(expid).zfill(8)
×
UNCOV
152
            filename = rawdatatemplate.format(zexpid=zfild_expid)
×
UNCOV
153
            h1 = fits.getheader(filename, 1)
×
154
            header_info = {keyword: 'unknown' for keyword in
×
155
                           ['SPCGRPHS', 'EXPTIME',
156
                            'FA_SURV', 'FAPRGRM'
157
                                       'OBSTYPE', 'TILEID']}
UNCOV
158
            for keyword in header_info.keys():
×
159
                if keyword in h1.keys():
×
160
                    header_info[keyword] = h1[keyword]
×
161

162
            if header_info['OBSTYPE'] in default_obstypes:
×
UNCOV
163
                header_info['EXPID'] = expid
×
UNCOV
164
                header_info['LASTSTEP'] = 'all'
×
165
                header_info['COMMENTS'] = []
×
UNCOV
166
                if header_info['SPCGRPHS'] != 'unknown':
×
UNCOV
167
                    specs = str(header_info['SPCGRPHS']).replace(' ', '').replace(',', '')
×
UNCOV
168
                    header_info['CAMWORD'] = f'a{specs}'
×
169
                else:
170
                    header_info['CAMWORD'] = header_info['SPCGRPHS']
×
UNCOV
171
                header_info.pop('SPCGRPHS')
×
UNCOV
172
                d_exp.add_row(header_info)
×
173
                unaccounted_for_expids.append(expid)
×
UNCOV
174
                unaccounted_for_tileids.append(header_info['TILEID'])
×
175

UNCOV
176
    try:
×
UNCOV
177
        d_processing = load_table(file_processing, tabletype='proctable')
×
UNCOV
178
    except:
×
UNCOV
179
        d_processing = None
×
UNCOV
180
        print('WARNING: Error reading proctable. Only exposures in preproc'
×
181
              + ' directory will be marked as processing.')
182

UNCOV
183
    return d_exp, d_processing, np.array(unaccounted_for_expids), \
×
184
           np.unique(unaccounted_for_tileids)
185

UNCOV
186
def get_terminal_steps(expected_by_type):
×
187
    ## Determine the last filetype that is expected for each obstype
UNCOV
188
    terminal_steps = dict()
×
UNCOV
189
    for obstype, expected in expected_by_type.items():
×
UNCOV
190
        terminal_steps[obstype] = None
×
191
        keys = list(expected.keys())
×
192
        for key in reversed(keys):
×
193
            if expected[key] > 0:
×
194
                terminal_steps[obstype] = key
×
UNCOV
195
                break
×
UNCOV
196
    return terminal_steps
×
197

198
def get_file_list(filename, doaction=True):
×
199
    if doaction and filename is not None and os.path.exists(filename):
×
200
        output = np.atleast_1d(np.loadtxt(filename, dtype=int)).tolist()
×
201
    else:
202
        output = []
×
203
    return output
×
204

UNCOV
205
def get_skipped_ids(expid_filename, skip_ids=True):
×
206
    return get_file_list(filename=expid_filename, doaction=skip_ids)
×
207

208
def what_night_is_it():
×
209
    """
210
    Return the current night
211
    """
212
    d = datetime.datetime.utcnow() - datetime.timedelta(7 / 24 + 0.5)
×
213
    tonight = int(d.strftime('%Y%m%d'))
×
UNCOV
214
    return tonight
×
215

216
def find_new_exps(fileglob, known_exposures):
×
217
    """
218
    Check the path given for new exposures
219
    """
UNCOV
220
    datafiles = sorted(glob.glob(fileglob))
×
UNCOV
221
    newexp = list()
×
222
    for filepath in datafiles:
×
223
        expid = int(os.path.basename(os.path.dirname(filepath)))
×
UNCOV
224
        if expid not in known_exposures:
×
UNCOV
225
            newexp.append(expid)
×
226

227
    return set(newexp)
×
228

229
def check_running(proc_name= 'desi_dailyproc',suppress_outputs=False):
×
230
    """
231
    Check if the desi_dailyproc process is running
232
    """
233
    import psutil
×
234
    running = False
×
235
    mypid = os.getpid()
×
236
    for p in psutil.process_iter():
×
237
        if p.pid != mypid and proc_name in ' '.join(p.cmdline()):
×
238
            if not suppress_outputs:
×
239
                print('ERROR: {} already running as PID {}:'.format(proc_name,p.pid))
×
UNCOV
240
                print('  ' + ' '.join(p.cmdline()))
×
241
            running = True
×
242
            break
×
243
    return running
×
244

245

246
#################################
247
### HTML Generating Functions ###
248
#################################
UNCOV
249
def return_color_profile():
×
250
    color_profile = dict()
×
251
    color_profile['DEFAULT'] = {'font':'#000000' ,'background':'#ccd1d1'} # gray
×
252
    color_profile['NULL'] = {'font': '#34495e', 'background': '#ccd1d1'}  # gray on gray
×
253
    color_profile['BAD'] = {'font':'#000000' ,'background':'#d98880'}  #  red
×
254
    color_profile['INCOMPLETE'] = {'font': '#000000','background':'#f39c12'}  #  orange
×
255
    color_profile['GOOD'] = {'font':'#000000' ,'background':'#7fb3d5'}   #  blue
×
256
    color_profile['OVERFULL'] = {'font': '#000000','background':'#c39bd3'}   # purple
×
257
    return color_profile
×
258

259

UNCOV
260
def make_html_page(monthly_tables, outfile, titlefill='Processing',
×
261
                   show_null=False, color_profile=None):
UNCOV
262
    if color_profile is None:
×
263
        color_profile = return_color_profile()
×
264

265
    html_page = _initialize_page(color_profile, titlefill=titlefill)
×
266

267
    for month, nightly_tables in monthly_tables.items():
×
268
        if len(nightly_tables) == 0:
×
269
            continue
×
270
        print(
×
271
            "Month: {}, nights: {}".format(month, list(nightly_tables.keys())))
272
        nightly_table_htmls, statuses = list(), list()
×
273
        for night, night_info in nightly_tables.items():
×
UNCOV
274
            if len(night_info) == 0:
×
275
                continue
×
276
            ####################################
277
            ### Table for individual night ####
278
            ####################################
279
            nightly_table_html, status = \
×
280
                generate_nightly_table_html(night_info, night, show_null)
281
            nightly_table_htmls.append(nightly_table_html)
×
282
            statuses.append(status)
×
283
        html_page += generate_monthly_table_html(nightly_table_htmls,
×
284
                                                 statuses, month)
285

286
    # html_page += js_import_str(os.environ['DESI_DASHBOARD'])
UNCOV
287
    html_page += js_str()
×
288
    html_page += _closing_str()
×
UNCOV
289
    with open(outfile, 'w') as hs:
×
UNCOV
290
        hs.write(html_page)
×
291
        print(f"Write to {outfile} complete.")
×
292

293
    if 'NERSC_HOST' in os.environ and outfile.startswith(
×
294
            '/global/cfs/cdirs/desi'):
295
        url = outfile.replace('/global/cfs/cdirs/desi',
×
296
                              'https://data.desi.lbl.gov/desi')
297
        print(f"This can be found via webserver at: {url}")
×
298

299
def generate_monthly_table_html(tables, statuses, month):
×
300
    """
301
    Add a collapsible and extendable table to the html file for a specific month
302
    Input
303
    tables: list of tables generated by 'nightly_table'
304
    month: string of YYYYMM, e.g. 202001
305
    output: The string to be added to the html file
306
    """
307
    month_dict = {'01':'January','02':'February','03':'March','04':'April','05':'May','06':'June',
×
308
                  '07':'July','08':'August','09':'September','10':'October','11':'November','12':'December'}
309

310
    heading = f"{month_dict[month[4:]]} {month[:4]} ({month})"
×
311
    month_table_str = '\n<!--Begin {}-->\n'.format(month)
×
312

313
    statuses = np.array(statuses)
×
UNCOV
314
    monthlystatus = 'DEFAULT'
×
UNCOV
315
    if np.all(statuses == 'GOOD'):
×
UNCOV
316
        monthlystatus = 'GOOD'
×
317
    elif np.any(statuses == 'BAD'):
×
318
        monthlystatus = 'BAD'
×
319
    elif np.any(statuses == 'INCOMPLETE'):
×
UNCOV
320
        monthlystatus = 'INCOMPLETE'
×
321
    elif np.any(statuses == 'OVERFULL'):
×
UNCOV
322
        monthlystatus = 'OVERFULL'
×
UNCOV
323
    month_table_str += f'<button class="collapsible" id="{monthlystatus}">' \
×
324
                       + heading + '</button>'
325

326
    month_table_str += '<div class="content" style="display:inline-block;min-height:0%;">\n'
×
327
    #month_table_str += "<table id='c'>"
328
    for table_str in tables:
×
329
        month_table_str += table_str
×
330

331
    #month_table_str += "</table></div>\n"
332
    month_table_str += "</div>\n"
×
UNCOV
333
    month_table_str += '<!--End {}-->\n\n'.format(month)
×
334

UNCOV
335
    return month_table_str
×
336

UNCOV
337
def generate_nightly_table_html(night_info, night, show_null):
×
338
    if len(night_info) == 0:
×
339
        return '', 'NULL'
×
340

341
    ngood, ninter, nbad, nnull, nover, n_notnull, noprocess, norecord = \
×
342
        0, 0, 0, 0, 0, 0, 0, 0
343

344
    main_body = ""
×
345
    for key, row_info in reversed(night_info.items()):
×
346
        table_row = _table_row(row_info)
×
347
        if not show_null and 'NULL' in table_row:
×
348
            continue
×
UNCOV
349
        main_body += ("\t" + table_row + "\n")
×
UNCOV
350
        status = str(row_info["STATUS"]).lower()
×
UNCOV
351
        if status == 'processing':
×
UNCOV
352
            if 'COLOR' in row_info:
×
UNCOV
353
                color = row_info['COLOR']
×
354
            else:
355
                color = table_row.split('">')[0].split('id="')[1]
×
356
            if color == 'GOOD':
×
357
                ngood += 1
×
UNCOV
358
                n_notnull += 1
×
359
            elif color == 'BAD':
×
360
                nbad += 1
×
361
                n_notnull += 1
×
362
            elif color == 'INCOMPLETE':
×
363
                ninter += 1
×
364
                n_notnull += 1
×
365
            elif color == 'OVERFULL':
×
366
                nover += 1
×
367
                n_notnull += 1
×
368
            else:
UNCOV
369
                nnull += 1
×
370
        elif status == 'unprocessed':
×
UNCOV
371
            noprocess += 1
×
372
        elif status == 'unrecorded':
×
373
            norecord += 1
×
374
        else:
375
            nnull += 1
×
376

377
    # Night dropdown table
378
    htmltab = r'&nbsp;&nbsp;&nbsp;&nbsp;'
×
379
    heading = (f"Night {night}{htmltab}"
×
380
               + f"Complete: {ngood}/{n_notnull}{htmltab}"
381
               + f"Incomplete: {ninter}/{n_notnull}{htmltab}"
382
               + f"Failed: {nbad}/{n_notnull}{htmltab}"
383
               + f"Overfull: {nover}/{n_notnull}{htmltab}"
384
               + f"Unprocessed: {noprocess}{htmltab}"
385
               + f"NoTabEntry: {norecord}{htmltab}"
386
               + f"Other: {nnull}"
387
               )
388

389
    night_status = 'DEFAULT'
×
UNCOV
390
    if ngood == n_notnull:
×
391
        night_status = "GOOD"
×
392
    elif ninter > 0:
×
393
        night_status = "INCOMPLETE"
×
UNCOV
394
    elif nbad > 0:
×
UNCOV
395
        night_status = "BAD"
×
UNCOV
396
    elif nover > 0:
×
397
        night_status = "OVERFULL"
×
398
    nightly_table_str = '<!--Begin {}-->\n'.format(night)
×
399
    nightly_table_str += f'<button class="collapsible" id="{night_status}">{heading}</button>'
×
400
    nightly_table_str += '<div class="content" style="display:inline-block;min-height:0%;">\n'
×
401
    # table header
UNCOV
402
    nightly_table_str += "<table id='c' class='nightTable'><tbody>\n\t<tr>"
×
403
    for col in list(row_info.keys()):
×
UNCOV
404
        colname = str(col).upper()
×
405
        if colname != 'COLOR':
×
UNCOV
406
            nightly_table_str += f"<th>{colname}</th>"
×
407
    nightly_table_str += "</tr>\n"
×
408

409
    # Add body
UNCOV
410
    nightly_table_str += main_body
×
411

412
    # End table
UNCOV
413
    nightly_table_str += "</tbody></table></div>\n"
×
UNCOV
414
    nightly_table_str += '<!--End {}-->\n\n'.format(night)
×
UNCOV
415
    return nightly_table_str, night_status
×
416

417

UNCOV
418
def read_json(filename_json):
×
UNCOV
419
    night_json_info = None
×
420
    if os.path.exists(filename_json):
×
421
        with open(filename_json) as json_file:
×
UNCOV
422
            try:
×
423
                night_json_info = json.load(json_file)
×
424
            except:
×
425
                print(f"Error trying to load {filename_json}, "
×
426
                      + "continuing without that information.")
427
    return night_json_info
×
428

429

430
def write_json(output_data, filename_json):
×
431
    ## write out the night_info to json file
432
    with open(filename_json, 'w') as json_file:
×
433
        try:
×
UNCOV
434
            json.dump(output_data, json_file, indent='\t')
×
UNCOV
435
        except:
×
436
            print(f"Error trying to dump {filename_json}, "
×
437
                  + "not saving that information.")
438

439

UNCOV
440
def _initialize_page(color_profile, titlefill='Processing'):
×
441
    """
442
    Initialize the html file for showing the statistics, giving all the headers and CSS setups.
443
    """
444
    # strTable="<html><style> table {font-family: arial, sans-serif;border-collapse: collapse;width: 100%;}"
445
    # strTable=strTable+"td, th {border: 1px solid #dddddd;text-align: left;padding: 8px;}"
446
    # strTable=strTable+"tr:nth-child(even) {background-color: #dddddd;}</style>"
447
    html_page = """<html><head>
×
448
<meta http-equiv="content-type" content="text/html; charset=UTF-8"><style>
449
    h1 {font-family: 'sans-serif';font-size:50px;color:#4CAF50}
450
    #c {font-family: 'Trebuchet MS', Arial, Helvetica, sans-serif;border-collapse: collapse;width: 100%;}
451
    #c td, #c th {border: 1px solid #ddd;padding: 8px;}
452
    /* #c tr:nth-child(even){background-color: #f2f2f2;} */
453
    #c tr:hover {background-color: #ddd;}
454
    #c th {padding-top: 12px;  padding-bottom: 12px;  text-align: left;  background-color: #34495e;  color: white;}
455
    .collapsible {background-color: #eee;color: #444;cursor: pointer;padding: 18px;width: 100%;border: none;text-align: left;outline: none;font-size: 25px;}
456
    .regular {background-color: #eee;color: #444;  cursor: pointer;  padding: 18px;  width: 25%;  border: 18px;  text-align: left;  outline: none;  font-size: 25px;}
457
    .active, .collapsible:hover { background-color: #ccc;}
458
    .content {padding: 0 18px;display: table;overflow: hidden;background-color: #f1f1f1;maxHeight:0px;}
459
    /* The Modal (background) */
460
    .modal {
461
    display: none;        /* Hidden by default */
462
    position: fixed;      /* Stay in place */
463
    z-index: 1;           /* Sit on top */
464
    padding-top: 100px;  /* Location of the box */
465
    left: 0;
466
    top: 0;
467
    width: 100%;         /* Full width */
468
    height: 90%;        /* Full height */
469
    overflow: auto;     /* Enable scroll if needed */
470
    background-color: rgb(0,0,0);      /* Fallback color */
471
    background-color: rgba(0,0,0,0.4); /* Black w/ opacity */
472
    }
473

474
    /* Modal Content */
475
    .modal-content {
476
    background-color: #fefefe;
477
    margin: auto;
478
    padding: 20px;
479
    border: 1px solid #888;
480
    width: 80%;
481
    }
482

483

484
   /* The Close Button */
485
   .close {
486
    color: #aaaaaa;
487
    float: right;
488
    font-size: 28px;
489
    font-weight: bold;
490
    }
491
    .close:hover,
492
    .close:focus {
493
         color: #000;
494
         text-decoration: none;
495
         cursor: pointer;
496
     }
497

498
    #obstypelist {
499

500
          background-position: 10px 10px;
501
          background-repeat: no-repeat;
502
          width: 10%;
503
          font-size: 16px;
504
          padding: 12px 20px 12px 40px;
505
          border: 1px solid #ddd;
506
          margin-bottom: 12px;
507
    }
508
    #exptimelist {
509

510
          background-position: 10px 10px;
511
          background-repeat: no-repeat;
512
          width: 10%;
513
          font-size: 16px;
514
          padding: 12px 20px 12px 40px;
515
          border: 1px solid #ddd;
516
          margin-bottom: 12px;
517
    }
518

519
    """
520

521
    for ctype,cdict in color_profile.items():
×
522
        background = cdict['background']
×
523
        html_page += f'\t#{ctype} ' + '{background-color:' + f'{background}' + ';}\n'
×
524

525
    html_page + "\n"
×
526
    ## Table rows shouldn't do the default background because of cell coloring
UNCOV
527
    for ctype,cdict in color_profile.items():
×
528
        font = cdict['font']
×
UNCOV
529
        background = '#eee'#cdict['background'] # no background for a whole table after implementing color codes for processing columns
×
UNCOV
530
        html_page += f'\ttable tr#{ctype} '+'{background-color:'+f'{background}; color:{font}'+';}\n'
×
531

532
    ## Finally there is a class of table element that is null in a good way
533
    ## Label as such
UNCOV
534
    html_page += 'table td#GOODNULL {background-color:#7fb3d5;color:gray}\n'
×
535

536
    html_page += '</style>\n\n'
×
537
    html_page += f"</head><body><h1>DESI '{os.environ['SPECPROD']}' "
×
538
    html_page += f'{titlefill} Status Monitor</h1>\n'
×
539

540
    timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
×
541
    # running='No'
542
    # if check_running(proc_name='desi_dailyproc',suppress_outputs=True):
543
    #     running='Yes'
544
    #     strTable=strTable+"<div style='color:#00FF00'>{} {} running: {}</div>\n".format(timestamp,'desi_dailyproc',running)
545
    script = os.path.basename(sys.argv[0])
×
UNCOV
546
    html_page += f'<div style="color:#00FF00;margin-bottom:20px"> {script} running at: {timestamp}</div>\n'
×
547

548
    html_page += 'Color Legend:\n'
×
UNCOV
549
    html_page += '<table style="margin-bottom:20px;margin-left:20px"><tr>\n'
×
550

551
    for ctype in color_profile.keys():
×
552
        html_page += f' <td id="{ctype}">{ctype}</td>\n'
×
553
    html_page += f'</tr></table>\n'
×
554

UNCOV
555
    html_page += '\n\n'
×
UNCOV
556
    html_page += """Filter By Status:
×
557
    <select style="margin-bottom:10px" id="statuslist" onchange="filterByStatus()" class='form-control'>
558
    <option>processing</option>
559
    <option>unprocessed</option>
560
    <option>unaccounted</option>
561
    <option>ALL</option>
562
    </select>
563
    """
564
    # The following codes are for filtering rows by obstype and exptime. Not in use for now, but basically can be enabled anytime.
565
    # html_page +="""Filter By OBSTYPE:
566
    # <select id="obstypelist" onchange="filterByObstype()" class='form-control'>
567
    # <option>ALL</option>
568
    # <option>SCIENCE</option>
569
    # <option>FLAT</option>
570
    # <option>ARC</option>
571
    # <option>DARK</option>
572
    # </select>
573
    # Exptime Limit:
574
    # <select id="exptimelist" onchange="filterByExptime()" class='form-control'>
575
    # <option>ALL</option>
576
    # <option>5</option>
577
    # <option>30</option>
578
    # <option>120</option>
579
    # <option>900</option>
580
    # <option>1200</option>
581
    # </select>
582
    # """
UNCOV
583
    return html_page
×
584

UNCOV
585
def _closing_str():
×
UNCOV
586
    closing = """<div class="crt-wrapper"></div>
×
587
                 <div class="aadvantage-wrapper"></div>
588
                 </body></html>"""
UNCOV
589
    return closing
×
590

UNCOV
591
def _table_row(dictionary):
×
UNCOV
592
    idlabel = dictionary.pop('COLOR')
×
UNCOV
593
    color_profile = return_color_profile()
×
UNCOV
594
    if dictionary["STATUS"] != 'processing':
×
UNCOV
595
        style_str = 'display:none;'
×
596
    else:
UNCOV
597
        style_str = ''
×
598

UNCOV
599
    if idlabel is None:
×
UNCOV
600
        row_str = '<tr style="{}">'.format(style_str)
×
601
    else:
UNCOV
602
        row_str = '<tr style="'+style_str+'" id="'+str(idlabel)+'">'
×
603

UNCOV
604
    for elem in dictionary.values():
×
UNCOV
605
        chars = str(elem).split('/')
×
UNCOV
606
        if len(chars)==2: # m/n
×
UNCOV
607
            if chars[0]=='0' and chars[1]=='0':
×
UNCOV
608
                row_str += _table_element_id(elem, 'GOODNULL')
×
UNCOV
609
            elif chars[0]=='0' and chars[1]!='0':
×
UNCOV
610
                row_str += _table_element_id(elem, 'BAD')
×
UNCOV
611
            elif chars[0]!='0' and int(chars[0])<int(chars[1]):
×
UNCOV
612
                row_str += _table_element_id(elem, 'INCOMPLETE')
×
UNCOV
613
            elif chars[0]!='0' and int(chars[0])==int(chars[1]):
×
UNCOV
614
                row_str += _table_element_id(elem, 'GOOD')
×
615
            else:
UNCOV
616
                row_str += _table_element_id(elem, 'OVERFULL')
×
617

618
        else:
UNCOV
619
            row_str += _table_element(elem)
×
UNCOV
620
    row_str += '</tr>'#\n'
×
UNCOV
621
    return row_str
×
622

UNCOV
623
def _table_element(elem):
×
UNCOV
624
    return '<td>{}</td>'.format(elem)
×
625

UNCOV
626
def _table_element_style(elem,style):
×
UNCOV
627
    return f'<td style="{style}">{elem}</td>'
×
628

UNCOV
629
def _table_element_id(elem,id):
×
UNCOV
630
    return f'<td id="{id}">{elem}</td>'
×
631

UNCOV
632
def _hyperlink(rel_path,displayname):
×
UNCOV
633
    hlink = f'<a href="{rel_path}" target="_blank"' \
×
634
            + f' rel="noopener noreferrer">{displayname}</a>'
UNCOV
635
    return hlink
×
636

UNCOV
637
def _str_frac(numerator,denominator):
×
UNCOV
638
    frac = f'{numerator}/{denominator}'
×
639
    return frac
×
640

641
def _js_path(output_dir):
×
UNCOV
642
    return os.path.join(output_dir,'js','open_nightly_table.js')
×
643

UNCOV
644
def js_import_str(output_dir):  # Not used
×
645
    output_path = _js_path(output_dir)
×
646
    if not os.path.exists(os.path.join(output_dir,'js')):
×
647
        os.makedirs(os.path.join(output_dir,'js'))
×
UNCOV
648
    if not os.path.exists(output_path):
×
649
        _write_js_script(output_path)
×
UNCOV
650
    return f'<script type="text/javascript" src="{output_path}"></script>'
×
651

652
def _write_js_script(output_path):
×
653
    """
654
    Return the javascript script to be added to the html file
655
    """
UNCOV
656
    s="""
×
657
        var coll = document.getElementsByClassName('collapsible');
658
        var i;
659
        for (i = 0; i < coll.length; i++) {
660
            coll[i].nextElementSibling.style.maxHeight='0px';
661
            coll[i].addEventListener('click', function() {
662
                this.classList.toggle('active');
663
                var content = this.nextElementSibling;
664
                if (content.style.maxHeight){
665
                   content.style.maxHeight = null;
666
                } else {
667
                  content.style.maxHeight = '0px';
668
                        }
669
                });
670
         };
671
         var b1 = document.getElementById('b1');
672
         b1.addEventListener('click',function() {
673
             for (i = 0; i < coll.length; i++) {
674
                 coll[i].nextElementSibling.style.maxHeight=null;
675
                                               }});
676
         var b2 = document.getElementById('b2');
677
         b2.addEventListener('click',function() {
678
             for (i = 0; i < coll.length; i++) {
679
                 coll[i].nextElementSibling.style.maxHeight='0px'
680
                         }});
681
        """
UNCOV
682
    with open(output_path,'w') as outjs:
×
UNCOV
683
        outjs.write(s)
×
684

UNCOV
685
def js_str(): # Used
×
686
    """
687
        Return the javascript script to be added to the html file
688
    """
UNCOV
689
    s="""
×
690
        <script >
691
            var coll = document.getElementsByClassName('collapsible');
692
            var i;
693
            for (i = 0; i < coll.length; i++) {
694
                coll[i].nextElementSibling.style.maxHeight='0px';
695
                coll[i].addEventListener('click', function() {
696
                    this.classList.toggle('active');
697
                    var content = this.nextElementSibling;
698
                    if (content.style.maxHeight){
699
                       content.style.maxHeight = null;
700
                    } else {
701
                      content.style.maxHeight = '0px';
702
                            }
703
                    });
704
             };
705
             var b1 = document.getElementById('b1');
706
             b1.addEventListener('click',function() {
707
                 for (i = 0; i < coll.length; i++) {
708
                     coll[i].nextElementSibling.style.maxHeight=null;
709
                                                   }});
710
             var b2 = document.getElementById('b2');
711
             b2.addEventListener('click',function() {
712
                 for (i = 0; i < coll.length; i++) {
713
                     coll[i].nextElementSibling.style.maxHeight='0px'
714
                             }});
715
           function filterByStatus() {
716
                var input, filter, table, tr, td, i;
717
                input = document.getElementById("statuslist");
718
                filter = input.value.toUpperCase();
719
                tables = document.getElementsByClassName("nightTable")
720
                for (j = 0; j < tables.length; j++){
721
                 table = tables[j]
722
                 tr = table.getElementsByTagName("tr");
723
                 for (i = 0; i < tr.length; i++) {
724
                   td = tr[i].getElementsByTagName("td")[15];
725
                   console.log(td)
726
                   if (td) {
727
                       if (td.innerHTML.toUpperCase().indexOf(filter) > -1 || filter==='ALL') {
728
                           tr[i].style.display = "";
729
                       } else {
730
                          tr[i].style.display = "none";
731
                              }
732
                       }
733
                                                                            }
734
                             }}
735

736
           function filterByObstype() {
737
                var input, filter, table, tr, td, i;
738
                input = document.getElementById("obstypelist");
739
                filter = input.value.toUpperCase();
740
                tables = document.getElementsByClassName("nightTable")
741
                for (j = 0; j < tables.length; j++){
742
                 table = tables[j]
743
                 tr = table.getElementsByTagName("tr");
744
                 for (i = 0; i < tr.length; i++) {
745
                   td = tr[i].getElementsByTagName("td")[2];
746
                   if (td) {
747
                       if (td.innerHTML.toUpperCase().indexOf(filter) > -1 || filter==='ALL') {
748
                           tr[i].style.display = "";
749
                       } else {
750
                          tr[i].style.display = "none";
751
                              }
752
                       }
753
                                                                            }
754
                             }}
755

756
            function filterByExptime() {
757
                var input, filter, table, tr, td, i;
758
                input = document.getElementById("exptimelist");
759
                filter = input.value.toUpperCase();
760
                tables = document.getElementsByClassName("nightTable")
761
                for (j = 0; j < tables.length; j++){
762
                 table = tables[j]
763
                 tr = table.getElementsByTagName("tr");
764
                 for (i = 0; i < tr.length; i++) {
765
                   td = tr[i].getElementsByTagName("td")[3];
766
                   if (td) {
767
                       if (filter==='ALL') {
768
                           tr[i].style.display = "";
769
                       } else if (parseInt(td.innerHTML) <= parseInt(filter)){
770
                           tr[i].style.display = ""; }
771
                       else {
772
                          tr[i].style.display = "none";
773
                              }
774
                       }
775
                                                                            }
776
                             }}
777

778

779
       </script>
780
        """
UNCOV
781
    return s
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc