• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

OGGM / oggm / 24836012552

23 Apr 2026 12:46PM UTC coverage: 84.098% (-0.7%) from 84.838%
24836012552

Pull #1905

github

web-flow
Merge 84ca5886e into 09cfa446e
Pull Request #1905: Merge master into dev

4036 of 5696 branches covered (70.86%)

196 of 257 new or added lines in 10 files covered. (76.26%)

109 existing lines in 3 files now uncovered.

13634 of 16212 relevant lines covered (84.1%)

7.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.84
/oggm/cli/prepro_levels.py
1
"""Command line arguments to the oggm_prepro command
2

3
Type `$ oggm_prepro -h` for help
4

5
"""
6

7
# External modules
8
import os
2✔
9
import sys
2✔
10
import shutil
2✔
11
import argparse
2✔
12
import time
2✔
13
import logging
2✔
14
import json
2✔
15
import pandas as pd
2✔
16
import numpy as np
2✔
17
import geopandas as gpd
2✔
18

19
# Locals
20
import oggm.cfg as cfg
2✔
21
from oggm import utils, workflow, tasks, GlacierDirectory
2✔
22
from oggm.core import gis
2✔
23
from oggm.exceptions import InvalidParamsError, InvalidDEMError, InvalidWorkflowError
2✔
24

25
# Module logger
26
from oggm.utils import get_prepro_base_url, file_downloader
2✔
27

28
log = logging.getLogger(__name__)
2✔
29

30

31
@utils.entity_task(log)
2✔
32
def _rename_dem_folder(gdir, source=''):
2✔
33
    """Put the DEM files in a subfolder of the gdir.
34

35
    Parameters
36
    ----------
37
    gdir : GlacierDirectory
38
    source : str
39
        the DEM source
40
    """
41

42
    # open tif-file to check if it's worth it
43
    dem_f = gdir.get_filepath('dem')
×
44
    try:
×
45
        dem = gis.read_geotiff_dem(gdir)
×
46
    except IOError:
×
47
        # Error reading file, no problem - still, delete the file if needed
48
        if os.path.exists(dem_f):
×
49
            os.remove(dem_f)
×
50
        gdir.log('{},DEM SOURCE,{}'.format(gdir.rgi_id, source),
×
51
                 err=InvalidDEMError('File does not exist'))
52
        return
×
53

54
    # Check the DEM
55
    isfinite = np.isfinite(dem)
×
56
    if np.all(~isfinite) or (np.min(dem) == np.max(dem)):
×
57
        # Remove the file and return
58
        if os.path.exists(dem_f):
×
59
            os.remove(dem_f)
×
60
        gdir.log('{},DEM SOURCE,{}'.format(gdir.rgi_id, source),
×
61
                 err=InvalidDEMError('DEM does not contain more than one '
62
                                     'valid values.'))
63
        return
×
64

65
    # Create a source dir and move the files
66
    out = os.path.join(gdir.dir, source)
×
67
    utils.mkdir(out)
×
68
    for fname in ['dem', 'dem_source']:
×
69
        f = gdir.get_filepath(fname)
×
70
        os.rename(f, os.path.join(out, os.path.basename(f)))
×
71

72
    # log SUCCESS for this DEM source
73
    gdir.log('{},DEM SOURCE,{}'.format(gdir.rgi_id, source))
×
74

75

76
@utils.entity_task(log)
2✔
77
def _move_hypsometry_to_dem_folder(gdir, source=''):
2✔
78
    """Move the hypsometry file to the DEM source folder if it exists.
79

80
    Parameters
81
    ----------
82
    gdir : GlacierDirectory
83
    source : str
84
        the DEM source
85
    """
86

NEW
87
    hypso_f = gdir.get_filepath('hypsometry')
×
NEW
88
    if not os.path.exists(hypso_f):
×
NEW
89
        return
×
90

NEW
91
    out = os.path.join(gdir.dir, source)
×
NEW
92
    if not os.path.exists(out):
×
NEW
93
        raise InvalidWorkflowError('We should not be there')
×
NEW
94
    os.rename(hypso_f, os.path.join(out, os.path.basename(hypso_f)))
×
95

96

97
def run_prepro_levels(rgi_version=None, rgi_reg=None, border=None,
2✔
98
                      output_folder='', working_dir='', dem_source='',
99
                      is_test=False, test_ids=None, test_rgidf=None,
100
                      test_intersects_file=None, test_topofile=None,
101
                      disable_mp=False, params_file=None,
102
                      elev_bands=False, centerlines=False,
103
                      override_params=None, skip_inversion=False,
104
                      mb_calibration_strategy='informed_threestep',
105
                      select_source_from_dir=None, keep_dem_folders=False,
106
                      add_consensus_thickness=False, add_itslive_velocity=False,
107
                      add_millan_thickness=False, add_millan_velocity=False,
108
                      add_hugonnet_dhdt=False, add_bedmachine=False,
109
                      add_glathida=False, add_distributed_thickness=False,
110
                      add_export_thickness_geotiff=False, compute_hypsometry=False,
111
                      start_level=None, start_base_url=None, max_level=5,
112
                      logging_level='WORKFLOW',
113
                      dynamic_spinup=False, ref_mb_err_scaling_factor=0.2,
114
                      dynamic_spinup_start_year=1979,
115
                      continue_on_error=True, store_fl_diagnostics=False):
116
    """Generate the preprocessed OGGM glacier directories for this OGGM version
117

118
    Parameters
119
    ----------
120
    rgi_version : str
121
        the RGI version to use (defaults to cfg.PARAMS)
122
    rgi_reg : str
123
        the RGI region to process
124
    border : int
125
        the number of pixels at the maps border
126
    output_folder : str
127
        path to the output folder (where to put the preprocessed tar files)
128
    dem_source : str
129
        which DEM source to use: default, SOURCE_NAME, STANDARD or ALL
130
        ALL is to generate RGITOPO
131
        "STANDARD" is doina small RGITOPO using COPDEM + NASADEM
132
        default is the current default lookup tables found at
133
        https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.6/rgitopo/2025.4/
134
    working_dir : str
135
        path to the OGGM working directory
136
    params_file : str
137
        path to the OGGM parameter file (to override defaults)
138
    is_test : bool
139
        to test on a couple of glaciers only!
140
    test_ids : list
141
        if is_test: list of ids to process
142
    test_rgidf : shapefile
143
        for testing purposes only
144
    test_intersects_file : shapefile
145
        for testing purposes only
146
    test_topofile : str
147
        for testing purposes only
148
    test_crudir : str
149
        for testing purposes only
150
    disable_mp : bool
151
        disable multiprocessing
152
    elev_bands : bool
153
        compute all flowlines based on the Huss & Farinotti 2012 method.
154
    centerlines : bool
155
        compute all flowlines based on the OGGM centerline(s) method.
156
    mb_calibration_strategy : str
157
        how to calibrate the massbalance. Currently one of:
158
        - 'informed_threestep' (default)
159
        - 'melt_temp'
160
        - 'temp_melt'
161
        Add the `_regional` suffix to use regional values instead,
162
        for example `informed_threestep_regional`
163
    select_source_from_dir : str
164
        if starting from a level 1 "ALL" or "STANDARD" DEM sources directory,
165
        select the chosen DEM source here. If you set it to "BY_RES" here,
166
        COPDEM will be used and its resolution chosen based on the gdir's
167
        map resolution (COPDEM30 for dx < 60 m, COPDEM90 elsewhere).
168
    keep_dem_folders : bool
169
        if `select_source_from_dir` is used, wether to keep the original
170
        DEM folders in or not.
171
    add_consensus_thickness : bool
172
        adds (reprojects) the consensus estimates thickness to the glacier
173
        directories. With elev_bands=True, the data will also be binned.
174
    add_itslive_velocity : bool
175
        adds (reprojects) the ITS_LIVE velocity to the glacier
176
        directories. With elev_bands=True, the data will also be binned.
177
    add_millan_thickness : bool
178
        adds (reprojects) the millan thickness to the glacier
179
        directories. With elev_bands=True, the data will also be binned.
180
    add_millan_velocity : bool
181
        adds (reprojects) the millan velocity to the glacier
182
        directories. With elev_bands=True, the data will also be binned.
183
    add_hugonnet_dhdt : bool
184
        adds (reprojects) the hugonnet dhdt maps to the glacier
185
        directories. With elev_bands=True, the data will also be binned.
186
    add_bedmachine : bool
187
        adds (reprojects) the bedmachine ice thickness maps to the glacier
188
        directories. With elev_bands=True, the data will also be binned.
189
    add_glathida : bool
190
        adds (reprojects) the glathida thickness data to the glacier
191
        directories. Data points are stored as csv files.
192
    add_distributed_thickness : bool
193
        adds a thickness field to gridded_data using
194
        distribute_thickness_per_altitude.
195
    add_export_thickness_geotiff : bool
196
        exports the distributed thickness field to GeoTIFF files in a
197
        subfolder of the L3 summary directory.
198
    compute_hypsometry : bool
199
        Compute the hypsometry tables for all glaciers,
200
        added to the glacier directory and compiled in
201
        the summary folder.
202
    start_level : int
203
        the pre-processed level to start from (default is to start from
204
        scratch). If set, you'll need to indicate start_base_url as well.
205
    start_base_url : str
206
        the pre-processed base-url to fetch the data from.
207
    max_level : int
208
        the maximum pre-processing level before stopping
209
    skip_inversion : bool
210
         do not run the inversion (level 3 files). This is a temporary
211
         workaround for workflows that wont run that far into level 3.
212
    logging_level : str
213
        the logging level to use (DEBUG, INFO, WARNING, WORKFLOW)
214
    override_params : dict
215
        a dict of parameters to override.
216
    dynamic_spinup : str
217
        include a dynamic spinup matching 'area/dmdtda' OR 'volume/dmdtda' at
218
        the RGI-date
219
    ref_mb_err_scaling_factor : float
220
        scaling factor to reduce individual geodetic mass balance uncertainty
221
    dynamic_spinup_start_year : int
222
        if dynamic_spinup is set, define the starting year for the simulation.
223
        The default is 1979, unless the climate data starts later.
224
    continue_on_error : bool
225
        if True the workflow continues if a task raises an error. For operational
226
        runs it should be set to True (the default).
227
    store_fl_diagnostics : bool
228
        if True, also compute and store flowline diagnostics during preprocessing.
229
        This can increase data usage quite a bit.
230
    """
231

232
    # Input check
233
    if max_level not in [1, 2, 3, 4, 5]:
2!
234
        raise InvalidParamsError('max_level should be one of [1, 2, 3, 4, 5]')
×
235

236
    if start_level is not None:
2✔
237
        if start_level not in [0, 1, 2, 3, 4]:
2!
238
            raise InvalidParamsError('start_level should be one of [0, 1, 2, 3, 4]')
×
239
        if start_level > 0 and start_base_url is None:
2✔
240
            raise InvalidParamsError('With start_level, please also indicate '
2✔
241
                                     'start_base_url')
242
    else:
243
        start_level = 0
2✔
244

245
    if dynamic_spinup:
2✔
246
        if dynamic_spinup not in ['area/dmdtda', 'volume/dmdtda']:
2!
247
            raise InvalidParamsError(f"Dynamic spinup option '{dynamic_spinup}' "
×
248
                                     "not supported")
249

250
    # Time
251
    start = time.time()
2✔
252

253
    def _time_log():
2✔
254
        # Log util
255
        m, s = divmod(time.time() - start, 60)
2✔
256
        h, m = divmod(m, 60)
2✔
257
        log.workflow('OGGM prepro_levels is done! Time needed: '
2✔
258
                     '{:02d}:{:02d}:{:02d}'.format(int(h), int(m), int(s)))
259

260
    # Local paths
261
    if override_params is None:
2✔
262
        override_params = {}
2✔
263

264
    # Use multiprocessing?
265
    override_params['use_multiprocessing'] = not disable_mp
2✔
266

267
    # How many grid points around the glacier?
268
    # Make it large if you expect your glaciers to grow large
269
    override_params['border'] = border
2✔
270

271
    # Some arbitrary heuristics on the length of tidewater extension
272
    extension = int(utils.clip_min(border / 2, 30))
2✔
273
    override_params['calving_line_extension'] = extension
2✔
274

275
    # Set to True for operational runs
276
    override_params['continue_on_error'] = continue_on_error
2✔
277

278
    # For centerlines we have to change the default evolution model and bed
279
    if centerlines:
2✔
280
        override_params['downstream_line_shape'] = 'parabola'
2✔
281
        override_params['evolution_model'] = 'FluxBased'
2✔
282

283
    # Other things that make sense
284
    override_params['store_model_geometry'] = True
2✔
285
    override_params['store_fl_diagnostics'] = store_fl_diagnostics
2✔
286

287
    utils.mkdir(working_dir)
2✔
288
    override_params['working_dir'] = working_dir
2✔
289

290
    # Initialize OGGM and set up the run parameters
291
    cfg.initialize(file=params_file, params=override_params,
2✔
292
                   logging_level=logging_level)
293

294
    # Prepare the download of climate file to be shared across processes
295
    # TODO
296

297
    # Log the parameters
298
    msg = '# OGGM Run parameters:'
2✔
299
    for k, v in cfg.PARAMS.items():
2✔
300
        if type(v) in [pd.DataFrame, dict]:
2✔
301
            continue
2✔
302
        msg += '\n    {}: {}'.format(k, v)
2✔
303
    log.workflow(msg)
2✔
304

305
    if rgi_version is None:
2!
306
        rgi_version = cfg.PARAMS['rgi_version']
×
307
    output_base_dir = os.path.join(output_folder,
2✔
308
                                   'RGI{}'.format(rgi_version),
309
                                   'b_{:03d}'.format(border))
310

311
    # Add a package version file
312
    utils.mkdir(output_base_dir)
2✔
313
    opath = os.path.join(output_base_dir, 'package_versions.txt')
2✔
314
    with open(opath, 'w') as vfile:
2✔
315
        vfile.write(utils.show_versions(logger=log))
2✔
316

317
    if test_rgidf is None:
2!
318

319
        # Get the RGI file
320
        rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,
×
321
                                                        version=rgi_version))
322
        # We use intersects
323
        if rgi_version != '70C':
×
324
            rgif = utils.get_rgi_intersects_region_file(rgi_reg,
×
325
                                                        version=rgi_version)
326
            cfg.set_intersects_db(rgif)
×
327

328
        if rgi_version == '62':
×
329
            # Some RGI input quality checks - this is based on visual checks
330
            # of large glaciers in the RGI
331
            ids_to_ice_cap = [
×
332
                'RGI60-05.10315',  # huge Greenland ice cap
333
                'RGI60-03.01466',  # strange thing next to Devon
334
                'RGI60-09.00918',  # Academy of sciences Ice cap
335
                'RGI60-09.00969',
336
                'RGI60-09.00958',
337
                'RGI60-09.00957',
338
            ]
NEW
339
            rgidf.loc[rgidf.RGIId.isin(ids_to_ice_cap), 'Form'] = 1
×
340

341
            # In AA almost all large ice bodies are actually ice caps
342
            if rgi_reg == '19':
×
343
                rgidf.loc[rgidf.Area > 100, 'Form'] = '1'
×
344

345
            # For greenland we omit connectivity level 2
346
            if rgi_reg == '05':
×
347
                rgidf = rgidf.loc[rgidf['Connect'] != 2]
×
348
    else:
349
        rgidf = test_rgidf
2✔
350
        cfg.set_intersects_db(test_intersects_file)
2✔
351

352
    if is_test:
2!
353
        if test_ids is not None:
2✔
354
            try:
2✔
355
                rgidf = rgidf.loc[rgidf.RGIId.isin(test_ids)]
2✔
356
            except AttributeError:
×
357
                # RGI7
358
                rgidf = rgidf.loc[rgidf.rgi_id.isin(test_ids)]
×
359
        else:
360
            rgidf = rgidf.sample(4)
2✔
361

362
    if len(rgidf) == 0:
2!
363
        raise InvalidParamsError('Zero glaciers selected!')
×
364

365
    log.workflow('Starting prepro run for RGI reg: {} '
2✔
366
                 'and border: {}'.format(rgi_reg, border))
367
    log.workflow('Number of glaciers: {}'.format(len(rgidf)))
2✔
368

369
    # Try to avoid concurrency
370
    if rgi_version == '70C':
2!
NEW
371
        fp = file_downloader('https://cluster.klima.uni-bremen.de/~oggm/'
×
372
                             'ref_mb_params/oggm_v1.6/inv_rgi7/'
373
                             'rgi7c_rgi_year_2025.1.csv')
NEW
374
        rgi_year_by_id = pd.read_csv(fp, index_col=0)['rgi_year'].astype(int).astype(str)
×
NEW
375
        rgidf['src_date'] = rgidf['rgi_id'].map(rgi_year_by_id) + '-01-01 00:00:00'
×
376

377
    # Add a new default source
378
    if not dem_source:
2✔
379
        fs_url = 'https://cluster.klima.uni-bremen.de/~oggm/gdirs/oggm_v1.6/rgitopo/2025.4/'
2✔
380
        if rgi_version == '62':
2!
381
            fs = utils.file_downloader(fs_url + 'chosen_dem_RGI62_20251029.csv')
×
382
            dfs = pd.read_csv(fs, index_col=0)
×
383
            rgidf['dem_source'] = dfs.loc[rgidf['RGIId'], 'dem_source'].values
×
384
        if rgi_version == '70G':
2!
385
            fs = utils.file_downloader(fs_url + 'chosen_dem_RGI70G_20251029.csv')
×
386
            dfs = pd.read_csv(fs, index_col=0)
×
387
            rgidf['dem_source'] = dfs.loc[rgidf['rgi_id'], 'dem_source'].values
×
388
        if rgi_version == '70C':
2!
389
            fs = utils.file_downloader(fs_url + 'chosen_dem_RGI70C_20251029.csv')
×
390
            dfs = pd.read_csv(fs, index_col=0)
×
391
            rgidf['dem_source'] = dfs.loc[rgidf['rgi_id'], 'dem_source'].values
×
392

393
    # L0 - go
394
    if start_level == 0:
2✔
395
        gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)
2✔
396

397
        # Glacier stats
398
        sum_dir = os.path.join(output_base_dir, 'L0', 'summary')
2✔
399
        utils.mkdir(sum_dir)
2✔
400
        opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
401
        utils.compile_glacier_statistics(gdirs, path=opath)
2✔
402

403
        # L0 OK - compress all in output directory
404
        log.workflow('L0 done. Writing to tar...')
2✔
405
        level_base_dir = os.path.join(output_base_dir, 'L0')
2✔
406
        workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
2✔
407
                                     base_dir=level_base_dir)
408
        utils.base_dir_to_tar(level_base_dir)
2✔
409
        if max_level == 0:
2!
410
            _time_log()
×
411
            return
×
412
    else:
413
        gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True,
2✔
414
                                                  from_prepro_level=start_level,
415
                                                  prepro_border=border,
416
                                                  prepro_rgi_version=rgi_version,
417
                                                  prepro_base_url=start_base_url
418
                                                  )
419

420
    # L1 - Add dem files
421
    if start_level == 0:
2✔
422
        if test_topofile:
2!
423
            cfg.PATHS['dem_file'] = test_topofile
2✔
424

425
        # Which DEM source?
426
        if dem_source.upper() in ['ALL', 'STANDARD']:
2✔
427
            # This is the complex one, just do the job and leave
428

429
            if dem_source.upper() == 'ALL':
2!
430
                sources = utils.DEM_SOURCES
2✔
431
            if dem_source.upper() == 'STANDARD':
2!
432
                sources = ['COPDEM30', 'COPDEM90', 'NASADEM']
×
433

434
            log.workflow('Running prepro on several sources')
2✔
435
            for i, s in enumerate(sources):
2✔
436
                rs = i == 0
2✔
437
                log.workflow('Running prepro on sources: {}'.format(s))
2✔
438
                gdirs = workflow.init_glacier_directories(rgidf, reset=rs,
2✔
439
                                                          force=rs)
440
                workflow.execute_entity_task(tasks.define_glacier_region, gdirs,
2✔
441
                                             source=s)
442
                workflow.execute_entity_task(_rename_dem_folder, gdirs, source=s)
2✔
443

444
            # make a GeoTiff mask of the glacier, choose any source
445
            workflow.execute_entity_task(gis.rasterio_glacier_mask,
2✔
446
                                         gdirs, source='ALL')
447

448
            # Glacier stats
449
            sum_dir = os.path.join(output_base_dir, 'L1', 'summary')
2✔
450
            utils.mkdir(sum_dir)
2✔
451
            opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
452
            utils.compile_glacier_statistics(gdirs, path=opath)
2✔
453

454
            # Add hypsometry files
455
            if compute_hypsometry:
2!
NEW
456
                for dem_source in utils.DEM_SOURCES:
×
NEW
457
                    from oggm.shop.rgitopo import select_dem_from_dir
×
NEW
458
                    workflow.execute_entity_task(select_dem_from_dir, gdirs,
×
459
                                                 dem_source=dem_source,
460
                                                 keep_dem_folders=True)
NEW
461
                    workflow.execute_entity_task(tasks.rasterio_glacier_mask, gdirs,
×
462
                                                 no_nunataks=True,
463
                                                 overwrite=False)
NEW
464
                    workflow.execute_entity_task(tasks.rasterio_glacier_exterior_mask,
×
465
                                                 gdirs,
466
                                                 overwrite=False)
NEW
467
                    workflow.execute_entity_task(tasks.compute_hypsometry_attributes, gdirs)
×
NEW
468
                    opath = os.path.join(sum_dir, f'hypsometry_{rgi_reg}_{dem_source}.csv')
×
NEW
469
                    utils.compile_glacier_hypsometry(gdirs, path=opath,
×
470
                                                     add_column=('dem_source', dem_source))
NEW
471
                    workflow.execute_entity_task(_move_hypsometry_to_dem_folder,
×
472
                                                 gdirs, source=dem_source)
473

474
            # L1 OK - compress all in output directory
475
            log.workflow('L1 done. Writing to tar...')
2✔
476
            level_base_dir = os.path.join(output_base_dir, 'L1')
2✔
477
            workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
2✔
478
                                         base_dir=level_base_dir)
479
            utils.base_dir_to_tar(level_base_dir)
2✔
480

481
            _time_log()
2✔
482
            return
2✔
483

484
        # Force a given source
485
        source = dem_source.upper() if dem_source else None
2✔
486

487
        # L1 - go
488
        workflow.execute_entity_task(tasks.define_glacier_region, gdirs,
2✔
489
                                     source=source)
490

491
        # Summaries
492
        sum_dir = os.path.join(output_base_dir, 'L1', 'summary')
2✔
493
        utils.mkdir(sum_dir)
2✔
494

495
        # Add hypsometry files
496
        if compute_hypsometry:
2!
NEW
497
            workflow.execute_entity_task(tasks.rasterio_glacier_mask, gdirs)
×
NEW
498
            workflow.execute_entity_task(tasks.rasterio_glacier_mask, gdirs,
×
499
                                         no_nunataks=True)
NEW
500
            workflow.execute_entity_task(tasks.rasterio_glacier_exterior_mask, gdirs)
×
NEW
501
            workflow.execute_entity_task(tasks.compute_hypsometry_attributes, gdirs)
×
NEW
502
            opath = os.path.join(sum_dir, f'hypsometry_{rgi_reg}.csv')
×
NEW
503
            utils.compile_glacier_hypsometry(gdirs, path=opath)
×
504

505
        # Glacier stats
506
        opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
507
        utils.compile_glacier_statistics(gdirs, path=opath)
2✔
508

509
        # L1 OK - compress all in output directory
510
        log.workflow('L1 done. Writing to tar...')
2✔
511
        level_base_dir = os.path.join(output_base_dir, 'L1')
2✔
512
        workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
2✔
513
                                     base_dir=level_base_dir)
514
        utils.base_dir_to_tar(level_base_dir)
2✔
515
        if max_level == 1:
2!
516
            _time_log()
×
517
            return
×
518

519
    # L2 - Tasks
520
    if start_level <= 1:
2!
521
        # Check which glaciers will be processed as what
522
        if elev_bands:
2✔
523
            gdirs_band = gdirs
2✔
524
            gdirs_cent = []
2✔
525
        elif centerlines:
2!
526
            gdirs_band = []
2✔
527
            gdirs_cent = gdirs
2✔
528
        else:
529
            raise InvalidParamsError('Need to specify if `elev_bands` or '
×
530
                                     '`centerlines` type.')
531

532
        log.workflow('Start flowline processing with: '
2✔
533
                     'N centerline type: {}, '
534
                     'N elev bands type: {}.'
535
                     ''.format(len(gdirs_cent), len(gdirs_band)))
536

537
        # If we are coming from a multi-dem setup, let's select it from there
538
        if select_source_from_dir is not None:
2!
539
            from oggm.shop.rgitopo import select_dem_from_dir
×
540
            workflow.execute_entity_task(select_dem_from_dir, gdirs_band,
×
541
                                         dem_source=select_source_from_dir,
542
                                         keep_dem_folders=keep_dem_folders)
543
            workflow.execute_entity_task(select_dem_from_dir, gdirs_cent,
×
544
                                         dem_source=select_source_from_dir,
545
                                         keep_dem_folders=keep_dem_folders)
546

547
        # HH2015 method
548
        workflow.execute_entity_task(tasks.simple_glacier_masks, gdirs_band)
2✔
549

550
        # Centerlines OGGM
551
        workflow.execute_entity_task(tasks.glacier_masks, gdirs_cent)
2✔
552

553
        bin_variables = []
2✔
554
        if add_consensus_thickness:
2!
555
            from oggm.shop.bedtopo import add_consensus_thickness
×
556
            workflow.execute_entity_task(add_consensus_thickness, gdirs)
×
557
            bin_variables.append('consensus_ice_thickness')
×
558
        if add_itslive_velocity:
2!
559
            from oggm.shop.its_live import itslive_velocity_to_gdir
×
560
            workflow.execute_entity_task(itslive_velocity_to_gdir, gdirs)
×
561
            bin_variables.append('itslive_v')
×
562
        if add_millan_thickness:
2!
563
            from oggm.shop.millan22 import millan_thickness_to_gdir
×
564
            workflow.execute_entity_task(millan_thickness_to_gdir, gdirs)
×
565
            bin_variables.append('millan_ice_thickness')
×
566
        if add_millan_velocity:
2!
567
            from oggm.shop.millan22 import millan_velocity_to_gdir
×
568
            workflow.execute_entity_task(millan_velocity_to_gdir, gdirs)
×
569
            bin_variables.append('millan_v')
×
570
        if add_hugonnet_dhdt:
2!
571
            from oggm.shop.hugonnet_maps import hugonnet_to_gdir
×
572
            workflow.execute_entity_task(hugonnet_to_gdir, gdirs)
×
573
            bin_variables.append('hugonnet_dhdt')
×
574
        if add_bedmachine:
2!
575
            from oggm.shop.bedmachine import bedmachine_to_gdir
×
576
            workflow.execute_entity_task(bedmachine_to_gdir, gdirs)
×
577
            bin_variables.append('bedmachine_ice_thickness')
×
578
        if add_glathida:
2!
579
            from oggm.shop.glathida import glathida_to_gdir
×
580
            workflow.execute_entity_task(glathida_to_gdir, gdirs)
×
581
        if rgi_version == '70C':
2!
582
            # Some additional data for the 70C glaciers
583
            workflow.execute_entity_task(tasks.rgi7g_to_complex, gdirs)
×
584

585
        if bin_variables and gdirs_band:
2!
586
            workflow.execute_entity_task(tasks.elevation_band_flowline,
×
587
                                         gdirs_band,
588
                                         bin_variables=bin_variables)
589
            workflow.execute_entity_task(tasks.fixed_dx_elevation_band_flowline,
×
590
                                         gdirs_band,
591
                                         bin_variables=bin_variables)
592
        else:
593
            # HH2015 method without it
594
            task_list = [
2✔
595
                tasks.elevation_band_flowline,
596
                tasks.fixed_dx_elevation_band_flowline,
597
            ]
598
            for task in task_list:
2✔
599
                workflow.execute_entity_task(task, gdirs_band)
2✔
600

601
        # Centerlines OGGM
602
        task_list = [
2✔
603
            tasks.compute_centerlines,
604
            tasks.initialize_flowlines,
605
            tasks.catchment_area,
606
            tasks.catchment_intersections,
607
            tasks.catchment_width_geom,
608
            tasks.catchment_width_correction,
609
        ]
610
        for task in task_list:
2✔
611
            workflow.execute_entity_task(task, gdirs_cent)
2✔
612

613
        # Same for all glaciers
614
        if border >= 20:
2!
615
            task_list = [
2✔
616
                tasks.compute_downstream_line,
617
                tasks.compute_downstream_bedshape,
618
            ]
619
            for task in task_list:
2✔
620
                workflow.execute_entity_task(task, gdirs)
2✔
621
        else:
622
            log.workflow('L2: for map border values < 20, wont compute '
×
623
                         'downstream lines.')
624

625
        # Glacier stats
626
        sum_dir = os.path.join(output_base_dir, 'L2', 'summary')
2✔
627
        utils.mkdir(sum_dir)
2✔
628
        opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
629
        utils.compile_glacier_statistics(gdirs, path=opath)
2✔
630

631
        if add_itslive_velocity:
2!
632
            from oggm.shop.its_live import compile_itslive_statistics
×
633
            opath = os.path.join(sum_dir, 'itslive_statistics_{}.csv'.format(rgi_reg))
×
634
            compile_itslive_statistics(gdirs, path=opath)
×
635
        if add_millan_thickness or add_millan_velocity:
2!
636
            from oggm.shop.millan22 import compile_millan_statistics
×
637
            opath = os.path.join(sum_dir, 'millan_statistics_{}.csv'.format(rgi_reg))
×
638
            compile_millan_statistics(gdirs, path=opath)
×
639
        if add_consensus_thickness:
2!
640
            from oggm.shop.bedtopo import compile_consensus_statistics
×
641
            opath = os.path.join(sum_dir, 'consensus_statistics_{}.csv'.format(rgi_reg))
×
642
            compile_consensus_statistics(gdirs, path=opath)
×
643
        if add_hugonnet_dhdt:
2!
644
            from oggm.shop.hugonnet_maps import compile_hugonnet_statistics
×
645
            opath = os.path.join(sum_dir, 'hugonnet_statistics_{}.csv'.format(rgi_reg))
×
646
            compile_hugonnet_statistics(gdirs, path=opath)
×
647
        if add_bedmachine:
2!
648
            from oggm.shop.bedmachine import compile_bedmachine_statistics
×
649
            opath = os.path.join(sum_dir, 'bedmachine_statistics_{}.csv'.format(rgi_reg))
×
650
            compile_bedmachine_statistics(gdirs, path=opath)
×
651
        if add_glathida:
2!
652
            from oggm.shop.glathida import compile_glathida_statistics
×
653
            opath = os.path.join(sum_dir, 'glathida_statistics_{}.csv'.format(rgi_reg))
×
654
            compile_glathida_statistics(gdirs, path=opath)
×
655

656
        # And for level 2: shapes
657
        if len(gdirs_cent) > 0:
2✔
658
            opath = os.path.join(sum_dir, f'centerlines_{rgi_reg}.shp')
2✔
659
            utils.write_centerlines_to_shape(gdirs_cent, to_tar=True,
2✔
660
                                             path=opath)
661
            opath = os.path.join(sum_dir, f'centerlines_smoothed_{rgi_reg}.shp')
2✔
662
            utils.write_centerlines_to_shape(gdirs_cent, to_tar=True,
2✔
663
                                             ensure_exterior_match=True,
664
                                             simplify_line_before=0.75,
665
                                             corner_cutting=3,
666
                                             path=opath)
667
            opath = os.path.join(sum_dir, f'flowlines_{rgi_reg}.shp')
2✔
668
            utils.write_centerlines_to_shape(gdirs_cent, to_tar=True,
2✔
669
                                             flowlines_output=True,
670
                                             path=opath)
671
            opath = os.path.join(sum_dir, f'geom_widths_{rgi_reg}.shp')
2✔
672
            utils.write_centerlines_to_shape(gdirs_cent, to_tar=True,
2✔
673
                                             geometrical_widths_output=True,
674
                                             path=opath)
675
            opath = os.path.join(sum_dir, f'widths_{rgi_reg}.shp')
2✔
676
            utils.write_centerlines_to_shape(gdirs_cent, to_tar=True,
2✔
677
                                             corrected_widths_output=True,
678
                                             path=opath)
679

680
        # L2 OK - compress all in output directory
681
        log.workflow('L2 done. Writing to tar...')
2✔
682
        level_base_dir = os.path.join(output_base_dir, 'L2')
2✔
683
        workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
2✔
684
                                     base_dir=level_base_dir)
685
        utils.base_dir_to_tar(level_base_dir)
2✔
686
        if max_level == 2:
2!
687
            _time_log()
×
688
            return
×
689

690
    # L3 - Tasks
691
    if start_level <= 2:
2!
692
        sum_dir = os.path.join(output_base_dir, 'L3', 'summary')
2✔
693
        utils.mkdir(sum_dir)
2✔
694

695
        # Climate
696
        workflow.execute_entity_task(tasks.process_climate_data, gdirs)
2✔
697

698
        # Small optim to avoid concurrency
699
        utils.get_geodetic_mb_dataframe()
2✔
700
        utils.get_temp_bias_dataframe(dataset='w5e5')
2✔
701
        utils.get_temp_bias_dataframe(dataset='w5e5', regional=True)
2✔
702
        utils.get_temp_bias_dataframe(dataset='w5e5', rgi_version='70G', regional=True)
2✔
703
        utils.get_temp_bias_dataframe(dataset='w5e5', rgi_version='70C', regional=True)
2✔
704
        utils.get_temp_bias_dataframe(dataset='era5')
2✔
705

706
        use_regional_avg = False
2✔
707
        if '_regional' in mb_calibration_strategy:
2!
708
            use_regional_avg = True
×
709
            mb_calibration_strategy = mb_calibration_strategy.replace('_regional', '')
×
710

711
        if mb_calibration_strategy == 'informed_threestep':
2✔
712
            workflow.execute_entity_task(tasks.mb_calibration_from_hugonnet_mb,
2✔
713
                                         gdirs,
714
                                         informed_threestep=True,
715
                                         use_regional_avg=use_regional_avg)
716
        elif mb_calibration_strategy == 'melt_temp':
2!
717
            workflow.execute_entity_task(tasks.mb_calibration_from_hugonnet_mb,
2✔
718
                                         gdirs,
719
                                         calibrate_param1='melt_f',
720
                                         calibrate_param2='temp_bias',
721
                                         use_regional_avg=use_regional_avg)
722
        elif mb_calibration_strategy == 'temp_melt':
×
723
            workflow.execute_entity_task(tasks.mb_calibration_from_hugonnet_mb,
×
724
                                         gdirs,
725
                                         calibrate_param1='temp_bias',
726
                                         calibrate_param2='melt_f',
727
                                         use_regional_avg=use_regional_avg)
728
        else:
729
            raise InvalidParamsError('mb_calibration_strategy not understood: '
×
730
                                     f'{mb_calibration_strategy}')
731

732
        if not skip_inversion:
2!
733
            workflow.execute_entity_task(tasks.apparent_mb_from_any_mb, gdirs)
2✔
734

735
            # Inversion: we match the consensus
736
            filter = border >= 20
2✔
737

738
            if rgi_version == '70G':
2!
739
                purl = ('https://cluster.klima.uni-bremen.de/~oggm/ref_mb_params/'
×
740
                        'oggm_v1.6/inv_rgi7/rgi6_regional_inv_params_2025.1.csv')
741
                params_df = pd.read_csv(utils.file_downloader(purl), index_col=0)
×
742
                workflow.invert_from_params(gdirs,
×
743
                                            params_df=params_df,
744
                                            filter_inversion_output=filter)
745
            elif rgi_version == '70C':
2!
746
                purl = ('https://cluster.klima.uni-bremen.de/~oggm/ref_mb_params/'
×
747
                        'oggm_v1.6/inv_rgi7/rgi7c_glacier_inv_ref_2025.1.csv')
748
                ref_vol_df = pd.read_csv(utils.file_downloader(purl), index_col=0)
×
749
                # Small optim
750
                ref_vol_df = ref_vol_df.loc[ref_vol_df.rgi_region == int(rgi_reg)]
×
751
                ref_vol_df = ref_vol_df['inv_volume_km3'] * 1e9
×
752
                workflow.execute_entity_task(workflow.calibrate_inversion_from_volume,
×
753
                                             gdirs,
754
                                             vol_ref_m3=ref_vol_df,
755
                                             apply_fs_on_mismatch=True,
756
                                             error_on_mismatch=False,
757
                                             filter_inversion_output=filter)
758
            else:
759
                workflow.calibrate_inversion_from_consensus(gdirs,
2✔
760
                                                            apply_fs_on_mismatch=True,
761
                                                            error_on_mismatch=False,
762
                                                            filter_inversion_output=filter)
763

764
            # Distribute thickness per altitude for gridded data
765
            if add_distributed_thickness:
2✔
766
                workflow.execute_entity_task(tasks.distribute_thickness_per_altitude, gdirs)
2✔
767

768
            # We get ready for modelling
769
            if border >= 20:
2!
770
                workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs)
2✔
771
            else:
772
                log.workflow('L3: for map border values < 20, wont initialize glaciers '
×
773
                             'for the run.')
774
        # Glacier stats
775
        opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
776
        utils.compile_glacier_statistics(gdirs, path=opath)
2✔
777

778
        # Export thickness to GeoTIFF if requested
779
        if add_export_thickness_geotiff and add_distributed_thickness:
2✔
780
            thickness_dir = os.path.join(sum_dir, 'distributed_thickness')
2✔
781
            utils.mkdir(thickness_dir)
2✔
782
            workflow.execute_entity_task(tasks.gridded_data_var_to_geotiff, gdirs,
2✔
783
                                         varname='distributed_thickness',
784
                                         output_folder=thickness_dir)
785
        opath = os.path.join(sum_dir, 'climate_statistics_{}.csv'.format(rgi_reg))
2✔
786
        utils.compile_climate_statistics(gdirs, path=opath)
2✔
787
        opath = os.path.join(sum_dir, 'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
2✔
788
        utils.compile_fixed_geometry_mass_balance(gdirs, path=opath)
2✔
789

790
        # L3 OK - compress all in output directory
791
        log.workflow('L3 done. Writing to tar...')
2✔
792
        level_base_dir = os.path.join(output_base_dir, 'L3')
2✔
793
        workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
2✔
794
                                     base_dir=level_base_dir)
795
        utils.base_dir_to_tar(level_base_dir)
2✔
796
        if max_level == 3:
2✔
797
            _time_log()
2✔
798
            return
2✔
799
        if border < 20:
2!
800
            log.workflow('L3: for map border values < 20, wont compute L4 and L5.')
×
801
            _time_log()
×
802
            return
×
803

804
        # is needed to copy some files for L4 and L5
805
        sum_dir_L3 = sum_dir
2✔
806

807
    # L4 - Tasks (add historical runs (old default) and dynamic spinup runs)
808
    if start_level <= 3:
2!
809
        sum_dir = os.path.join(output_base_dir, 'L4', 'summary')
2✔
810
        utils.mkdir(sum_dir)
2✔
811

812
        # Copy L3 files for consistency
813
        for bn in ['glacier_statistics', 'climate_statistics',
2✔
814
                   'fixed_geometry_mass_balance']:
815
            if start_level <= 2:
2!
816
                ipath = os.path.join(sum_dir_L3, bn + '_{}.csv'.format(rgi_reg))
2✔
817
            else:
818
                ipath = file_downloader(os.path.join(
×
819
                    get_prepro_base_url(base_url=start_base_url,
820
                                        rgi_version=rgi_version, border=border,
821
                                        prepro_level=start_level), 'summary',
822
                    bn + '_{}.csv'.format(rgi_reg)))
823

824
            opath = os.path.join(sum_dir, bn + '_{}.csv'.format(rgi_reg))
2✔
825
            shutil.copyfile(ipath, opath)
2✔
826

827
        # Get end date. The first gdir might have blown up, try some others
828
        i = 0
2✔
829
        while True:
2✔
830
            if i >= len(gdirs):
2!
831
                raise RuntimeError('Found no valid glaciers!')
×
832
            try:
2✔
833
                y0 = gdirs[i].get_climate_info()['baseline_yr_0']
2✔
834
                # One adds 1 because the run ends at the end of the year
835
                ye = gdirs[i].get_climate_info()['baseline_yr_1'] + 1
2✔
836
                break
2✔
837
            except BaseException:
×
838
                i += 1
×
839

840
        # conduct historical run before dynamic melt_f calibration
841
        # (for comparison to old default behavior)
842
        workflow.execute_entity_task(tasks.run_from_climate_data, gdirs,
2✔
843
                                     min_ys=y0, ye=ye,
844
                                     output_filesuffix='_historical')
845
        # Now compile the output
846
        opath = os.path.join(sum_dir, f'historical_run_output_{rgi_reg}.nc')
2✔
847
        utils.compile_run_output(gdirs, path=opath, input_filesuffix='_historical')
2✔
848

849
        # conduct dynamic spinup if wanted
850
        if dynamic_spinup:
2✔
851
            if y0 > dynamic_spinup_start_year:
2!
852
                dynamic_spinup_start_year = y0
×
853

854
            minimise_for = dynamic_spinup.split('/')[0]
2✔
855

856
            melt_f_max = cfg.PARAMS['melt_f_max']
2✔
857
            workflow.execute_entity_task(
2✔
858
                tasks.run_dynamic_melt_f_calibration, gdirs,
859
                ref_mb_err_scaling_factor=ref_mb_err_scaling_factor,
860
                ys=dynamic_spinup_start_year, ye=ye,
861
                melt_f_max=melt_f_max,
862
                kwargs_run_function={'minimise_for': minimise_for},
863
                ignore_errors=True,
864
                kwargs_fallback_function={'minimise_for': minimise_for},
865
                output_filesuffix='_spinup_historical',)
866
            # Now compile the output
867
            opath = os.path.join(sum_dir, f'spinup_historical_run_output_{rgi_reg}.nc')
2✔
868
            utils.compile_run_output(gdirs, path=opath,
2✔
869
                                     input_filesuffix='_spinup_historical')
870

871
        # Glacier statistics we recompute here for error analysis
872
        opath = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
873
        utils.compile_glacier_statistics(gdirs, path=opath)
2✔
874

875
        # Add the extended files
876
        pf = os.path.join(sum_dir, 'historical_run_output_{}.nc'.format(rgi_reg))
2✔
877
        # We have copied the files above
878
        mf = os.path.join(sum_dir, 'fixed_geometry_mass_balance_{}.csv'.format(rgi_reg))
2✔
879
        sf = os.path.join(sum_dir, 'glacier_statistics_{}.csv'.format(rgi_reg))
2✔
880
        opath = os.path.join(sum_dir, 'historical_run_output_extended_{}.nc'.format(rgi_reg))
2✔
881
        utils.extend_past_climate_run(past_run_file=pf,
2✔
882
                                      fixed_geometry_mb_file=mf,
883
                                      glacier_statistics_file=sf,
884
                                      path=opath)
885

886
        # L4 OK - compress all in output directory
887
        log.workflow('L4 done. Writing to tar...')
2✔
888
        level_base_dir = os.path.join(output_base_dir, 'L4')
2✔
889
        workflow.execute_entity_task(utils.gdir_to_tar, gdirs, delete=False,
2✔
890
                                     base_dir=level_base_dir)
891
        utils.base_dir_to_tar(level_base_dir)
2✔
892

893
        sum_dir_L4 = sum_dir
2✔
894

895
        if max_level == 4:
2!
896
            _time_log()
×
897
            return
×
898

899
    # L5 - No tasks: make the dirs small
900
    sum_dir = os.path.join(output_base_dir, 'L5', 'summary')
2✔
901
    utils.mkdir(sum_dir)
2✔
902

903
    # Copy L4 files for consistency
904
    files_to_copy = ['glacier_statistics', 'climate_statistics',
2✔
905
                     'fixed_geometry_mass_balance', 'historical_run_output',
906
                     'historical_run_output_extended']
907
    files_suffixes = ['csv', 'csv', 'csv', 'nc', 'nc']
2✔
908
    if dynamic_spinup:
2✔
909
        files_to_copy.append('spinup_historical_run_output')
2✔
910
        files_suffixes.append('nc')
2✔
911
    for bn, suffix in zip(files_to_copy, files_suffixes):
2✔
912
        if start_level <= 3:
2!
913
            ipath = os.path.join(sum_dir_L4, bn + f'_{rgi_reg}.{suffix}')
2✔
914
        else:
915
            ipath = file_downloader(os.path.join(
×
916
                get_prepro_base_url(base_url=start_base_url,
917
                                    rgi_version=rgi_version, border=border,
918
                                    prepro_level=start_level), 'summary',
919
                bn + f'_{rgi_reg}.{suffix}'))
920
        opath = os.path.join(sum_dir, bn + f'_{rgi_reg}.{suffix}')
2✔
921
        shutil.copyfile(ipath, opath)
2✔
922

923
    # Copy mini data to new dir
924
    mini_base_dir = os.path.join(working_dir, 'mini_perglacier',
2✔
925
                                 'RGI{}'.format(rgi_version),
926
                                 'b_{:03d}'.format(border))
927
    mini_gdirs = workflow.execute_entity_task(tasks.copy_to_basedir, gdirs,
2✔
928
                                              base_dir=mini_base_dir,
929
                                              setup='run/spinup')
930

931
    # L5 OK - compress all in output directory
932
    log.workflow('L5 done. Writing to tar...')
2✔
933
    level_base_dir = os.path.join(output_base_dir, 'L5')
2✔
934
    workflow.execute_entity_task(utils.gdir_to_tar, mini_gdirs, delete=False,
2✔
935
                                 base_dir=level_base_dir)
936
    utils.base_dir_to_tar(level_base_dir)
2✔
937

938
    _time_log()
2✔
939

940

941
def parse_args(args):
2✔
942
    """Check input arguments and env variables"""
943

944
    # CLI args
945
    description = ('Generate the preprocessed OGGM glacier directories for '
2✔
946
                   'this OGGM version.')
947
    parser = argparse.ArgumentParser(description=description)
2✔
948
    parser.add_argument('--map-border', type=int,
2✔
949
                        help='the size of the map border. Is required if '
950
                             '$OGGM_MAP_BORDER is not set.')
951
    parser.add_argument('--rgi-reg', type=str,
2✔
952
                        help='the rgi region to process. Is required if '
953
                             '$OGGM_RGI_REG is not set.')
954
    parser.add_argument('--rgi-version', type=str,
2✔
955
                        help='the RGI version to use. Defaults to the OGGM '
956
                             'default.')
957
    parser.add_argument('--start-level', type=int, default=0,
2✔
958
                        help='the pre-processed level to start from (default '
959
                             'is to start from 0). If set, you will need to '
960
                             'indicate --start-base-url as well.')
961
    parser.add_argument('--start-base-url', type=str,
2✔
962
                        help='the pre-processed base-url to fetch the data '
963
                             'from when starting from level > 0.')
964
    parser.add_argument('--max-level', type=int, default=5,
2✔
965
                        help='the maximum level you want to run the '
966
                             'pre-processing for (1, 2, 3, 4 or 5).')
967
    parser.add_argument('--working-dir', type=str,
2✔
968
                        help='path to the directory where to write the '
969
                             'output. Defaults to current directory or '
970
                             '$OGGM_WORKDIR.')
971
    parser.add_argument('--params-file', type=str,
2✔
972
                        help='path to the OGGM parameter file to use in place '
973
                             'of the default one.')
974
    parser.add_argument('--output', type=str,
2✔
975
                        help='path to the directory where to write the '
976
                             'output. Defaults to current directory or '
977
                             '$OGGM_OUTDIR.')
978
    parser.add_argument('--logging-level', type=str, default='WORKFLOW',
2✔
979
                        help='the logging level to use (DEBUG, INFO, WARNING, '
980
                             'WORKFLOW).')
981
    parser.add_argument('--elev-bands', nargs='?', const=True, default=False,
2✔
982
                        help='compute the flowlines based on the Huss & Farinotti '
983
                             '2012 method.')
984
    parser.add_argument('--centerlines', nargs='?', const=True, default=False,
2✔
985
                        help='compute the flowlines based on the OGGM '
986
                             'centerline(s) method.')
987
    parser.add_argument('--skip-inversion', nargs='?', const=True, default=False,
2✔
988
                        help='do not run the inversion (level 3 files). '
989
                             'this is a temporary workaround for workflows '
990
                             'that wont run that far into level 3.')
991
    parser.add_argument('--mb-calibration-strategy', type=str,
2✔
992
                        default='informed_threestep',
993
                        help='how to calibrate the massbalance. Currently one of '
994
                             'informed_threestep (default) , melt_temp '
995
                             'or temp_melt. Add the _regional suffix to '
996
                             'use regional values instead, for example '
997
                             'informed_threestep_regional')
998
    parser.add_argument('--dem-source', type=str, default='',
2✔
999
                        help='which DEM source to use. Possible options are '
1000
                             'the name of a specific DEM (e.g. RAMP, SRTM...) '
1001
                             'or ALL, in which case all available DEMs will '
1002
                             'be processed and adjoined with a suffix at the '
1003
                             'end of the file name. The ALL option is only '
1004
                             'compatible with level 1 folders, after which '
1005
                             'the processing will stop. The default is to use '
1006
                             'the default OGGM DEM.')
1007
    parser.add_argument('--select-source-from-dir', type=str,
2✔
1008
                        default=None,
1009
                        help='if starting from a level 1 "ALL" or "STANDARD" DEM '
1010
                        'sources directory, select the chosen DEM source here. '
1011
                        'If you set it to "BY_RES" here, COPDEM will be used and '
1012
                        'its resolution chosen based on the gdirs map resolution '
1013
                        '(COPDEM30 for dx < 60 m, COPDEM90 elsewhere).')
1014
    parser.add_argument('--keep-dem-folders', nargs='?', const=True, default=False,
2✔
1015
                        help='if `select_source_from_dir` is used, wether to keep '
1016
                        'the original DEM folders in or not.')
1017
    parser.add_argument('--add-consensus-thickness', nargs='?', const=True, default=False,
2✔
1018
                        help='adds (reprojects) the consensus thickness '
1019
                             'estimates to the glacier directories. '
1020
                             'With --elev-bands, the data will also be '
1021
                             'binned.')
1022
    parser.add_argument('--add-itslive-velocity', nargs='?', const=True, default=False,
2✔
1023
                        help='adds (reprojects) the ITS_LIVE velocity '
1024
                             'estimates to the glacier directories. '
1025
                             'With --elev-bands, the data will also be '
1026
                             'binned.')
1027
    parser.add_argument('--add-millan-thickness', nargs='?', const=True, default=False,
2✔
1028
                        help='adds (reprojects) the millan thickness '
1029
                             'estimates to the glacier directories. '
1030
                             'With --elev-bands, the data will also be '
1031
                             'binned.')
1032
    parser.add_argument('--add-millan-velocity', nargs='?', const=True, default=False,
2✔
1033
                        help='adds (reprojects) the millan velocity '
1034
                             'estimates to the glacier directories. '
1035
                             'With --elev-bands, the data will also be '
1036
                             'binned.')
1037
    parser.add_argument('--add-hugonnet-dhdt', nargs='?', const=True, default=False,
2✔
1038
                        help='adds (reprojects) the hugonnet dhdt '
1039
                             'maps to the glacier directories. '
1040
                             'With --elev-bands, the data will also be '
1041
                             'binned.')
1042
    parser.add_argument('--add-bedmachine', nargs='?', const=True, default=False,
2✔
1043
                        help='adds (reprojects) the Bedmachine ice thickness '
1044
                             'maps to the glacier directories. '
1045
                             'With --elev-bands, the data will also be '
1046
                             'binned.')
1047
    parser.add_argument('--add-glathida', nargs='?', const=True, default=False,
2✔
1048
                        help='adds (reprojects) the glathida point thickness '
1049
                             'observations to the glacier directories. '
1050
                             'The data points are stored as csv.')
1051
    parser.add_argument('--add-distributed-thickness', nargs='?', const=True, default=False,
2✔
1052
                        help='adds a thickness field to gridded_data using '
1053
                             'distribute_thickness_per_altitude.')
1054
    parser.add_argument('--add-export-thickness-geotiff', nargs='?', const=True, default=False,
2✔
1055
                        help='exports the distributed thickness field to '
1056
                             'GeoTIFF files in a subfolder of the L3 summary '
1057
                             'directory. Requires --add-distributed-thickness.')
1058
    parser.add_argument('--compute-hypsometry', nargs='?', const=True, default=False,
2✔
1059
                        help='Compute the hypsometry tables for all glaciers, '
1060
                             'added to the glacier directory and compiled in '
1061
                             'the summary folder')
1062
    parser.add_argument('--test', nargs='?', const=True, default=False,
2✔
1063
                        help='if you want to do a test on a couple of '
1064
                             'glaciers first.')
1065
    parser.add_argument('--test-ids', nargs='+',
2✔
1066
                        help='if --test, specify the RGI ids to run separated '
1067
                             'by a space (default: 4 randomly selected).')
1068
    parser.add_argument('--disable-mp', nargs='?', const=True, default=False,
2✔
1069
                        help='if you want to disable multiprocessing.')
1070
    parser.add_argument('--dynamic-spinup', type=str, default='',
2✔
1071
                        help="include a dynamic spinup for matching glacier area "
1072
                             "('area/dmdtda') OR volume ('volume/dmdtda') at "
1073
                             "the RGI-date, AND mass-change from Hugonnet "
1074
                             "in the period 2000-2020 (dynamic mu* "
1075
                             "calibration).")
1076
    parser.add_argument('--ref-mb-err-scaling-factor', type=float, default=0.2,
2✔
1077
                        help="scaling factor to account for correlated "
1078
                             "uncertainties of geodetic mass balance "
1079
                             "observations when looking at regional scale. "
1080
                             "Should be smaller or equal to 1.")
1081
    parser.add_argument('--dynamic-spinup-start-year', type=int, default=1979,
2✔
1082
                        help="if --dynamic-spinup is set, define the starting"
1083
                             "year for the simulation. The default is 1979, "
1084
                             "unless the climate data starts later.")
1085
    parser.add_argument('--store-fl-diagnostics', nargs='?', const=True, default=False,
2✔
1086
                        help="Also compute and store flowline diagnostics during "
1087
                             "preprocessing. This can increase data usage quite "
1088
                             "a bit.")
1089
    parser.add_argument('--override-params', type=json.loads, default=None)
2✔
1090

1091
    args = parser.parse_args(args)
2✔
1092

1093
    # Check input
1094
    rgi_reg = args.rgi_reg
2✔
1095
    if not rgi_reg:
2✔
1096
        rgi_reg = os.environ.get('OGGM_RGI_REG', None)
2✔
1097
        if rgi_reg is None:
2✔
1098
            raise InvalidParamsError('--rgi-reg is required!')
2✔
1099
    rgi_reg = '{:02}'.format(int(rgi_reg))
2✔
1100
    ok_regs = ['{:02}'.format(int(r)) for r in range(1, 20)]
2✔
1101
    if rgi_reg not in ok_regs:
2!
1102
        raise InvalidParamsError('--rgi-reg should range from 01 to 19!')
×
1103

1104
    rgi_version = args.rgi_version
2✔
1105

1106
    border = args.map_border
2✔
1107
    if not border:
2✔
1108
        border = os.environ.get('OGGM_MAP_BORDER', None)
2✔
1109
        if border is None:
2✔
1110
            raise InvalidParamsError('--map-border is required!')
2✔
1111

1112
    working_dir = args.working_dir
2✔
1113
    if not working_dir:
2✔
1114
        working_dir = os.environ.get('OGGM_WORKDIR', '')
2✔
1115

1116
    output_folder = args.output
2✔
1117
    if not output_folder:
2✔
1118
        output_folder = os.environ.get('OGGM_OUTDIR', '')
2✔
1119

1120
    border = int(border)
2✔
1121
    output_folder = os.path.abspath(output_folder)
2✔
1122
    working_dir = os.path.abspath(working_dir)
2✔
1123

1124
    dynamic_spinup = False if args.dynamic_spinup == '' else args.dynamic_spinup
2✔
1125

1126
    # All good
1127
    return dict(rgi_version=rgi_version, rgi_reg=rgi_reg,
2✔
1128
                border=border, output_folder=output_folder,
1129
                working_dir=working_dir, params_file=args.params_file,
1130
                is_test=args.test, test_ids=args.test_ids,
1131
                dem_source=args.dem_source,
1132
                start_level=args.start_level, start_base_url=args.start_base_url,
1133
                max_level=args.max_level, disable_mp=args.disable_mp,
1134
                logging_level=args.logging_level,
1135
                elev_bands=args.elev_bands,
1136
                skip_inversion=args.skip_inversion,
1137
                centerlines=args.centerlines,
1138
                select_source_from_dir=args.select_source_from_dir,
1139
                keep_dem_folders=args.keep_dem_folders,
1140
                add_consensus_thickness=args.add_consensus_thickness,
1141
                add_millan_thickness=args.add_millan_thickness,
1142
                add_itslive_velocity=args.add_itslive_velocity,
1143
                add_millan_velocity=args.add_millan_velocity,
1144
                add_hugonnet_dhdt=args.add_hugonnet_dhdt,
1145
                add_bedmachine=args.add_bedmachine,
1146
                add_glathida=args.add_glathida,
1147
                add_distributed_thickness=args.add_distributed_thickness,
1148
                add_export_thickness_geotiff=args.add_export_thickness_geotiff,
1149
                compute_hypsometry=args.compute_hypsometry,
1150
                dynamic_spinup=dynamic_spinup,
1151
                ref_mb_err_scaling_factor=args.ref_mb_err_scaling_factor,
1152
                dynamic_spinup_start_year=args.dynamic_spinup_start_year,
1153
                mb_calibration_strategy=args.mb_calibration_strategy,
1154
                store_fl_diagnostics=args.store_fl_diagnostics,
1155
                override_params=args.override_params,
1156
                )
1157

1158

1159
def main():
2✔
1160
    """Script entry point"""
1161

1162
    run_prepro_levels(**parse_args(sys.argv[1:]))
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc