30524401593

Committed 30 Jul 2026 07:52AM UTC coverage: 84.749%. First build

Build # 30524401593

Build Type

Pull #1908

github

Committed by

web-flow

Commit Message

Merge e0ce02050 into 180c7c28f

Pull Request Pull Request #1908: feat: read and write zarr methods to replace read_pickle

Coverage Stats

4440 of 6148 branches covered (72.22%)

637 of 745 new or added lines in 13 files covered. (85.5%)

14537 of 17153 relevant lines covered (84.75%)

4.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.83

/oggm/cli/benchmark.py

"""Command line arguments to the oggm_benchmark command

Type `$ oggm_benchmark -h` for help

"""

# External modules
import os
import sys
import argparse
import time
import platform
import logging
import pandas as pd
import geopandas as gpd

# Locals
import oggm.cfg as cfg
from oggm import utils, workflow, tasks
from oggm.exceptions import InvalidParamsError


def _add_time_to_df(df, index, t):
    df.loc[index, 't'] = t
    m, s = divmod(t, 60)
    h, m = divmod(m, 60)
    df.loc[index, 'H'] = h
    df.loc[index, 'M'] = m
    df.loc[index, 'S'] = s


def run_benchmark(rgi_version=None, rgi_reg=None, border=None,
                  output_folder='', working_dir='', is_test=False,
                  logging_level='WORKFLOW', test_rgidf=None,
                  test_intersects_file=None, override_params=None,
                  test_topofile=None):
    """Does the actual job.

    Parameters
    ----------
    rgi_version : str
        the RGI version to use (defaults to cfg.PARAMS)
    rgi_reg : str
        the RGI region to process
    border : int
        the number of pixels at the maps border
    output_folder : str
        path to the output folder (where to put the preprocessed tar files)
    working_dir : str
        path to the OGGM working directory
    is_test : bool
        to test on a couple of glaciers only!
    test_rgidf : shapefile
        for testing purposes only
    test_intersects_file : shapefile
        for testing purposes only
    test_topofile : str
        for testing purposes only
    override_params : dict
        a dict of parameters to override.
    """

    # Module logger
    log = logging.getLogger(__name__)

    # Params
    if override_params is None:
        override_params = {}

    utils.mkdir(working_dir)
    override_params['working_dir'] = working_dir

    # Initialize OGGM and set up the run parameters
    cfg.initialize(logging_level=logging_level, params=override_params)

    # Allow multiprocessing override in tests/platform
    if 'use_multiprocessing' not in override_params:
        cfg.PARAMS['use_multiprocessing'] = platform.system() != 'Darwin'

    # How many grid points around the glacier?
    # Make it large if you expect your glaciers to grow large
    cfg.PARAMS['border'] = border

    # Set to True for operational runs
    cfg.PARAMS['continue_on_error'] = True

    # For statistics
    odf = pd.DataFrame()

    if rgi_version is None:
        rgi_version = cfg.PARAMS['rgi_version']
    base_dir = os.path.join(output_folder)

    # Add a package version file
    utils.mkdir(base_dir)
    opath = os.path.join(base_dir, 'package_versions.txt')
    with open(opath, 'w') as vfile:
        vfile.write(utils.show_versions(logger=log))

    # Read RGI
    start = time.time()
    if test_rgidf is None:
        # Get the RGI file
        rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,
                                                        version=rgi_version))
        # We use intersects
        rgif = utils.get_rgi_intersects_region_file(rgi_reg,
                                                    version=rgi_version)
        cfg.set_intersects_db(rgif)
    else:
        rgidf = test_rgidf
        cfg.set_intersects_db(test_intersects_file)

    if is_test:
        # Just for fun
        rgidf = rgidf.sample(2)
    _add_time_to_df(odf, 'Read RGI', time.time()-start)

    # Sort for more efficient parallel computing
    rgidf = rgidf.sort_values('Area', ascending=False)

    log.workflow('Starting prepro run for RGI reg: {} '
                 'and border: {}'.format(rgi_reg, border))
    log.workflow('Number of glaciers: {}'.format(len(rgidf)))

    # Input
    if test_topofile:
        cfg.PATHS['dem_file'] = test_topofile

    # Initialize working directories
    start = time.time()
    gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)
    _add_time_to_df(odf, 'init_glacier_directories', time.time()-start)

    # Tasks
    task_list = [
        tasks.define_glacier_region,
        tasks.process_cru_data,
        tasks.simple_glacier_masks,
        tasks.elevation_band_flowline,
        tasks.fixed_dx_elevation_band_flowline,
        tasks.compute_downstream_line,
        tasks.compute_downstream_bedshape,
        tasks.mb_calibration_from_geodetic_mb,
        tasks.apparent_mb_from_any_mb,
        tasks.prepare_for_inversion,
        tasks.mass_conservation_inversion,
        tasks.filter_inversion_output,
        tasks.init_present_time_glacier,
    ]
    for task in task_list:
        start = time.time()
        workflow.execute_entity_task(task, gdirs)
        _add_time_to_df(odf, task.__name__, time.time()-start)

    # Runs
    start = time.time()
    workflow.execute_entity_task(tasks.run_constant_climate, gdirs,
                                 nyears=250, y0=1995,
                                 temperature_bias=-0.5,
                                 output_filesuffix='_constant')
    _add_time_to_df(odf, 'run_constant_climate_commit_250', time.time()-start)

    start = time.time()
    workflow.execute_entity_task(tasks.run_random_climate, gdirs,
                                 nyears=250, y0=1995, seed=0,
                                 output_filesuffix='_random')
    _add_time_to_df(odf, 'run_random_climate_commit_250', time.time()-start)

    # Compile results
    start = time.time()
    utils.compile_glacier_statistics(gdirs)
    _add_time_to_df(odf, 'compile_glacier_statistics', time.time()-start)

    start = time.time()
    utils.compile_climate_statistics(gdirs,
                                     add_climate_period=[1920, 1960, 2000])
    _add_time_to_df(odf, 'compile_climate_statistics', time.time()-start)

    start = time.time()
    utils.compile_run_output(gdirs, input_filesuffix='_constant')
    _add_time_to_df(odf, 'compile_run_output_constant', time.time()-start)

    start = time.time()
    utils.compile_run_output(gdirs, input_filesuffix='_random')
    _add_time_to_df(odf, 'compile_run_output_random', time.time()-start)

    # Log
    opath = os.path.join(base_dir, 'benchmarks_b{:03d}.csv'.format(border))
    odf.index.name = 'Task'
    odf.to_csv(opath)
    log.workflow('OGGM benchmarks is done!')


def parse_args(args):
    """Check input arguments and env variables"""

    # CLI args
    description = ('Run an OGGM benchmark on a selected RGI Region. '
                   'This writes a benchmark_{border}.txt file where '
                   'the results are summarized')
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('--map-border', type=int,
                        help='the size of the map border. Is required if '
                             '$OGGM_MAP_BORDER is not set.')
    parser.add_argument('--rgi-reg', type=str,
                        help='the rgi region to process. Is required if '
                             '$OGGM_RGI_REG is not set.')
    parser.add_argument('--rgi-version', type=str,
                        help='the RGI version to use. Defaults to the OGGM '
                             'default.')
    parser.add_argument('--working-dir', type=str,
                        help='path to the directory where to write the '
                             'output. Defaults to current directory or '
                             '$OGGM_WORKDIR.')
    parser.add_argument('--output', type=str,
                        help='path to the directory where to write the '
                             'output. Defaults to current directory or'
                             '$OGGM_OUTDIR.')
    parser.add_argument('--test', nargs='?', const=True, default=False,
                        help='if you want to do a test on a couple of '
                             'glaciers first.')
    args = parser.parse_args(args)

    # Check input
    rgi_reg = args.rgi_reg
    if not rgi_reg:
        rgi_reg = os.environ.get('OGGM_RGI_REG', None)
        if rgi_reg is None:
            raise InvalidParamsError('--rgi-reg is required!')
    rgi_reg = '{:02}'.format(int(rgi_reg))

    rgi_version = args.rgi_version

    border = args.map_border
    if not border:
        border = os.environ.get('OGGM_MAP_BORDER', None)
        if border is None:
            raise InvalidParamsError('--map-border is required!')

    working_dir = args.working_dir
    if not working_dir:
        working_dir = os.environ.get('OGGM_WORKDIR', '')

    output_folder = args.output
    if not output_folder:
        output_folder = os.environ.get('OGGM_OUTDIR', '')

    border = int(border)
    output_folder = os.path.abspath(output_folder)
    working_dir = os.path.abspath(working_dir)

    # All good
    return dict(rgi_version=rgi_version, rgi_reg=rgi_reg,
                border=border, output_folder=output_folder,
                working_dir=working_dir, is_test=args.test)


def main():
    """Script entry point"""

    run_benchmark(**parse_args(sys.argv[1:]))

1	"""Command line arguments to the oggm_benchmark command
2
3	Type `$ oggm_benchmark -h` for help
4
5	"""
6
7	# External modules
8	import os	1✔
9	import sys	1✔
10	import argparse	1✔
11	import time	1✔
12	import platform	1✔
13	import logging	1✔
14	import pandas as pd	1✔
15	import geopandas as gpd	1✔
16
17	# Locals
18	import oggm.cfg as cfg	1✔
19	from oggm import utils, workflow, tasks	1✔
20	from oggm.exceptions import InvalidParamsError	1✔
21
22
23	def _add_time_to_df(df, index, t):	1✔
24	df.loc[index, 't'] = t	1✔
25	m, s = divmod(t, 60)	1✔
26	h, m = divmod(m, 60)	1✔
27	df.loc[index, 'H'] = h	1✔
28	df.loc[index, 'M'] = m	1✔
29	df.loc[index, 'S'] = s	1✔
30
31
32	def run_benchmark(rgi_version=None, rgi_reg=None, border=None,	1✔
33	output_folder='', working_dir='', is_test=False,
34	logging_level='WORKFLOW', test_rgidf=None,
35	test_intersects_file=None, override_params=None,
36	test_topofile=None):
37	"""Does the actual job.
38
39	Parameters
40	----------
41	rgi_version : str
42	the RGI version to use (defaults to cfg.PARAMS)
43	rgi_reg : str
44	the RGI region to process
45	border : int
46	the number of pixels at the maps border
47	output_folder : str
48	path to the output folder (where to put the preprocessed tar files)
49	working_dir : str
50	path to the OGGM working directory
51	is_test : bool
52	to test on a couple of glaciers only!
53	test_rgidf : shapefile
54	for testing purposes only
55	test_intersects_file : shapefile
56	for testing purposes only
57	test_topofile : str
58	for testing purposes only
59	override_params : dict
60	a dict of parameters to override.
61	"""
62
63	# Module logger
64	log = logging.getLogger(__name__)	1✔
65
66	# Params
67	if override_params is None:	1!
68	override_params = {}	×
69
70	utils.mkdir(working_dir)	1✔
71	override_params['working_dir'] = working_dir	1✔
72
73	# Initialize OGGM and set up the run parameters
74	cfg.initialize(logging_level=logging_level, params=override_params)	1✔
75
76	# Allow multiprocessing override in tests/platform
77	if 'use_multiprocessing' not in override_params:	1!
NEW 78	cfg.PARAMS['use_multiprocessing'] = platform.system() != 'Darwin'	×
79
80	# How many grid points around the glacier?
81	# Make it large if you expect your glaciers to grow large
82	cfg.PARAMS['border'] = border	1✔
83
84	# Set to True for operational runs
85	cfg.PARAMS['continue_on_error'] = True	1✔
86
87	# For statistics
88	odf = pd.DataFrame()	1✔
89
90	if rgi_version is None:	1!
91	rgi_version = cfg.PARAMS['rgi_version']	1✔
92	base_dir = os.path.join(output_folder)	1✔
93
94	# Add a package version file
95	utils.mkdir(base_dir)	1✔
96	opath = os.path.join(base_dir, 'package_versions.txt')	1✔
97	with open(opath, 'w') as vfile:	1✔
98	vfile.write(utils.show_versions(logger=log))	1✔
99
100	# Read RGI
101	start = time.time()	1✔
102	if test_rgidf is None:	1!
103	# Get the RGI file
104	rgidf = gpd.read_file(utils.get_rgi_region_file(rgi_reg,	×
105	version=rgi_version))
106	# We use intersects
107	rgif = utils.get_rgi_intersects_region_file(rgi_reg,	×
108	version=rgi_version)
109	cfg.set_intersects_db(rgif)	×
110	else:
111	rgidf = test_rgidf	1✔
112	cfg.set_intersects_db(test_intersects_file)	1✔
113
114	if is_test:	1!
115	# Just for fun
116	rgidf = rgidf.sample(2)	1✔
117	_add_time_to_df(odf, 'Read RGI', time.time()-start)	1✔
118
119	# Sort for more efficient parallel computing
120	rgidf = rgidf.sort_values('Area', ascending=False)	1✔
121
122	log.workflow('Starting prepro run for RGI reg: {} '	1✔
123	'and border: {}'.format(rgi_reg, border))
124	log.workflow('Number of glaciers: {}'.format(len(rgidf)))	1✔
125
126	# Input
127	if test_topofile:	1!
128	cfg.PATHS['dem_file'] = test_topofile	1✔
129
130	# Initialize working directories
131	start = time.time()	1✔
132	gdirs = workflow.init_glacier_directories(rgidf, reset=True, force=True)	1✔
133	_add_time_to_df(odf, 'init_glacier_directories', time.time()-start)	1✔
134
135	# Tasks
136	task_list = [	1✔
137	tasks.define_glacier_region,
138	tasks.process_cru_data,
139	tasks.simple_glacier_masks,
140	tasks.elevation_band_flowline,
141	tasks.fixed_dx_elevation_band_flowline,
142	tasks.compute_downstream_line,
143	tasks.compute_downstream_bedshape,
144	tasks.mb_calibration_from_geodetic_mb,
145	tasks.apparent_mb_from_any_mb,
146	tasks.prepare_for_inversion,
147	tasks.mass_conservation_inversion,
148	tasks.filter_inversion_output,
149	tasks.init_present_time_glacier,
150	]
151	for task in task_list:	1✔
152	start = time.time()	1✔
153	workflow.execute_entity_task(task, gdirs)	1✔
154	_add_time_to_df(odf, task.__name__, time.time()-start)	1✔
155
156	# Runs
157	start = time.time()	1✔
158	workflow.execute_entity_task(tasks.run_constant_climate, gdirs,	1✔
159	nyears=250, y0=1995,
160	temperature_bias=-0.5,
161	output_filesuffix='_constant')
162	_add_time_to_df(odf, 'run_constant_climate_commit_250', time.time()-start)	1✔
163
164	start = time.time()	1✔
165	workflow.execute_entity_task(tasks.run_random_climate, gdirs,	1✔
166	nyears=250, y0=1995, seed=0,
167	output_filesuffix='_random')
168	_add_time_to_df(odf, 'run_random_climate_commit_250', time.time()-start)	1✔
169
170	# Compile results
171	start = time.time()	1✔
172	utils.compile_glacier_statistics(gdirs)	1✔
173	_add_time_to_df(odf, 'compile_glacier_statistics', time.time()-start)	1✔
174
175	start = time.time()	1✔
176	utils.compile_climate_statistics(gdirs,	1✔
177	add_climate_period=[1920, 1960, 2000])
178	_add_time_to_df(odf, 'compile_climate_statistics', time.time()-start)	1✔
179
180	start = time.time()	1✔
181	utils.compile_run_output(gdirs, input_filesuffix='_constant')	1✔
182	_add_time_to_df(odf, 'compile_run_output_constant', time.time()-start)	1✔
183
184	start = time.time()	1✔
185	utils.compile_run_output(gdirs, input_filesuffix='_random')	1✔
186	_add_time_to_df(odf, 'compile_run_output_random', time.time()-start)	1✔
187
188	# Log
189	opath = os.path.join(base_dir, 'benchmarks_b{:03d}.csv'.format(border))	1✔
190	odf.index.name = 'Task'	1✔
191	odf.to_csv(opath)	1✔
192	log.workflow('OGGM benchmarks is done!')	1✔
193
194
195	def parse_args(args):	1✔
196	"""Check input arguments and env variables"""
197
198	# CLI args
199	description = ('Run an OGGM benchmark on a selected RGI Region. '	1✔
200	'This writes a benchmark_{border}.txt file where '
201	'the results are summarized')
202	parser = argparse.ArgumentParser(description=description)	1✔
203	parser.add_argument('--map-border', type=int,	1✔
204	help='the size of the map border. Is required if '
205	'$OGGM_MAP_BORDER is not set.')
206	parser.add_argument('--rgi-reg', type=str,	1✔
207	help='the rgi region to process. Is required if '
208	'$OGGM_RGI_REG is not set.')
209	parser.add_argument('--rgi-version', type=str,	1✔
210	help='the RGI version to use. Defaults to the OGGM '
211	'default.')
212	parser.add_argument('--working-dir', type=str,	1✔
213	help='path to the directory where to write the '
214	'output. Defaults to current directory or '
215	'$OGGM_WORKDIR.')
216	parser.add_argument('--output', type=str,	1✔
217	help='path to the directory where to write the '
218	'output. Defaults to current directory or'
219	'$OGGM_OUTDIR.')
220	parser.add_argument('--test', nargs='?', const=True, default=False,	1✔
221	help='if you want to do a test on a couple of '
222	'glaciers first.')
223	args = parser.parse_args(args)	1✔
224
225	# Check input
226	rgi_reg = args.rgi_reg	1✔
227	if not rgi_reg:	1✔
228	rgi_reg = os.environ.get('OGGM_RGI_REG', None)	1✔
229	if rgi_reg is None:	1✔
230	raise InvalidParamsError('--rgi-reg is required!')	1✔
231	rgi_reg = '{:02}'.format(int(rgi_reg))	1✔
232
233	rgi_version = args.rgi_version	1✔
234
235	border = args.map_border	1✔
236	if not border:	1✔
237	border = os.environ.get('OGGM_MAP_BORDER', None)	1✔
238	if border is None:	1✔
239	raise InvalidParamsError('--map-border is required!')	1✔
240
241	working_dir = args.working_dir	1✔
242	if not working_dir:	1✔
243	working_dir = os.environ.get('OGGM_WORKDIR', '')	1✔
244
245	output_folder = args.output	1✔
246	if not output_folder:	1✔
247	output_folder = os.environ.get('OGGM_OUTDIR', '')	1✔
248
249	border = int(border)	1✔
250	output_folder = os.path.abspath(output_folder)	1✔
251	working_dir = os.path.abspath(working_dir)	1✔
252
253	# All good
254	return dict(rgi_version=rgi_version, rgi_reg=rgi_reg,	1✔
255	border=border, output_folder=output_folder,
256	working_dir=working_dir, is_test=args.test)
257
258
259	def main():	1✔
260	"""Script entry point"""
261
262	run_benchmark(**parse_args(sys.argv[1:]))	×

OGGM / oggm / 30524401593

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous