• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

TUW-GEO / ecmwf_models / 11944977952

06 Nov 2024 01:46PM CUT coverage: 80.452%. Remained the same
11944977952

push

github

web-flow
Update CHANGELOG.rst

189 of 300 branches covered (63.0%)

712 of 885 relevant lines covered (80.45%)

4.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.75
/src/ecmwf_models/cli.py
1
import click
6✔
2
from datetime import datetime
6✔
3

4
from ecmwf_models.era5.download import (download_and_move,
6✔
5
                                        download_record_extension)
6
from ecmwf_models.utils import (default_variables)
6✔
7

8
from ecmwf_models.era5.reshuffle import Reshuffler, extend_ts
6✔
9

10

11
@click.command(
6✔
12
    "download",
13
    context_settings={
14
        'show_default': True,
15
        'help_option_names': ['-h', '--help']
16
    },
17
    short_help="Download ERA5 reanalysis image data between two "
18
    "dates from the Copernicus Climate Data Store (CDS). "
19
    "Before this program can be used, you have to "
20
    "register an account at CDS, and should setup a `.cdsapirc` "
21
    "file as described here: "
22
    "https://cds.climate.copernicus.eu/how-to-api")
23
@click.argument("PATH", type=click.Path(writable=True))
6✔
24
@click.option(
6✔
25
    '--start',
26
    '-s',
27
    type=click.STRING,
28
    default=str(datetime(1979, 1, 1)),
29
    help="First date of the period to download data for. Format: YYYY-MM-DD")
30
@click.option(
6✔
31
    '--end',
32
    '-e',
33
    type=click.STRING,
34
    default=str(datetime.now().date()),
35
    help="Last date of the period to download data for. Format: YYYY-MM-DD")
36
@click.option(
6✔
37
    '--variables',
38
    '-v',
39
    type=click.STRING,
40
    default=','.join(default_variables('era5', 'short_name')),
41
    help="Name of variables to download. To download multiple variables pass "
42
    "comma-separated names (e.g. `-v swvl1,stl1`). "
43
    "For all available variables see the docs. ")
44
@click.option(
6✔
45
    "-k",
46
    "--keep_original",
47
    type=click.BOOL,
48
    default=False,
49
    help="To keep the original image stacks as downloaded from CDS, "
50
    "instead of deleting them after extracting individual images, pass "
51
    "`--keep_original True`")
52
@click.option(
6✔
53
    "-grb",
54
    "--as_grib",
55
    type=click.BOOL,
56
    default=False,
57
    help="To download data in grib format instead of netcdf pass "
58
    "`--as_grib True`.")
59
@click.option(
6✔
60
    "--h_steps",
61
    type=str,
62
    default="0,6,12,18",
63
    help="Temporal sampling of downloaded images. To download multiple time "
64
    "stamps or each day, pass comma-separated values. "
65
    "Pass a set of full hours here, like '--h_steps 0,12' to download "
66
    "two images for each day, at 0:00 and 12:00 respectively. "
67
    "By default, we download 6-hourly images starting at 0:00 UTC, "
68
    "(i.e. `--h_steps 0,6,12,18`")
69
@click.option(
6✔
70
    '--bbox',
71
    nargs=4,
72
    type=click.FLOAT,
73
    default=None,
74
    help="4 NUMBERS | min_lon min_lat max_lon max_lat. "
75
    "Set Bounding Box (lower left and upper right corner) "
76
    "of area to download (WGS84). Default is global.")
77
@click.option(
6✔
78
    "--keep_prelim",
79
    type=click.BOOL,
80
    default=True,
81
    help="The last 1-2 month of data are usually 'preliminary' (labelled as "
82
    "'ERA5-T' and 'ERA5-Land-T') and might be changed if an issue is "
83
    "detected. When this option is deactivated (`--keep_prelim False`), "
84
    "only the final data will be kept, while the ERA5-T data is discarded "
85
    "after download. By default, we also keep preliminary files, but they "
86
    "get a different file name as the final data.")
87
@click.option(
6✔
88
    "--max_request_size",
89
    type=int,
90
    default=1000,
91
    help="Maximum number of requests to pass to the CDS API. "
92
    "The default is 1000, but what is allowed, depends on server "
93
    "settings. Server settings may change at some point. Change "
94
    "accordingly here in case that 'the request is too large'. "
95
    "A smaller number will results in smaller download chunks (slower).")
96
@click.option(
6✔
97
    "--cds_token",
98
    type=click.STRING,
99
    default=None,
100
    help="To identify with the CDS. Required only if no `.cdsapirc` file "
101
    "exists in the home directory (see documentation). "
102
    "You can find your token/key on your CDS user profile page. "
103
    "Alternatively, you can also set an environment variable "
104
    "`CDSAPI_KEY` with your token.")
105
def cli_download_era5(path, start, end, variables, keep_original, as_grib,
6✔
106
                      h_steps, bbox, keep_prelim, max_request_size, cds_token):
107
    """
108
    Download ERA5 image data within the chosen period. NOTE: Before using this
109
    program, create a CDS account and set up a `.cdsapirc` file as described
110
    here: https://cds.climate.copernicus.eu/how-to-api
111

112
    \b
113
    Required Parameters
114
    -------------------
115
    > PATH: string (required)
116
          Root of local filesystem where the downloaded data will be stored.
117
          Make sure to set up the CDS API for your account as described at
118
          https://cds.climate.copernicus.eu/how-to-api
119
    """
120
    # The docstring above is slightly different to the normal python one to
121
    # display it properly on the command line.
122

123
    h_steps = [int(h.strip()) for h in h_steps.split(',')]
×
124
    variables = [str(v.strip()) for v in variables.split(',')]
×
125

126
    status_code = download_and_move(
×
127
        target_path=path,
128
        startdate=start,
129
        enddate=end,
130
        product="era5",
131
        variables=variables,
132
        h_steps=h_steps,
133
        grb=as_grib,
134
        bbox=bbox,
135
        keep_original=keep_original,
136
        stepsize='month',
137
        n_max_request=max_request_size,
138
        keep_prelim=keep_prelim,
139
        cds_token=cds_token,
140
    )
141

142
    return status_code
×
143

144

145
@click.command(
6✔
146
    "download",
147
    context_settings={
148
        'show_default': True,
149
        'help_option_names': ['-h', '--help']
150
    },
151
    short_help="Download ERA5 reanalysis image data between two "
152
    "dates from the Copernicus Climate Data Store (CDS). "
153
    "Before this program can be used, you have to "
154
    "register an account at CDS, and should setup a `.cdsapirc` "
155
    "file as described here: "
156
    "https://cds.climate.copernicus.eu/how-to-api")
157
@click.argument("PATH", type=click.Path(writable=True))
6✔
158
@click.option(
6✔
159
    '--start',
160
    '-s',
161
    type=click.STRING,
162
    default=str(datetime(1979, 1, 1)),
163
    help="First date of the period to download data for. Format: YYYY-MM-DD")
164
@click.option(
6✔
165
    '--end',
166
    '-e',
167
    type=click.STRING,
168
    default=str(datetime.now().date()),
169
    help="Last date of the period to download data for. Format: YYYY-MM-DD")
170
@click.option(
6✔
171
    '--variables',
172
    '-v',
173
    type=click.STRING,
174
    default=','.join(default_variables('era5-land',
175
                                       'short_name')),
176
    help="Name of variables to download. To download multiple variables pass "
177
    "comma-separated names (e.g. `-v swvl1,stl1`). "
178
    "For all available variables see the docs. ")
179
@click.option(
6✔
180
    "-k",
181
    "--keep_original",
182
    type=click.BOOL,
183
    default=False,
184
    help="To keep the original image stacks as downloaded from CDS, "
185
    "instead of deleting them after extracting individual images, pass "
186
    "`--keep_original True`")
187
@click.option(
6✔
188
    "-grb",
189
    "--as_grib",
190
    type=click.BOOL,
191
    default=False,
192
    help="To download data in grib format instead of netcdf pass "
193
    "`--as_grib True`.")
194
@click.option(
6✔
195
    "--h_steps",
196
    type=click.STRING,
197
    default='0,6,12,18',
198
    help="Temporal sampling of downloaded images. To download multiple time "
199
    "stamps or each day, pass comma-separated values. "
200
    "Pass a set of full hours here, like '--h_steps 0,12' to download "
201
    "two images for each day, at 0:00 and 12:00 respectively. "
202
    "By default, we download 6-hourly images starting at 0:00 UTC, "
203
    "(i.e. `--h_steps 0,6,12,18`")
204
@click.option(
6✔
205
    "--keep_prelim",
206
    type=click.BOOL,
207
    default=True,
208
    help="The last 1-2 month of data are usually 'preliminary' (labelled as "
209
    "'ERA5-T' and 'ERA5-Land-T') and might be changed if an issue is "
210
    "detected. When this option is deactivated (`--keep_prelim False`), "
211
    "only the final data will be kept, while the ERA5-T data is discarded "
212
    "after download. By default, we also keep preliminary files, but they "
213
    "get a different file name as the final data.")
214
@click.option(
6✔
215
    "--max_request_size",
216
    type=int,
217
    default=1000,
218
    help="Maximum number of requests to pass to the CDS API. "
219
    "The default is 1000, but what is allowed, depends on server "
220
    "settings. Server settings may change at some point. Change "
221
    "accordingly here in case that 'the request is too large'. "
222
    "A smaller number will results in smaller download chunks (slower).")
223
@click.option(
6✔
224
    "--cds_token",
225
    type=click.STRING,
226
    default=None,
227
    help="To identify with the CDS. Required only if no `.cdsapirc` file "
228
    "exists in the home directory (see documentation). "
229
    "You can find your token/key on your CDS user profile page. "
230
    "Alternatively, you can also set an environment variable "
231
    "`CDSAPI_KEY` with your token.")
232
def cli_download_era5land(path, start, end, variables, keep_original, as_grib,
6✔
233
                          h_steps, keep_prelim, max_request_size, cds_token):
234
    """
235
    Download ERA5-Land image data within a chosen period.
236
    NOTE: Before using this program, create a CDS account and set up a
237
    `.cdsapirc` file as described here:
238
    https://cds.climate.copernicus.eu/how-to-api
239

240
    \b
241
    Required Parameters
242
    -------------------
243
    > PATH: string (required)
244
          Root of local filesystem where the downloaded data will be stored.
245
          Make sure to set up the CDS API for your account as described at
246
          https://cds.climate.copernicus.eu/how-to-api
247
    """
248
    # The docstring above is slightly different to the normal python one to
249
    # display it properly on the command line.
250

251
    h_steps = [int(h.strip()) for h in h_steps.split(',')]
×
252
    variables = [str(v.strip()) for v in variables.split(',')]
×
253

254
    status_code = download_and_move(
×
255
        target_path=path,
256
        startdate=start,
257
        enddate=end,
258
        product="era5-land",
259
        variables=variables,
260
        h_steps=h_steps,
261
        grb=as_grib,
262
        keep_original=keep_original,
263
        stepsize='month',
264
        n_max_request=max_request_size,
265
        keep_prelim=keep_prelim,
266
        cds_token=cds_token)
267

268
    return status_code
×
269

270

271
@click.command(
6✔
272
    "update_img",
273
    context_settings={
274
        'show_default': True,
275
        'help_option_names': ['-h', '--help']
276
    },
277
    short_help="Extend an existing set of images by downloading new data "
278
    "with the same settings as before."
279
    "NOTE: First use the `era5 download` or `era5land download` "
280
    "programs.")
281
@click.argument("path", type=click.Path(writable=True))
6✔
282
@click.option(
6✔
283
    "--cds_token",
284
    type=click.STRING,
285
    default=None,
286
    help="To identify with the CDS. Required only if no `.cdsapirc` file "
287
    "exists in the home directory (see documentation). "
288
    "You can find your token/key on your CDS user profile page. "
289
    "Alternatively, you can also set an environment variable "
290
    "`CDSAPI_KEY` with your token.")
291
def cli_update_img(path, cds_token=None):
6✔
292
    """
293
    Download new images from CDS to your existing local archive. Use the same
294
    settings as before.
295
    NOTE: First use the `era5 download` or `era5land download` programs.
296
    The so-created archive can then be updated using this program.
297

298
    \b
299
    Required Parameters
300
    -------------------
301
    > PATH: string (required)
302
          Path where previously downloaded images are stored. There must
303
          be a file `overview.yml` which contains the download settings.
304
          Make sure to set up the CDS API for your account as described at
305
          https://cds.climate.copernicus.eu/how-to-api,
306
    """
307
    # The docstring above is slightly different to the normal python one to
308
    # display it properly on the command line.
309

310
    download_record_extension(path, cds_token=cds_token)
×
311

312

313
@click.command(
6✔
314
    "reshuffle",
315
    context_settings={
316
        'show_default': True,
317
        'help_option_names': ['-h', '--help']
318
    },
319
    short_help="Convert previously downloaded ERA5/ERA5-Land image "
320
    "data into a time series format.")
321
@click.argument("IMG_PATH", type=click.Path(readable=True))
6✔
322
@click.argument("TS_PATH", type=click.Path(writable=True))
6✔
323
@click.option(
6✔
324
    '--start',
325
    '-s',
326
    type=click.STRING,
327
    default=None,
328
    help="First date of the period to reshuffle data for. By default, the "
329
         "first available image is used. Format: YYYY-MM-DD")
330
@click.option(
6✔
331
    '--end',
332
    '-e',
333
    type=click.STRING,
334
    default=None,
335
    help="Last date of the period to reshuffle data for. By default, the last"
336
         "available image is used. Format: YYYY-MM-DD")
337
@click.option(
6✔
338
    '--variables',
339
    '-v',
340
    type=click.STRING,
341
    default=None,
342
    help="Subset of variables to convert. Pass comma-separated names"
343
    "to select multiple variables (short names, as in the input images, "
344
    "e.g. `-v swvl1,stl1`). If not specified, all variables from the "
345
    "first image file are used.")
346
@click.option(
6✔
347
    '--land_points',
348
    '-l',
349
    type=click.BOOL,
350
    show_default=True,
351
    default=False,
352
    help="To store only time series for points that are over land, pass "
353
         "`--land_points True`. ")
354
@click.option(
6✔
355
    '--bbox',
356
    nargs=4,
357
    type=click.FLOAT,
358
    default=None,
359
    help="4 NUMBERS | min_lon min_lat max_lon max_lat. "
360
    "Set Bounding Box (lower left and upper right corner) "
361
    "of area to reshuffle (WGS84). Default is global.")
362
@click.option(
6✔
363
    "--h_steps",
364
    type=click.STRING,
365
    default="0,6,12,18",
366
    help="Full hour time stamps of images to include in time series. "
367
    "To select multiple, pass comma-separated values here, "
368
    "e.g. '--h_steps 0,12' will only include data from images at "
369
    "0:00 and 12:00 UTC and ignore all other available time stamps."
370
    "By default, we include data for every 6th hour each day.")
371
@click.option(
6✔
372
    '--imgbuffer',
373
    '-b',
374
    type=click.INT,
375
    default=50,
376
    help="Number of images to read into memory at once before "
377
    "conversion to time series. A larger buffer means faster "
378
    "processing but requires more memory.")
379
def cli_reshuffle(img_path, ts_path, start, end, variables, land_points, bbox,
6✔
380
                  h_steps, imgbuffer):
381
    """
382
    Convert previously downloaded ERA5 or ERA5-Land image data into a
383
    time series format.
384

385
    \b
386
    Required Parameters
387
    -------------------
388
    > IMG_PATH: string (required)
389
          Directory where the downloaded image data is stored, i.e., where
390
          annual folders are found.
391
    > TS_PATH: string (required)
392
          Root of local filesystem where the time series should be stored.
393
    """
394

395
    h_steps = [int(h.strip()) for h in h_steps.split(',')]
6✔
396

397
    if variables is not None:
6!
398
        variables = [str(v.strip()) for v in variables.split(',')]
6✔
399

400
    print(f"Converting ERA5/ERA5-Land images in period from "
6✔
401
          f"{'first available image' if start is None else start} to "
402
          f"{'last available image' if end is None else end} into "
403
          f"time series to {ts_path}.")
404

405
    reshuffler = Reshuffler(img_path, ts_path,
6✔
406
                            variables=variables,
407
                            h_steps=h_steps,
408
                            land_points=land_points,
409
                            product=None  # Infer prod automatically from files
410
                            )
411
    reshuffler.reshuffle(startdate=start, enddate=end, bbox=bbox,
6✔
412
                         imgbuffer=imgbuffer)
413

414

415
@click.command(
6✔
416
    "update_ts",
417
    context_settings={
418
        'show_default': True,
419
        'help_option_names': ['-h', '--help']
420
    },
421
    short_help="Append new image data to an existing time series archive.")
422
@click.argument("TS_PATH", type=click.Path(writable=True))
6✔
423
@click.option(
6✔
424
    '--imgpath',
425
    '-p',
426
    type=click.STRING,
427
    default=None,
428
    help="Manually specify where the image data to extend the time "
429
    "series are located. If this is not specified, we use the previously "
430
    "used path from the `overview.yml` file stored with the time series "
431
    "data.")
432
@click.option(
6✔
433
    '--imgbuffer',
434
    '-b',
435
    type=click.INT,
436
    default=50,
437
    help="Number of images to read into memory at once before "
438
    "conversion to time series. A larger buffer means faster "
439
    "processing but requires more memory.")
440
def cli_extend_ts(ts_path, imgpath, imgbuffer):
6✔
441
    """
442
    Append new image data to an existing time series archive. The archive must
443
    be created first using the `reshuffle` program. We will use the same
444
    settings as in the initial run (see `overview.yml` in TS_PATH) for
445
    consistent extensions.
446

447
    \b
448
    Required Parameters
449
    -------------------
450
    > TS_PATH: string (required)
451
          Root of local filesystem where the time series from a previous call
452
          of `reshuffle` are stored. New data will be appended to the existing
453
          files!
454
    """
455
    kwargs = dict(ts_path=ts_path, imgbuffer=imgbuffer)
6✔
456

457
    if imgpath is not None:  # otherwise use path from yml
6!
458
        kwargs["input_root"] = imgpath
×
459

460
    extend_ts(**kwargs)
6✔
461

462

463
@click.group(short_help="ERA5 Command Line Programs imported from the "
6✔
464
             "`ecmwf_models` pip package.")
465
def era5():
6✔
466
    pass
6✔
467

468

469
era5.add_command(cli_download_era5)
6✔
470
era5.add_command(cli_update_img)
6✔
471
era5.add_command(cli_reshuffle)
6✔
472
era5.add_command(cli_extend_ts)
6✔
473

474

475
@click.group(short_help="ERA5-Land Command Line Programs imported from the "
6✔
476
             "`ecmwf_models` pip package.")
477
def era5land():
6✔
478
    pass
×
479

480

481
era5land.add_command(cli_download_era5land)
6✔
482
era5land.add_command(cli_update_img)
6✔
483
era5land.add_command(cli_reshuffle)
6✔
484
era5land.add_command(cli_extend_ts)
6✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc