11265563327

Committed 09 Oct 2024 08:21AM UTC coverage: 78.676% (-12.0%) from 90.69%

Build # 11265563327

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #15 from wpreimes/smos_l2

Smos l2

Run Details

172 of 271 branches covered (63.47%)

Branch coverage included in aggregate %.

160 of 236 new or added lines in 4 files covered. (67.8%)

1 existing line in 1 file now uncovered.

577 of 681 relevant lines covered (84.73%)

2.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.13

/src/smos/smos_l2/reshuffle.py

import pandas as pd
import os
import yaml
from qa4sm_preprocessing.level2.smos import SMOSL2Reader
from smos.smos_l2.download import get_avail_img_range
from datetime import datetime

def read_summary_yml(path: str) -> dict:
    """
    Read image summary and return fields as dict.
    """
    path = os.path.join(path, 'overview.yml')

    with open(path, 'r') as stream:
        props = yaml.safe_load(stream)

    return props


def swath2ts(img_path, ts_path, startdate=None, enddate=None, memory=4):
    """
    Convert SMOS L2 swath data to time series in IndexedRaggedTs format.

    Parameters
    ----------
    img_path: str
        Local (root) directory where the annual folder containing SMOS L2 SM
        swath data are found.
    ts_path: str
        Local directory where the converted time series data will be stored.
    startdate: str or datetime, optional (default: None)
        First day of the available swath data that should be included in the
        time series. If None is passed, then the first available day is used.
    enddate: str or datetime, optional (default: None)
        Last day of the available swath data that should be included in the
        time series. If None is passed, then the last available day is used.
    memory : float, optional (default: 4)
        Size of available memory in GB. More memory will lead to a faster
        conversion.
    """
    reader = SMOSL2Reader(img_path)

    first_day, last_day = get_avail_img_range(img_path)

    start = pd.to_datetime(startdate).to_pydatetime() if startdate is not None else first_day
    end = pd.to_datetime(enddate).to_pydatetime() if enddate is not None else last_day

    out_file = os.path.join(ts_path, f"overview.yml")

    if os.path.isfile(out_file):
        props = read_summary_yml(ts_path)
        if start < pd.to_datetime(props['enddate']).to_pydatetime():
            raise ValueError("Cannot prepend data to time series, or replace "
                             "existing values. Choose different start date.")

    props = {'enddate': str(end), 'last_update': str(datetime.now()),
             'parameters': [str(v) for v in reader.varnames]}

    r = reader.repurpose(
        outpath=ts_path,
        start=start,
        end=end,
        memory=memory,
        overwrite=False,
        imgbaseconnection=True,
    )

    if r is not None:
        with open(out_file, 'w') as f:
            yaml.dump(props, f, default_flow_style=False, sort_keys=False)

def extend_ts(img_path, ts_path, memory=4):
    """
    Append new image data to an existing time series record.
    This will use the enddate from summary.yml in the time series
    directory to decide which date the update should start from and
    the available image directories to decide how many images can be
    appended.

    Parameters
    ----------
    img_path: str
        Path where the annual folders containing downloaded SMOS L2 images
        are stored
    ts_path: str
        Path where the converted time series (initially created using the
        reshuffle / swath2ts command) are stored.
    memory: int, optional (default: 4)
        Available memory in GB
    """
    out_file = os.path.join(ts_path, f"overview.yml")
    if not os.path.isfile(out_file):
        raise ValueError("No overview.yml found in the time series directory."
                         "Please use reshuffle / swath2ts for initial time "
                         f"series setup or provide overview.yml in {ts_path}.")

    props = read_summary_yml(ts_path)
    startdate = pd.to_datetime(props['enddate']).to_pydatetime()
    _, enddate = get_avail_img_range(img_path)

    reader = SMOSL2Reader(img_path)

    print(f"From: {startdate}, To: {enddate}")

    r = reader.repurpose(
        outpath=ts_path,
        start=startdate,
        end=enddate,
        memory=memory,
        imgbaseconnection=True,
        overwrite=False,
        append=True,
    )

    if r is not None:
        props['enddate'] = str(enddate)
        props['last_update'] = str(datetime.now())

        with open(out_file, 'w') as f:
            yaml.dump(props, f, default_flow_style=False, sort_keys=False)

1	import pandas as pd	3✔
2	import os	3✔
3	import yaml	3✔
4	from qa4sm_preprocessing.level2.smos import SMOSL2Reader	3✔
5	from smos.smos_l2.download import get_avail_img_range	3✔
6	from datetime import datetime	3✔
7
8	def read_summary_yml(path: str) -> dict:	3✔
9	"""
10	Read image summary and return fields as dict.
11	"""
12	path = os.path.join(path, 'overview.yml')	3✔
13
14	with open(path, 'r') as stream:	3✔
15	props = yaml.safe_load(stream)	3✔
16
17	return props	3✔
18
19
20	def swath2ts(img_path, ts_path, startdate=None, enddate=None, memory=4):	3✔
21	"""
22	Convert SMOS L2 swath data to time series in IndexedRaggedTs format.
23
24	Parameters
25	----------
26	img_path: str
27	Local (root) directory where the annual folder containing SMOS L2 SM
28	swath data are found.
29	ts_path: str
30	Local directory where the converted time series data will be stored.
31	startdate: str or datetime, optional (default: None)
32	First day of the available swath data that should be included in the
33	time series. If None is passed, then the first available day is used.
34	enddate: str or datetime, optional (default: None)
35	Last day of the available swath data that should be included in the
36	time series. If None is passed, then the last available day is used.
37	memory : float, optional (default: 4)
38	Size of available memory in GB. More memory will lead to a faster
39	conversion.
40	"""
41	reader = SMOSL2Reader(img_path)	3✔
42
43	first_day, last_day = get_avail_img_range(img_path)	3✔
44
45	start = pd.to_datetime(startdate).to_pydatetime() if startdate is not None else first_day	3✔
46	end = pd.to_datetime(enddate).to_pydatetime() if enddate is not None else last_day	3✔
47
48	out_file = os.path.join(ts_path, f"overview.yml")	3✔
49
50	if os.path.isfile(out_file):	3!
NEW 51	props = read_summary_yml(ts_path)	×
NEW 52	if start < pd.to_datetime(props['enddate']).to_pydatetime():	×
NEW 53	raise ValueError("Cannot prepend data to time series, or replace "	×
54	"existing values. Choose different start date.")
55
56	props = {'enddate': str(end), 'last_update': str(datetime.now()),	3✔
57	'parameters': [str(v) for v in reader.varnames]}
58
59	r = reader.repurpose(	3✔
60	outpath=ts_path,
61	start=start,
62	end=end,
63	memory=memory,
64	overwrite=False,
65	imgbaseconnection=True,
66	)
67
68	if r is not None:	3!
69	with open(out_file, 'w') as f:	3✔
70	yaml.dump(props, f, default_flow_style=False, sort_keys=False)	3✔
71
72	def extend_ts(img_path, ts_path, memory=4):	3✔
73	"""
74	Append new image data to an existing time series record.
75	This will use the enddate from summary.yml in the time series
76	directory to decide which date the update should start from and
77	the available image directories to decide how many images can be
78	appended.
79
80	Parameters
81	----------
82	img_path: str
83	Path where the annual folders containing downloaded SMOS L2 images
84	are stored
85	ts_path: str
86	Path where the converted time series (initially created using the
87	reshuffle / swath2ts command) are stored.
88	memory: int, optional (default: 4)
89	Available memory in GB
90	"""
91	out_file = os.path.join(ts_path, f"overview.yml")	3✔
92	if not os.path.isfile(out_file):	3!
NEW 93	raise ValueError("No overview.yml found in the time series directory."	×
94	"Please use reshuffle / swath2ts for initial time "
95	f"series setup or provide overview.yml in {ts_path}.")
96
97	props = read_summary_yml(ts_path)	3✔
98	startdate = pd.to_datetime(props['enddate']).to_pydatetime()	3✔
99	_, enddate = get_avail_img_range(img_path)	3✔
100
101	reader = SMOSL2Reader(img_path)	3✔
102
103	print(f"From: {startdate}, To: {enddate}")	3✔
104
105	r = reader.repurpose(	3✔
106	outpath=ts_path,
107	start=startdate,
108	end=enddate,
109	memory=memory,
110	imgbaseconnection=True,
111	overwrite=False,
112	append=True,
113	)
114
115	if r is not None:	3!
116	props['enddate'] = str(enddate)	3✔
117	props['last_update'] = str(datetime.now())	3✔
118
119	with open(out_file, 'w') as f:	3✔
120	yaml.dump(props, f, default_flow_style=False, sort_keys=False)	3✔

TUW-GEO / smos / 11265563327

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous