11251040335

Committed 09 Oct 2024 08:21AM UTC coverage: 78.564% (-12.1%) from 90.69%

Build # 11251040335

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #15 from wpreimes/smos_l2

Smos l2

Run Details

178 of 280 branches covered (63.57%)

Branch coverage included in aggregate %.

160 of 236 new or added lines in 4 files covered. (67.8%)

1 existing line in 1 file now uncovered.

577 of 681 relevant lines covered (84.73%)

2.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

43.69

/src/smos/smos_l2/download.py

"""
Module to synchronize SMOS L2 data from FTP to local disk
"""
import os
from pathlib import Path
import subprocess
from datetime import datetime
from tqdm import tqdm
import pandas as pd
from calendar import monthrange
from pathlib import PurePosixPath
from glob import glob

L2_START_DATE = datetime(2010, 6, 1)

def load_dotrc(path=None) -> dict:
    """
    Read FTP login credentials from .smosrc file.

    Parameters
    ----------
    path: str, optional (default: None)
        Path to the dotrc file. None will look in the default branch
        with is the home folder.

    Returns
    -------
    config: dict
        Elements from the dotrc file
    """
    if path is None:
        path = os.path.join(str(Path.home()), '.smosapirc')
    if not os.path.exists(path):
        raise ValueError(f'.smosapirc file not found at {path}. '
                         f'Create an account at https://eoiam-idp.eo.esa.int')
    config = {}
    with open(path) as f:
        for line in f.readlines():
            if ":" in line:
                k, v = line.strip().split(":", 1)
                if k in ("disseo_username", "disseo_password"):
                    config[k] = v.strip()
    return config

def get_avail_img_range(path) -> (datetime, datetime):
    """
    Derive first and last day (available folder) with data from the
    local SMOS L2 data
    Folder structure: $PATH/YEAR/MONTH/DAY/*.nc

    Parameters
    ----------
    path: str
        Local root path (contains annual folders)

    Returns
    -------
    first_day: datetime
        First day for which image data is available.
    last_day: datetime
        Last day for which data is available.
    """
    years = glob(os.path.join(path, '[0-9][0-9][0-9][0-9]'))
    years = [int(os.path.basename(y)) for y in years]
    years.sort()
    if len(years) == 0:
        raise ValueError(f"No SMOS L2 data found in {path}.")

    first_year = years[0]
    last_year = years[-1]

    months = glob(os.path.join(path, str(last_year), '[0-9][0-9]'))
    months = [int(os.path.basename(m)) for m in months]
    months.sort()
    if len(years) == 0:
        raise ValueError(f"No SMOS L2 data found in {path}.")
    first_month = months[0]
    last_month = months[-1]

    days = glob(os.path.join(path, str(last_year), f"{last_month:02}", '[0-9][0-9]'))
    days = [int(os.path.basename(d)) for d in days]
    days.sort()
    if len(years) == 0:
        raise ValueError(f"No SMOS L2 data found in {path}.")
    first_day = days[0]
    last_day = days[-1]

    return (datetime(int(first_year), first_month, first_day),
            datetime(int(last_year), last_month, last_day))


class SmosDissEoFtp:
    def __init__(self, local_root, username=None, password=None, dotrc=None,
                 skip_lftp_verify=False):
        """
        Access to SMOS L2 data from FTP.

        Parameters
        ----------
        local_root: str
            Local root folder where the data from the FTP server is transferred
            into.
        username: str, optional (default: None)
            Username of your EO Sign In account. If None is passed here
            it will be derived from the .smosapirc file in the home directory.
            Create an account at https://eoiam-idp.eo.esa.int
        password: str, optional (default: None)
            Password for the EO Sign in account. If None is passed here
            it will be derived from the .smosapirc file in the home directory.
            Create an account at https://eoiam-idp.eo.esa.int
        dotrc: str, optional (default: None)
            Path to the .smosapirc file containing the FTP username and password.
            If None, then the file is assumed to be at $HOME/.smosapirc
                disseo_username: xxxx
                disseo_password: xxxx
            Create an account at https://eoiam-idp.eo.esa.int
        skip_lftp_verify: bool, optional (default: False)
            Skip checking if lftp is available (for testing).
        """
        self.host = "ftps://smos-diss.eo.esa.int"
        self.ftp_root = PurePosixPath("/", "SMOS", "L2SM", "MIR_SMUDP2_nc")

        self.username = username
        self.password = password

        self.local_root = Path(local_root)
        os.makedirs(self.local_root, exist_ok=True)

        if self.username is None or self.password is None:
            config = load_dotrc(dotrc)
            if self.username is None:
                self.username = config['disseo_username']
            if self.password is None:
                self.password = config['disseo_password']

        if not skip_lftp_verify:
            self.verify_lftp_installed()

    def verify_lftp_installed(self):
        """
        Call lftp command to check if program is available.
        Otherwise it has to be installed e.g. via apt-get install.
        """
        r = subprocess.run(["lftp", "--version"])

        if r.returncode != 0:
            raise ValueError("lftp command is not available. "
                             "Please install lftp: https://lftp.yar.ru/")

    def exec(self, cmd):
        cmd = [
            "lftp", "-c",
            f"open {self.host} && set ssl:verify-certificate no && "
            f"user {self.username} {self.password} && "
            f"{cmd} && "
            f"quit"
        ]

        r = subprocess.run(cmd, capture_output=True)

        return r

    def list(self, subpath='', filter='all'):
        """
        Create a list of all files and subdirectories under the passed
        path on the server.
        Directories end with /, files should have a file extension.

        Parameters
        ----------
        subpath: str, optional (default: '')
            Subdirectory on the server to look into.
            e.g. '/2020/01/01'
        filter: str, optional (default: 'all')
            - all: returns fils and folders
            - file: returns only files
            - dir: returns only directories

        Returns
        -------
        elements: list
            List of all files and/or folders under the subpath on the server
        """
        path = self.ftp_root
        if subpath not in [None, '']:
            path += '/' + str(subpath)
        cmd = f"cls {path}"
        r = self.exec(cmd)
        lst = r.stdout.decode("utf-8").splitlines()

        data = []
        for l in lst:
            d = l.split('/')[-1]
            if d == '':
                d = l.split('/')[-2]+'/'
            if d.endswith('/') and (filter in ['dir', 'all']):
                data.append(d)
            if not d.endswith('/') and (filter in ['file', 'all']):
                data.append(d)

        return data

    def list_all_available_days(self, date_from=L2_START_DATE,
                                date_to=datetime.now(), progressbar=True):
        """
        Shortcut to get a list of all available days (i.e. folders) on the
        server within the selected time frame.

        Parameters
        ----------
        date_from: str or datetime, optional
            First date of the time frame to check available days for on server.
            By default, we use the first date of SMOS L2 (2010-06-01)
        date_to: str or datetime, optional
            Last date of the time frame to check available days for on server.
            By default, we use the current date.
        progressbar: bool, optional (default: True)
            This operation will send some request to the server and may take
            some time. (De)activate a visual progress representation.

        Returns
        -------
        dates: list
            List of dates for which a folder exists on the server
        """
        date_to = pd.to_datetime(date_to).to_pydatetime()
        date_from = pd.to_datetime(date_from).to_pydatetime()

        dates = []
        years = [int(y.replace('/', '')) for y in self.list(filter='dir')]
        years = [y for y in years if ((y >= date_from.year) and (y <= date_to.year))]

        for year in tqdm(years, disable=not progressbar):
            months = [int(m.replace('/', '')) for m in self.list(subpath=str(year), filter='dir')]
            if year == date_from.year:
                months = [m for m in months if m >= date_from.month]
            if year == date_to.year:
                months = [m for m in months if m <= date_to.month]
            for month in months:
                days = self.list(subpath=f"{year}/{month:02}", filter='dir')
                for day in days:
                    dt = datetime(int(year),
                                  int(month),
                                  int(day.replace('/', '')))
                    if date_from <= dt <= date_to:
                        dates.append(dt)

        return dates

    def sync(self, year, month, day=None, opts='', dry_run=False):
        """
        Download data from remote to local folder for a certain day.

        Parameters
        ----------
        year: int
            Year part of the date to download
        month: int
            Month part of the date to download
        day: int, optional
            Day part of the date to download. If not set, then
            the whole month is synced.
        opts: str, optional, default: ''
            Additional options that are added to the command
            mirror OPTS root_dir target_dir
            For all options see https://lftp.yar.ru/lftp-man.html
        dry_run: bool, optional, default=False
            Dry run does not actually download anything.
            Instead of the return value, the full command is returned

        Returns
        -------
        ret: str
            Return value or command (if dry_run)
        """
        _d = datetime(year, month, day if day is not None else monthrange(year, month)[1])
        if _d < L2_START_DATE:
            raise ValueError(f"Chosen date must be after {L2_START_DATE}")

        subpath = Path(str(year), f"{month:02}")

        if day is not None:
            subpath = subpath / f"{day:02}"

        target_path = self.local_root / subpath

        cmd = ["mirror -c"]
        if len(opts) > 0:
            cmd.append(opts)
        cmd.append(str(self.ftp_root / PurePosixPath(subpath)))
        cmd.append(f"{target_path}")
        cmd.append('−−no−perms')
        cmd = ' '.join(cmd)

        if dry_run:
            return cmd
        else:
            return self.exec(cmd)

    def sync_period(self, startdate, enddate, dry_run=False):
        """
        Synchronize SMOS L2 data between local root and FTP folder for days
        in the passed time frame.

        Parameters
        ----------
        startdate: str or datetime
            First day to download data for (if available)
        enddate: str or datetime
            Last day to download data for (if available)

        Returns:
        -------
        ret: list
            List of return values or commands (if dry_run was chosen)
        """
        startdate = pd.to_datetime(startdate)
        enddate = pd.to_datetime(enddate)

        df = pd.Series(index=pd.date_range(startdate, enddate, freq='D'),
                       data=1)

        ret = []

        for year, ys in df.groupby(df.index.year):
            for month, ms in ys.groupby(df.index.month):
                if len(ms) == monthrange(year, month)[1]:  # complete month (fast)
                    r = self.sync(int(year), int(month), day=None, dry_run=dry_run)
                    ret.append(r)
                else:  # individual days (slow)
                    for dt in ms.index.values:
                        dt = pd.Timestamp(dt).to_pydatetime()
                        r = self.sync(dt.year, dt.month, dt.day, dry_run=dry_run)
                        ret.append(r)

        return ret

1	"""
2	Module to synchronize SMOS L2 data from FTP to local disk
3	"""
4	import os	3✔
5	from pathlib import Path	3✔
6	import subprocess	3✔
7	from datetime import datetime	3✔
8	from tqdm import tqdm	3✔
9	import pandas as pd	3✔
10	from calendar import monthrange	3✔
11	from pathlib import PurePosixPath	3✔
12	from glob import glob	3✔
13
14	L2_START_DATE = datetime(2010, 6, 1)	3✔
15
16	def load_dotrc(path=None) -> dict:	3✔
17	"""
18	Read FTP login credentials from .smosrc file.
19
20	Parameters
21	----------
22	path: str, optional (default: None)
23	Path to the dotrc file. None will look in the default branch
24	with is the home folder.
25
26	Returns
27	-------
28	config: dict
29	Elements from the dotrc file
30	"""
NEW 31	if path is None:	×
NEW 32	path = os.path.join(str(Path.home()), '.smosapirc')	×
NEW 33	if not os.path.exists(path):	×
NEW 34	raise ValueError(f'.smosapirc file not found at {path}. '	×
35	f'Create an account at https://eoiam-idp.eo.esa.int')
NEW 36	config = {}	×
NEW 37	with open(path) as f:	×
NEW 38	for line in f.readlines():	×
NEW 39	if ":" in line:	×
NEW 40	k, v = line.strip().split(":", 1)	×
NEW 41	if k in ("disseo_username", "disseo_password"):	×
NEW 42	config[k] = v.strip()	×
NEW 43	return config	×
44
45	def get_avail_img_range(path) -> (datetime, datetime):	3✔
46	"""
47	Derive first and last day (available folder) with data from the
48	local SMOS L2 data
49	Folder structure: $PATH/YEAR/MONTH/DAY/*.nc
50
51	Parameters
52	----------
53	path: str
54	Local root path (contains annual folders)
55
56	Returns
57	-------
58	first_day: datetime
59	First day for which image data is available.
60	last_day: datetime
61	Last day for which data is available.
62	"""
63	years = glob(os.path.join(path, '[0-9][0-9][0-9][0-9]'))	3✔
64	years = [int(os.path.basename(y)) for y in years]	3✔
65	years.sort()	3✔
66	if len(years) == 0:	3!
NEW 67	raise ValueError(f"No SMOS L2 data found in {path}.")	×
68
69	first_year = years[0]	3✔
70	last_year = years[-1]	3✔
71
72	months = glob(os.path.join(path, str(last_year), '[0-9][0-9]'))	3✔
73	months = [int(os.path.basename(m)) for m in months]	3✔
74	months.sort()	3✔
75	if len(years) == 0:	3!
NEW 76	raise ValueError(f"No SMOS L2 data found in {path}.")	×
77	first_month = months[0]	3✔
78	last_month = months[-1]	3✔
79
80	days = glob(os.path.join(path, str(last_year), f"{last_month:02}", '[0-9][0-9]'))	3✔
81	days = [int(os.path.basename(d)) for d in days]	3✔
82	days.sort()	3✔
83	if len(years) == 0:	3!
NEW 84	raise ValueError(f"No SMOS L2 data found in {path}.")	×
85	first_day = days[0]	3✔
86	last_day = days[-1]	3✔
87
88	return (datetime(int(first_year), first_month, first_day),	3✔
89	datetime(int(last_year), last_month, last_day))
90
91
92	class SmosDissEoFtp:	3✔
93	def __init__(self, local_root, username=None, password=None, dotrc=None,	3✔
94	skip_lftp_verify=False):
95	"""
96	Access to SMOS L2 data from FTP.
97
98	Parameters
99	----------
100	local_root: str
101	Local root folder where the data from the FTP server is transferred
102	into.
103	username: str, optional (default: None)
104	Username of your EO Sign In account. If None is passed here
105	it will be derived from the .smosapirc file in the home directory.
106	Create an account at https://eoiam-idp.eo.esa.int
107	password: str, optional (default: None)
108	Password for the EO Sign in account. If None is passed here
109	it will be derived from the .smosapirc file in the home directory.
110	Create an account at https://eoiam-idp.eo.esa.int
111	dotrc: str, optional (default: None)
112	Path to the .smosapirc file containing the FTP username and password.
113	If None, then the file is assumed to be at $HOME/.smosapirc
114	disseo_username: xxxx
115	disseo_password: xxxx
116	Create an account at https://eoiam-idp.eo.esa.int
117	skip_lftp_verify: bool, optional (default: False)
118	Skip checking if lftp is available (for testing).
119	"""
120	self.host = "ftps://smos-diss.eo.esa.int"	3✔
121	self.ftp_root = PurePosixPath("/", "SMOS", "L2SM", "MIR_SMUDP2_nc")	3✔
122
123	self.username = username	3✔
124	self.password = password	3✔
125
126	self.local_root = Path(local_root)	3✔
127	os.makedirs(self.local_root, exist_ok=True)	3✔
128
129	if self.username is None or self.password is None:	3!
NEW 130	config = load_dotrc(dotrc)	×
NEW 131	if self.username is None:	×
NEW 132	self.username = config['disseo_username']	×
NEW 133	if self.password is None:	×
NEW 134	self.password = config['disseo_password']	×
135
136	if not skip_lftp_verify:	3!
NEW 137	self.verify_lftp_installed()	×
138
139	def verify_lftp_installed(self):	3✔
140	"""
141	Call lftp command to check if program is available.
142	Otherwise it has to be installed e.g. via apt-get install.
143	"""
NEW 144	r = subprocess.run(["lftp", "--version"])	×
145
NEW 146	if r.returncode != 0:	×
NEW 147	raise ValueError("lftp command is not available. "	×
148	"Please install lftp: https://lftp.yar.ru/")
149
150	def exec(self, cmd):	3✔
NEW 151	cmd = [	×
152	"lftp", "-c",
153	f"open {self.host} && set ssl:verify-certificate no && "
154	f"user {self.username} {self.password} && "
155	f"{cmd} && "
156	f"quit"
157	]
158
NEW 159	r = subprocess.run(cmd, capture_output=True)	×
160
NEW 161	return r	×
162
163	def list(self, subpath='', filter='all'):	3✔
164	"""
165	Create a list of all files and subdirectories under the passed
166	path on the server.
167	Directories end with /, files should have a file extension.
168
169	Parameters
170	----------
171	subpath: str, optional (default: '')
172	Subdirectory on the server to look into.
173	e.g. '/2020/01/01'
174	filter: str, optional (default: 'all')
175	- all: returns fils and folders
176	- file: returns only files
177	- dir: returns only directories
178
179	Returns
180	-------
181	elements: list
182	List of all files and/or folders under the subpath on the server
183	"""
NEW 184	path = self.ftp_root	×
NEW 185	if subpath not in [None, '']:	×
NEW 186	path += '/' + str(subpath)	×
NEW 187	cmd = f"cls {path}"	×
NEW 188	r = self.exec(cmd)	×
NEW 189	lst = r.stdout.decode("utf-8").splitlines()	×
190
NEW 191	data = []	×
NEW 192	for l in lst:	×
NEW 193	d = l.split('/')[-1]	×
NEW 194	if d == '':	×
NEW 195	d = l.split('/')[-2]+'/'	×
NEW 196	if d.endswith('/') and (filter in ['dir', 'all']):	×
NEW 197	data.append(d)	×
NEW 198	if not d.endswith('/') and (filter in ['file', 'all']):	×
NEW 199	data.append(d)	×
200
NEW 201	return data	×
202
203	def list_all_available_days(self, date_from=L2_START_DATE,	3✔
204	date_to=datetime.now(), progressbar=True):
205	"""
206	Shortcut to get a list of all available days (i.e. folders) on the
207	server within the selected time frame.
208
209	Parameters
210	----------
211	date_from: str or datetime, optional
212	First date of the time frame to check available days for on server.
213	By default, we use the first date of SMOS L2 (2010-06-01)
214	date_to: str or datetime, optional
215	Last date of the time frame to check available days for on server.
216	By default, we use the current date.
217	progressbar: bool, optional (default: True)
218	This operation will send some request to the server and may take
219	some time. (De)activate a visual progress representation.
220
221	Returns
222	-------
223	dates: list
224	List of dates for which a folder exists on the server
225	"""
NEW 226	date_to = pd.to_datetime(date_to).to_pydatetime()	×
NEW 227	date_from = pd.to_datetime(date_from).to_pydatetime()	×
228
NEW 229	dates = []	×
NEW 230	years = [int(y.replace('/', '')) for y in self.list(filter='dir')]	×
NEW 231	years = [y for y in years if ((y >= date_from.year) and (y <= date_to.year))]	×
232
NEW 233	for year in tqdm(years, disable=not progressbar):	×
NEW 234	months = [int(m.replace('/', '')) for m in self.list(subpath=str(year), filter='dir')]	×
NEW 235	if year == date_from.year:	×
NEW 236	months = [m for m in months if m >= date_from.month]	×
NEW 237	if year == date_to.year:	×
NEW 238	months = [m for m in months if m <= date_to.month]	×
NEW 239	for month in months:	×
NEW 240	days = self.list(subpath=f"{year}/{month:02}", filter='dir')	×
NEW 241	for day in days:	×
NEW 242	dt = datetime(int(year),	×
243	int(month),
244	int(day.replace('/', '')))
NEW 245	if date_from <= dt <= date_to:	×
NEW 246	dates.append(dt)	×
247
NEW 248	return dates	×
249
250	def sync(self, year, month, day=None, opts='', dry_run=False):	3✔
251	"""
252	Download data from remote to local folder for a certain day.
253
254	Parameters
255	----------
256	year: int
257	Year part of the date to download
258	month: int
259	Month part of the date to download
260	day: int, optional
261	Day part of the date to download. If not set, then
262	the whole month is synced.
263	opts: str, optional, default: ''
264	Additional options that are added to the command
265	mirror OPTS root_dir target_dir
266	For all options see https://lftp.yar.ru/lftp-man.html
267	dry_run: bool, optional, default=False
268	Dry run does not actually download anything.
269	Instead of the return value, the full command is returned
270
271	Returns
272	-------
273	ret: str
274	Return value or command (if dry_run)
275	"""
276	_d = datetime(year, month, day if day is not None else monthrange(year, month)[1])	3✔
277	if _d < L2_START_DATE:	3!
NEW 278	raise ValueError(f"Chosen date must be after {L2_START_DATE}")	×
279
280	subpath = Path(str(year), f"{month:02}")	3✔
281
282	if day is not None:	3!
283	subpath = subpath / f"{day:02}"	3✔
284
285	target_path = self.local_root / subpath	3✔
286
287	cmd = ["mirror -c"]	3✔
288	if len(opts) > 0:	3✔
289	cmd.append(opts)	3✔
290	cmd.append(str(self.ftp_root / PurePosixPath(subpath)))	3✔
291	cmd.append(f"{target_path}")	3✔
292	cmd.append('−−no−perms')	3✔
293	cmd = ' '.join(cmd)	3✔
294
295	if dry_run:	3!
296	return cmd	3✔
297	else:
NEW 298	return self.exec(cmd)	×
299
300	def sync_period(self, startdate, enddate, dry_run=False):	3✔
301	"""
302	Synchronize SMOS L2 data between local root and FTP folder for days
303	in the passed time frame.
304
305	Parameters
306	----------
307	startdate: str or datetime
308	First day to download data for (if available)
309	enddate: str or datetime
310	Last day to download data for (if available)
311
312	Returns:
313	-------
314	ret: list
315	List of return values or commands (if dry_run was chosen)
316	"""
317	startdate = pd.to_datetime(startdate)	3✔
318	enddate = pd.to_datetime(enddate)	3✔
319
320	df = pd.Series(index=pd.date_range(startdate, enddate, freq='D'),	3✔
321	data=1)
322
323	ret = []	3✔
324
325	for year, ys in df.groupby(df.index.year):	3✔
326	for month, ms in ys.groupby(df.index.month):	3✔
327	if len(ms) == monthrange(year, month)[1]: # complete month (fast)	3!
NEW 328	r = self.sync(int(year), int(month), day=None, dry_run=dry_run)	×
NEW 329	ret.append(r)	×
330	else: # individual days (slow)
331	for dt in ms.index.values:	3✔
332	dt = pd.Timestamp(dt).to_pydatetime()	3✔
333	r = self.sync(dt.year, dt.month, dt.day, dry_run=dry_run)	3✔
334	ret.append(r)	3✔
335
336	return ret	3✔

TUW-GEO / smos / 11251040335

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous