8832985991

Committed 25 Apr 2024 12:55PM UTC coverage: 74.229% (-0.7%) from 74.884%

Build # 8832985991

Build Type

Pull #62

github

Committed by

web-flow

Commit Message

Merge 1a622e438 into 311997dcf

Pull Request Pull Request #62: PPS product and level1c collector

Run Details

236 of 403 branches covered (58.56%)

Branch coverage included in aggregate %.

59 of 89 new or added lines in 3 files covered. (66.29%)

15 existing lines in 2 files now uncovered.

1449 of 1867 relevant lines covered (77.61%)

3.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.63

/nwcsafpps_runner/message_utils.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c) 2021 Adam.Dybbroe

# Author(s):

#   Adam.Dybbroe <a000680@c21856.ad.smhi.se>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Message utilities."""

import logging
import os

from posttroll.message import Message
from nwcsafpps_runner.utils import create_pps_file_from_lvl1c


LOG = logging.getLogger(__name__)

 
def remove_non_pps_products(msg_data):
    for ind in range(len(msg_data["collection"])):
        list_of_files_as_dicts = msg_data["collection"][ind]['dataset']
        list_of_files_to_keep = []
        for item in list_of_files_as_dicts:
            if "S_NWC" in item["uid"]:
                list_of_files_to_keep.append(item)
            msg_data["collection"][ind]['dataset'] = list_of_files_to_keep


def get_pps_sensor_from_msg(sensor_msg):
    """ Get pps sensor from msg sensor."""
    sensor = None
    if type(sensor_msg) is list and len(sensor_msg) == 1:
        sensor = sensor_msg[0]
    if sensor is None:  
        for pps_sensor in ['viirs', 'avhrr', 'modis', 'mersi2', 'metimage', 'slstr']:
            if pps_sensor in sensor_msg:
                sensor = pps_sensor
    if "avhrr/3" in sensor_msg:
        sensor = "avhrr"
    return sensor


def add_lvl1c_to_msg(msg_data, options):
    """Add PPS lvl1c file to a collection of PPS products."""
    level1c_path = os.environ.get('SM_IMAGER_DIR', options.get('pps_lvl1c_dir', './'))
    sensor = options.get('sensor', get_pps_sensor_from_msg(msg_data["sensor"]))   
    for ind in range(len(msg_data["collection"])):
        msg_data["collection"][ind]['dataset']
        pps_file = msg_data["collection"][ind]['dataset'][0]["uri"]
        lvl1c_file = create_pps_file_from_lvl1c(pps_file, level1c_path,
                                                name_tag=sensor, file_type='nc')
        msg_data["collection"][ind]['dataset'].append({
            "uri": lvl1c_file,
            "uid": os.path.basename(lvl1c_file)})


def prepare_pps_collector_message(msg, options):
    to_send = msg.data.copy()
    remove_non_pps_products(to_send)
    add_lvl1c_to_msg(to_send, options)
    return to_send
    
def prepare_nwp_message(result_file, publish_topic):
    """Prepare message for NWP files."""
    to_send = {}
    to_send["uri"] = result_file
    filename = os.path.basename(result_file)
    to_send["uid"] = filename
    to_send['format'] = 'NWP grib'
    to_send['type'] = 'grib'
    return to_send


def prepare_l1c_message(result_file, mda, **kwargs):
    """Prepare the output message for the level-1c file creation."""
    if not result_file:
        return

    # Now publish:
    to_send = mda.copy()
    # Delete the input level-1 dataset from the message:
    try:
        del to_send['dataset']
    except KeyError:
        LOG.warning("Couldn't remove dataset from message")
    try:
        del to_send['filename']
    except KeyError:
        LOG.warning("Couldn't remove filename from message")

    if ('orbit' in kwargs) and ('orbit_number' in to_send.keys()):
        to_send["orig_orbit_number"] = to_send["orbit_number"]
        to_send["orbit_number"] = kwargs['orbit']

    to_send["uri"] = result_file
    filename = os.path.basename(result_file)
    to_send["uid"] = filename

    to_send['format'] = 'PPS-L1C'
    to_send['type'] = 'NETCDF'
    to_send['data_processing_level'] = '1c'

    return to_send


def publish_l1c(publisher, publish_msg, publish_topic):
    """Publish the messages that l1c files are ready."""
    LOG.debug('Publish topic = %s', publish_topic)
    for topic in publish_topic:
        msg = Message(topic, "file", publish_msg).encode()
        LOG.debug("sending: %s", str(msg))
        publisher.send(msg)

1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3
4	# Copyright (c) 2021 Adam.Dybbroe
5
6	# Author(s):
7
8	# Adam.Dybbroe <a000680@c21856.ad.smhi.se>
9
10	# This program is free software: you can redistribute it and/or modify
11	# it under the terms of the GNU General Public License as published by
12	# the Free Software Foundation, either version 3 of the License, or
13	# (at your option) any later version.
14
15	# This program is distributed in the hope that it will be useful,
16	# but WITHOUT ANY WARRANTY; without even the implied warranty of
17	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18	# GNU General Public License for more details.
19
20	# You should have received a copy of the GNU General Public License
21	# along with this program. If not, see <http://www.gnu.org/licenses/>.
22
23	"""Message utilities."""	4✔
24
25	import logging	4✔
26	import os	4✔
27
28	from posttroll.message import Message	4✔
29	from nwcsafpps_runner.utils import create_pps_file_from_lvl1c	4✔
30
31
32	LOG = logging.getLogger(__name__)	4✔
33
34
35	def remove_non_pps_products(msg_data):	4✔
36	for ind in range(len(msg_data["collection"])):	4✔
37	list_of_files_as_dicts = msg_data["collection"][ind]['dataset']	4✔
38	list_of_files_to_keep = []	4✔
39	for item in list_of_files_as_dicts:	4✔
40	if "S_NWC" in item["uid"]:	4✔
41	list_of_files_to_keep.append(item)	4✔
42	msg_data["collection"][ind]['dataset'] = list_of_files_to_keep	4✔
43
44
45	def get_pps_sensor_from_msg(sensor_msg):	4✔
46	""" Get pps sensor from msg sensor."""
47	sensor = None	4✔
48	if type(sensor_msg) is list and len(sensor_msg) == 1:	4!
49	sensor = sensor_msg[0]	4✔
50	if sensor is None:	4!
NEW 51	for pps_sensor in ['viirs', 'avhrr', 'modis', 'mersi2', 'metimage', 'slstr']:	×
NEW 52	if pps_sensor in sensor_msg:	×
NEW 53	sensor = pps_sensor	×
54	if "avhrr/3" in sensor_msg:	4!
NEW 55	sensor = "avhrr"	×
56	return sensor	4✔
57
58
59	def add_lvl1c_to_msg(msg_data, options):	4✔
60	"""Add PPS lvl1c file to a collection of PPS products."""
61	level1c_path = os.environ.get('SM_IMAGER_DIR', options.get('pps_lvl1c_dir', './'))	4✔
62	sensor = options.get('sensor', get_pps_sensor_from_msg(msg_data["sensor"]))	4✔
63	for ind in range(len(msg_data["collection"])):	4✔
64	msg_data["collection"][ind]['dataset']	4✔
65	pps_file = msg_data["collection"][ind]['dataset'][0]["uri"]	4✔
66	lvl1c_file = create_pps_file_from_lvl1c(pps_file, level1c_path,	4✔
67	name_tag=sensor, file_type='nc')
68	msg_data["collection"][ind]['dataset'].append({	4✔
69	"uri": lvl1c_file,
70	"uid": os.path.basename(lvl1c_file)})
71
72
73	def prepare_pps_collector_message(msg, options):	4✔
74	to_send = msg.data.copy()	4✔
75	remove_non_pps_products(to_send)	4✔
76	add_lvl1c_to_msg(to_send, options)	4✔
77	return to_send	4✔
78
79	def prepare_nwp_message(result_file, publish_topic):	4✔
80	"""Prepare message for NWP files."""
81	to_send = {}	4✔
82	to_send["uri"] = result_file	4✔
83	filename = os.path.basename(result_file)	4✔
84	to_send["uid"] = filename	4✔
85	to_send['format'] = 'NWP grib'	4✔
86	to_send['type'] = 'grib'	4✔
87	return to_send	4✔
88
89
90	def prepare_l1c_message(result_file, mda, **kwargs):	4✔
91	"""Prepare the output message for the level-1c file creation."""
92	if not result_file:	4!
UNCOV 93	return	×
94
95	# Now publish:
96	to_send = mda.copy()	4✔
97	# Delete the input level-1 dataset from the message:
98	try:	4✔
99	del to_send['dataset']	4✔
UNCOV 100	except KeyError:	×
UNCOV 101	LOG.warning("Couldn't remove dataset from message")	×
102	try:	4✔
103	del to_send['filename']	4✔
104	except KeyError:	4✔
105	LOG.warning("Couldn't remove filename from message")	4✔
106
107	if ('orbit' in kwargs) and ('orbit_number' in to_send.keys()):	4!
UNCOV 108	to_send["orig_orbit_number"] = to_send["orbit_number"]	×
109	to_send["orbit_number"] = kwargs['orbit']	×
110
111	to_send["uri"] = result_file	4✔
112	filename = os.path.basename(result_file)	4✔
113	to_send["uid"] = filename	4✔
114
115	to_send['format'] = 'PPS-L1C'	4✔
116	to_send['type'] = 'NETCDF'	4✔
117	to_send['data_processing_level'] = '1c'	4✔
118
119	return to_send	4✔
120
121
122	def publish_l1c(publisher, publish_msg, publish_topic):	4✔
123	"""Publish the messages that l1c files are ready."""
124	LOG.debug('Publish topic = %s', publish_topic)	4✔
125	for topic in publish_topic:	4✔
126	msg = Message(topic, "file", publish_msg).encode()	4✔
127	LOG.debug("sending: %s", str(msg))	4✔
128	publisher.send(msg)	4✔

pytroll / pytroll-pps-runner / 8832985991

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous