25081021046

Committed 28 Apr 2026 10:30PM UTC coverage: 88.986% (-0.02%) from 89.008%

Build # 25081021046

Build Type

push

github

Committed by

web-flow

Commit Message

RA Toolkit support for not printing default availability values of 1 (#1365)

Coverage Stats

3 of 11 new or added lines in 5 files covered. (27.27%)

3 existing lines in 3 files now uncovered.

28173 of 31660 relevant lines covered (88.99%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/data_toolkit/project/create_sync_gen_input_csvs_common.py

# Copyright 2016-2023 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os.path
from argparse import Namespace
from multiprocessing import get_context
from sqlite3 import Connection

import pandas as pd

from data_toolkit.project.common_methods import create_iterations_csv
from db.common_functions import connect_to_database


def get_sync_project_pool_and_make_profile_csvs(
    db_path,
    param_name,
    raw_data_table_name,
    raw_data_units_table_name,
    profile_scenario_id,
    profile_scenario_name,
    stage_id,
    output_directory,
    overwrite,
    varies_by_weather,
    varies_by_hydro,
    include_hydro_iteration_column,
    n_parallel_projects,
    print_default_values,
    default_value,
):

    conn = connect_to_database(db_path=db_path)
    c = conn.cursor()

    projects = [
        prj[0]
        for prj in c.execute(
            f"SELECT DISTINCT project FROM {raw_data_units_table_name};"
        ).fetchall()
    ]

    pool_data = tuple(
        [
            [
                db_path,
                prj,
                param_name,
                raw_data_table_name,
                raw_data_units_table_name,
                profile_scenario_id,
                profile_scenario_name,
                stage_id,
                output_directory,
                overwrite,
                varies_by_weather,
                varies_by_hydro,
                include_hydro_iteration_column,
                print_default_values,
                default_value,
            ]
            for prj in projects
        ]
    )

    # Pool must use spawn to work properly on Linux
    pool = get_context("spawn").Pool(int(n_parallel_projects))

    pool.map(create_project_profile_csv_pool, pool_data)
    pool.close()

    conn.close()


def create_project_profile_csv(
    db_path,
    project,
    profile_scenario_id,
    profile_scenario_name,
    stage_id,
    output_directory,
    overwrite,
    param_name,
    raw_data_table_name,
    raw_data_units_table_name,
    varies_by_weather,
    varies_by_hydro,
    include_hydro_iteration_column,
    print_default_values,
    default_value,
):
    conn = connect_to_database(db_path=db_path)

    # Get the weighted value for each of the project's constituent units,
    # get the UNION of these tables, and then find the project value
    # with SUM and GROUP BY
    hydro_iter_sql = "0 AS hydro_iteration," if include_hydro_iteration_column else ""
    query = f"""
        SELECT year AS weather_iteration, 
        {hydro_iter_sql}
        {stage_id} AS stage_id, 
        hour_of_year as timepoint, sum(weighted_{param_name}) as {param_name}
            FROM (
            SELECT year, month, day_of_month, hour_of_day, unit, 
            project, unit_weight, value, unit_weight * value as 
            weighted_{param_name},
                (CAST(
                    strftime('%j',
                        year || '-' || 
                        CASE
                        WHEN month > 9 THEN month
                        ELSE '0' || month END
                        || '-' || 
                        CASE
                        WHEN day_of_month > 9 THEN day_of_month
                        ELSE '0' || day_of_month END
                        ) AS DECIMAL
                    ) - 1) * 24 + hour_of_day AS hour_of_year
            FROM {raw_data_table_name}
            JOIN {raw_data_units_table_name}
            USING (unit)
            WHERE project = '{project}'
            )
        GROUP BY year, hour_of_year
        ORDER BY year, hour_of_year
    """

    # Put into a dataframe and add to file
    df = pd.read_sql(query, con=conn)

    # Filter out rows where the value is at the default, unless
    # print_default_values is True
    if not print_default_values:
        df = df[df[param_name] != default_value]

    filename = os.path.join(
        output_directory,
        f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
    )

    if overwrite:
        mode = "w"
    else:
        mode = "a"

    write_header = not os.path.exists(filename)

    df.to_csv(
        filename,
        mode=mode,
        header=True if mode == "w" or write_header else False,
        index=False,
    )

    # Create iterations CSV
    iterations_directory = os.path.join(output_directory, "iterations")
    os.makedirs(iterations_directory, exist_ok=True)
    create_iterations_csv(
        iterations_directory=iterations_directory,
        project=project,
        profile_id=profile_scenario_id,
        profile_name=profile_scenario_name,
        varies_by_weather=varies_by_weather,
        varies_by_hydro=varies_by_hydro,
        overwrite=overwrite,
    )

    conn.close()


def create_project_profile_csv_pool(pool_datum):
    [
        db_path,
        project,
        param_name,
        raw_data_table_name,
        raw_data_units_table_name,
        profile_scenario_id,
        profile_scenario_name,
        stage_id,
        output_directory,
        overwrite,
        varies_by_weather,
        varies_by_hydro,
        include_hydro_iteration_column,
        print_default_values,
        default_value,
    ] = pool_datum

    create_project_profile_csv(
        db_path=db_path,
        project=project,
        profile_scenario_id=profile_scenario_id,
        profile_scenario_name=profile_scenario_name,
        stage_id=stage_id,
        output_directory=output_directory,
        overwrite=overwrite,
        param_name=param_name,
        raw_data_table_name=raw_data_table_name,
        raw_data_units_table_name=raw_data_units_table_name,
        varies_by_weather=varies_by_weather,
        varies_by_hydro=varies_by_hydro,
        include_hydro_iteration_column=include_hydro_iteration_column,
        print_default_values=print_default_values,
        default_value=default_value,
    )

1	# Copyright 2016-2023 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	import os.path	1✔
16	from argparse import Namespace	1✔
17	from multiprocessing import get_context	1✔
18	from sqlite3 import Connection	1✔
19
20	import pandas as pd	1✔
21
22	from data_toolkit.project.common_methods import create_iterations_csv	1✔
23	from db.common_functions import connect_to_database	1✔
24
25
26	def get_sync_project_pool_and_make_profile_csvs(	1✔
27	db_path,
28	param_name,
29	raw_data_table_name,
30	raw_data_units_table_name,
31	profile_scenario_id,
32	profile_scenario_name,
33	stage_id,
34	output_directory,
35	overwrite,
36	varies_by_weather,
37	varies_by_hydro,
38	include_hydro_iteration_column,
39	n_parallel_projects,
40	print_default_values,
41	default_value,
42	):
43
44	conn = connect_to_database(db_path=db_path)	1✔
45	c = conn.cursor()	1✔
46
47	projects = [	1✔
48	prj[0]
49	for prj in c.execute(
50	f"SELECT DISTINCT project FROM {raw_data_units_table_name};"
51	).fetchall()
52	]
53
54	pool_data = tuple(	1✔
55	[
56	[
57	db_path,
58	prj,
59	param_name,
60	raw_data_table_name,
61	raw_data_units_table_name,
62	profile_scenario_id,
63	profile_scenario_name,
64	stage_id,
65	output_directory,
66	overwrite,
67	varies_by_weather,
68	varies_by_hydro,
69	include_hydro_iteration_column,
70	print_default_values,
71	default_value,
72	]
73	for prj in projects
74	]
75	)
76
77	# Pool must use spawn to work properly on Linux
78	pool = get_context("spawn").Pool(int(n_parallel_projects))	1✔
79
80	pool.map(create_project_profile_csv_pool, pool_data)	1✔
81	pool.close()	1✔
82
83	conn.close()	1✔
84
85
86	def create_project_profile_csv(	1✔
87	db_path,
88	project,
89	profile_scenario_id,
90	profile_scenario_name,
91	stage_id,
92	output_directory,
93	overwrite,
94	param_name,
95	raw_data_table_name,
96	raw_data_units_table_name,
97	varies_by_weather,
98	varies_by_hydro,
99	include_hydro_iteration_column,
100	print_default_values,
101	default_value,
102	):
103	conn = connect_to_database(db_path=db_path)	×
104
105	# Get the weighted value for each of the project's constituent units,
106	# get the UNION of these tables, and then find the project value
107	# with SUM and GROUP BY
108	hydro_iter_sql = "0 AS hydro_iteration," if include_hydro_iteration_column else ""	×
109	query = f"""	×
110	SELECT year AS weather_iteration,
111	{hydro_iter_sql}
112	{stage_id} AS stage_id,
113	hour_of_year as timepoint, sum(weighted_{param_name}) as {param_name}
114	FROM (
115	SELECT year, month, day_of_month, hour_of_day, unit,
116	project, unit_weight, value, unit_weight * value as
117	weighted_{param_name},
118	(CAST(
119	strftime('%j',
120	year \|\| '-' \|\|
121	CASE
122	WHEN month > 9 THEN month
123	ELSE '0' \|\| month END
124	\|\| '-' \|\|
125	CASE
126	WHEN day_of_month > 9 THEN day_of_month
127	ELSE '0' \|\| day_of_month END
128	) AS DECIMAL
129	) - 1) * 24 + hour_of_day AS hour_of_year
130	FROM {raw_data_table_name}
131	JOIN {raw_data_units_table_name}
132	USING (unit)
133	WHERE project = '{project}'
134	)
135	GROUP BY year, hour_of_year
136	ORDER BY year, hour_of_year
137	"""
138
139	# Put into a dataframe and add to file
140	df = pd.read_sql(query, con=conn)	×
141
142	# Filter out rows where the value is at the default, unless
143	# print_default_values is True
NEW 144	if not print_default_values:	×
NEW 145	df = df[df[param_name] != default_value]	×
146
UNCOV 147	filename = os.path.join(	×
148	output_directory,
149	f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
150	)
151
152	if overwrite:	×
153	mode = "w"	×
154	else:
155	mode = "a"	×
156
157	write_header = not os.path.exists(filename)	×
158
159	df.to_csv(	×
160	filename,
161	mode=mode,
162	header=True if mode == "w" or write_header else False,
163	index=False,
164	)
165
166	# Create iterations CSV
167	iterations_directory = os.path.join(output_directory, "iterations")	×
168	os.makedirs(iterations_directory, exist_ok=True)	×
169	create_iterations_csv(	×
170	iterations_directory=iterations_directory,
171	project=project,
172	profile_id=profile_scenario_id,
173	profile_name=profile_scenario_name,
174	varies_by_weather=varies_by_weather,
175	varies_by_hydro=varies_by_hydro,
176	overwrite=overwrite,
177	)
178
179	conn.close()	×
180
181
182	def create_project_profile_csv_pool(pool_datum):	1✔
183	[	×
184	db_path,
185	project,
186	param_name,
187	raw_data_table_name,
188	raw_data_units_table_name,
189	profile_scenario_id,
190	profile_scenario_name,
191	stage_id,
192	output_directory,
193	overwrite,
194	varies_by_weather,
195	varies_by_hydro,
196	include_hydro_iteration_column,
197	print_default_values,
198	default_value,
199	] = pool_datum
200
201	create_project_profile_csv(	×
202	db_path=db_path,
203	project=project,
204	profile_scenario_id=profile_scenario_id,
205	profile_scenario_name=profile_scenario_name,
206	stage_id=stage_id,
207	output_directory=output_directory,
208	overwrite=overwrite,
209	param_name=param_name,
210	raw_data_table_name=raw_data_table_name,
211	raw_data_units_table_name=raw_data_units_table_name,
212	varies_by_weather=varies_by_weather,
213	varies_by_hydro=varies_by_hydro,
214	include_hydro_iteration_column=include_hydro_iteration_column,
215	print_default_values=print_default_values,
216	default_value=default_value,
217	)

blue-marble / gridpath / 25081021046

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous