17838580391

Committed 18 Sep 2025 06:56PM UTC coverage: 89.046% (+0.09%) from 88.959%

Build # 17838580391

Build Type

push

github

Committed by

web-flow

Commit Message

Maintenance upgrades (#1289)

Support Python 3.12 and 3.13, drop testing on Python 3.9 and 3.10.

Upgrade dependencies to latest versions.

Notes testing with coverage sees significant slowdown with Python 3.13.

Coverage Stats

55 of 65 new or added lines in 48 files covered. (84.62%)

2 existing lines in 2 files now uncovered.

27680 of 31085 relevant lines covered (89.05%)

2.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

26.67

/data_toolkit/project/create_sync_gen_input_csvs_common.py

# Copyright 2016-2023 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os.path
import pandas as pd

from db.common_functions import connect_to_database


def create_profile_csvs(
    db_path,
    project,
    profile_scenario_id,
    profile_scenario_name,
    stage_id,
    output_directory,
    overwrite,
    param_name,
    raw_data_table_name,
    raw_data_units_table_name,
    no_hydro_iteration=False,
):
    conn = connect_to_database(db_path=db_path)

    # Get the weighted cap factor for each of the project's constituent units,
    # get the UNION of these tables, and then find the project cap factor
    # with SUM and GROUP BY
    hydro_iter_sql = "" if no_hydro_iteration else "0 AS hydro_iteration,"
    query = f"""
        SELECT year AS weather_iteration, 
        {hydro_iter_sql}
        {stage_id} AS stage_id, 
        hour_of_year as timepoint, sum(weighted_{param_name}) as {param_name}
            FROM (
            SELECT year, month, day_of_month, hour_of_day, unit, 
            project, unit_weight, {param_name}, unit_weight * {param_name} as weighted_{param_name},
                (CAST(
                    strftime('%j',
                        year || '-' || 
                        CASE
                        WHEN month > 9 THEN month
                        ELSE '0' || month END
                        || '-' || 
                        CASE
                        WHEN day_of_month > 9 THEN day_of_month
                        ELSE '0' || day_of_month END
                        ) AS DECIMAL
                    ) - 1) * 24 + hour_of_day AS hour_of_year
            FROM {raw_data_table_name}
            JOIN {raw_data_units_table_name}
            USING (unit)
            WHERE project = '{project}'
            )
        GROUP BY year, hour_of_year
        ORDER BY year, hour_of_year
    """

    # Put into a dataframe and add to file
    df = pd.read_sql(query, con=conn)

    filename = os.path.join(
        output_directory,
        f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
    )

    if overwrite:
        mode = "w"
    else:
        mode = "a"

    write_header = not os.path.exists(filename)

    df.to_csv(
        filename,
        mode=mode,
        header=True if mode == "w" or write_header else False,
        index=False,
    )

    conn.close()

1	# Copyright 2016-2023 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	import os.path	3✔
16	import pandas as pd	3✔
17
18	from db.common_functions import connect_to_database	3✔
19
20
21	def create_profile_csvs(	3✔
22	db_path,
23	project,
24	profile_scenario_id,
25	profile_scenario_name,
26	stage_id,
27	output_directory,
28	overwrite,
29	param_name,
30	raw_data_table_name,
31	raw_data_units_table_name,
32	no_hydro_iteration=False,
33	):
34	conn = connect_to_database(db_path=db_path)	×
35
36	# Get the weighted cap factor for each of the project's constituent units,
37	# get the UNION of these tables, and then find the project cap factor
38	# with SUM and GROUP BY
39	hydro_iter_sql = "" if no_hydro_iteration else "0 AS hydro_iteration,"	×
40	query = f"""	×
41	SELECT year AS weather_iteration,
42	{hydro_iter_sql}
43	{stage_id} AS stage_id,
44	hour_of_year as timepoint, sum(weighted_{param_name}) as {param_name}
45	FROM (
46	SELECT year, month, day_of_month, hour_of_day, unit,
47	project, unit_weight, {param_name}, unit_weight * {param_name} as weighted_{param_name},
48	(CAST(
49	strftime('%j',
50	year \|\| '-' \|\|
51	CASE
52	WHEN month > 9 THEN month
53	ELSE '0' \|\| month END
54	\|\| '-' \|\|
55	CASE
56	WHEN day_of_month > 9 THEN day_of_month
57	ELSE '0' \|\| day_of_month END
58	) AS DECIMAL
59	) - 1) * 24 + hour_of_day AS hour_of_year
60	FROM {raw_data_table_name}
61	JOIN {raw_data_units_table_name}
62	USING (unit)
63	WHERE project = '{project}'
64	)
65	GROUP BY year, hour_of_year
66	ORDER BY year, hour_of_year
67	"""
68
69	# Put into a dataframe and add to file
70	df = pd.read_sql(query, con=conn)	×
71
72	filename = os.path.join(	×
73	output_directory,
74	f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
75	)
76
77	if overwrite:	×
78	mode = "w"	×
79	else:
80	mode = "a"	×
81
82	write_header = not os.path.exists(filename)	×
83
84	df.to_csv(	×
85	filename,
86	mode=mode,
87	header=True if mode == "w" or write_header else False,
88	index=False,
89	)
90
NEW 91	conn.close()	×

blue-marble / gridpath / 17838580391

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous