11562648117

Committed 28 Oct 2024 09:06PM CUT coverage: 86.932% (-1.5%) from 88.385%

Build # 11562648117

Build Type

Pull #1169

github

Committed by

web-flow

Commit Message

Merge 842f984da into 0e3de16ef

Pull Request Pull Request #1169: GridPath Data Toolkit

Run Details

310 of 764 new or added lines in 34 files covered. (40.58%)

17 existing lines in 2 files now uncovered.

22951 of 26401 relevant lines covered (86.93%)

2.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

30.77

/data_toolkit/project/create_sync_gen_input_csvs_common.py

# Copyright 2016-2023 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os.path
import pandas as pd

from db.common_functions import connect_to_database


def create_profile_csvs(
    db_path,
    project,
    profile_scenario_id,
    profile_scenario_name,
    stage_id,
    output_directory,
    overwrite,
    param_name,
    raw_data_table_name,
    raw_data_units_table_name,
):
    conn = connect_to_database(db_path=db_path)

    # Get the weighted cap factor for each of the project's constituent units,
    # get the UNION of these tables, and then find the project cap factor
    # with SUM and GROUP BY
    query = f"""
        SELECT year AS weather_iteration,
        {stage_id} AS stage_id, 
        hour_of_year as timepoint, sum(weighted_{param_name}) as {param_name}
            FROM (
            SELECT year, month, day_of_month, hour_of_day, unit, 
            project, unit_weight, {param_name}, unit_weight * {param_name} as weighted_{param_name},
                (CAST(
                    strftime('%j',
                        year || '-' || 
                        CASE
                        WHEN month > 9 THEN month
                        ELSE '0' || month END
                        || '-' || 
                        CASE
                        WHEN day_of_month > 9 THEN day_of_month
                        ELSE '0' || day_of_month END
                        ) AS DECIMAL
                    ) - 1) * 24 + hour_of_day AS hour_of_year
            FROM {raw_data_table_name}
            JOIN {raw_data_units_table_name}
            USING (unit)
            WHERE project = '{project}'
            )
        GROUP BY year, hour_of_year
        ORDER BY year, hour_of_year
    """

    # Put into a dataframe and add to file
    df = pd.read_sql(query, con=conn)

    filename = os.path.join(
        output_directory,
        f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
    )

    if overwrite:
        mode = "w"
    else:
        mode = "a"

    write_header = not os.path.exists(filename)

    df.to_csv(
        filename,
        mode=mode,
        header=True if mode == "w" or write_header else False,
        index=False,
    )

1	# Copyright 2016-2023 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	import os.path	3✔
16	import pandas as pd	3✔
17
18	from db.common_functions import connect_to_database	3✔
19
20
21	def create_profile_csvs(	3✔
22	db_path,
23	project,
24	profile_scenario_id,
25	profile_scenario_name,
26	stage_id,
27	output_directory,
28	overwrite,
29	param_name,
30	raw_data_table_name,
31	raw_data_units_table_name,
32	):
33	conn = connect_to_database(db_path=db_path)	×
34
35	# Get the weighted cap factor for each of the project's constituent units,
36	# get the UNION of these tables, and then find the project cap factor
37	# with SUM and GROUP BY
38	query = f"""	×
39	SELECT year AS weather_iteration,
40	{stage_id} AS stage_id,
41	hour_of_year as timepoint, sum(weighted_{param_name}) as {param_name}
42	FROM (
43	SELECT year, month, day_of_month, hour_of_day, unit,
44	project, unit_weight, {param_name}, unit_weight * {param_name} as weighted_{param_name},
45	(CAST(
46	strftime('%j',
47	year \|\| '-' \|\|
48	CASE
49	WHEN month > 9 THEN month
50	ELSE '0' \|\| month END
51	\|\| '-' \|\|
52	CASE
53	WHEN day_of_month > 9 THEN day_of_month
54	ELSE '0' \|\| day_of_month END
55	) AS DECIMAL
56	) - 1) * 24 + hour_of_day AS hour_of_year
57	FROM {raw_data_table_name}
58	JOIN {raw_data_units_table_name}
59	USING (unit)
60	WHERE project = '{project}'
61	)
62	GROUP BY year, hour_of_year
63	ORDER BY year, hour_of_year
64	"""
65
66	# Put into a dataframe and add to file
67	df = pd.read_sql(query, con=conn)	×
68
69	filename = os.path.join(	×
70	output_directory,
71	f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
72	)
73
74	if overwrite:	×
75	mode = "w"	×
76	else:
77	mode = "a"	×
78
79	write_header = not os.path.exists(filename)	×
80
81	df.to_csv(	×
82	filename,
83	mode=mode,
84	header=True if mode == "w" or write_header else False,
85	index=False,
86	)

blue-marble / gridpath / 11562648117

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous