17811364834

Committed 17 Sep 2025 09:37PM UTC coverage: 89.049% (+0.09%) from 88.959%

Build # 17811364834

Build Type

Pull #1289

github

Committed by

web-flow

Commit Message

Merge 1e7c06b61 into bdb0a2ac5

Pull Request Pull Request #1289: Maintenance dependency upgrades

Coverage Stats

11 of 16 new or added lines in 7 files covered. (68.75%)

141 existing lines in 44 files now uncovered.

27640 of 31039 relevant lines covered (89.05%)

2.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.19

/data_toolkit/system/create_sync_load_input_csvs.py

# Copyright 2016-2025 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Sync Loads
**********

Create GridPath sync load profile inputs.

=====
Usage
=====

>>> gridpath_run_data_toolkit --single_step create_sync_load_input_csvs --settings_csv PATH/TO/SETTINGS/CSV

===================
Input prerequisites
===================

This module assumes the following raw input database tables have been populated:
    * raw_data_system_load
    * user_defined_load_zone_units

=========
Settings
=========
    * database
    * output_directory
    * load_scenario_id
    * load_scenario_name
    * overwrite

"""

import sys
from argparse import ArgumentParser
import os.path
import pandas as pd

from data_toolkit.system.common_methods import (
    create_load_scenario_csv,
    create_load_components_scenario_csv,
)
from db.common_functions import connect_to_database

LOAD_SCENARIO_ID_DEFAULT = 1  # it's 6 in the test examples
LOAD_SCENARIO_NAME_DEFAULT = "ra_toolkit"
LOAD_COMPONENTS_SCENARIO_ID_DEFAULT = 1  # it's 6 in the test examples
LOAD_COMPONENTS_SCENARIO_NAME_DEFAULT = "ra_toolkit"
LOAD_LEVELS_SCENARIO_ID_DEFAULT = 1  # it's 6 in the test examples
LOAD_LEVELS_SCENARIO_NAME_DEFAULT = "ra_toolkit"
STAGE_ID_DEFAULT = 1
LOAD_COMPONENT_NAME_DEFAULT = "all"


def parse_arguments(args):
    """
    :param args: the script arguments specified by the user
    :return: the parsed known argument values (<class 'argparse.Namespace'>
    Python object)

    Parse the known arguments.
    """
    parser = ArgumentParser(add_help=True)

    parser.add_argument("-db", "--database")

    parser.add_argument(
        "-out_dir",
        "--output_directory",
        help="""This will be the location of the load_scenario_id file. The 
        load components and load levels files are assumed to be in the 
        'load_components' and 'load_levels' subdirectories respectively.""",
    )
    parser.add_argument(
        "-id",
        "--load_scenario_id",
        default=LOAD_SCENARIO_ID_DEFAULT,
        help=f"Defaults to {LOAD_SCENARIO_ID_DEFAULT}.",
    )
    parser.add_argument(
        "-name",
        "--load_scenario_name",
        default=LOAD_SCENARIO_NAME_DEFAULT,
        help=f"Defaults to '{LOAD_SCENARIO_NAME_DEFAULT}'.",
    )
    parser.add_argument(
        "-lc_id",
        "--load_components_scenario_id",
        default=LOAD_COMPONENTS_SCENARIO_ID_DEFAULT,
        help=f"Defaults to {LOAD_COMPONENTS_SCENARIO_ID_DEFAULT}.",
    )
    parser.add_argument(
        "-lc_name",
        "--load_components_scenario_name",
        default=LOAD_COMPONENTS_SCENARIO_NAME_DEFAULT,
        help=f"Defaults to '{LOAD_COMPONENTS_SCENARIO_NAME_DEFAULT}'.",
    )
    parser.add_argument(
        "-ll_id",
        "--load_levels_scenario_id",
        default=LOAD_LEVELS_SCENARIO_ID_DEFAULT,
        help=f"Defaults to {LOAD_LEVELS_SCENARIO_ID_DEFAULT}.",
    )
    parser.add_argument(
        "-ll_name",
        "--load_levels_scenario_name",
        default=LOAD_LEVELS_SCENARIO_NAME_DEFAULT,
        help=f"Defaults to '{LOAD_LEVELS_SCENARIO_NAME_DEFAULT}'.",
    )

    parser.add_argument(
        "-stage",
        "--stage_id",
        default=STAGE_ID_DEFAULT,
        help=f"Defaults to '{STAGE_ID_DEFAULT}",
    )

    parser.add_argument(
        "-comp",
        "--load_component",
        default=LOAD_COMPONENT_NAME_DEFAULT,
        help=f"Defaults to '{LOAD_COMPONENT_NAME_DEFAULT}",
    )

    parser.add_argument(
        "-l_o",
        "--load_scenario_overwrite",
        default=False,
        action="store_true",
        help="Overwrite existing CSV files.",
    )
    parser.add_argument(
        "-lc_o",
        "--load_components_overwrite",
        default=False,
        action="store_true",
        help="Overwrite existing CSV files.",
    )
    parser.add_argument(
        "-ll_o",
        "--load_levels_overwrite",
        default=False,
        action="store_true",
        help="Overwrite existing CSV files.",
    )

    parser.add_argument(
        "-skip_l",
        "--skip_load_scenario",
        default=False,
        action="store_true",
        help="Don't create load_scenario file.",
    )
    parser.add_argument(
        "-skip_lc",
        "--skip_load_components",
        default=False,
        action="store_true",
        help="Don't create load components file.",
    )
    parser.add_argument(
        "-skip_ll",
        "--skip_load_levels",
        default=False,
        action="store_true",
        help="Don't create load levels file.",
    )

    parser.add_argument("-q", "--quiet", default=False, action="store_true")

    parsed_arguments = parser.parse_known_args(args=args)[0]

    return parsed_arguments


def create_load_levels_csv(
    conn,
    output_directory,
    load_levels_scenario_id,
    load_levels_scenario_name,
    stage_id,
    load_component_name,
    overwrite,
):
    """
    This module currently assumes timepoint IDs will be 1 through 8760 for
    each year. The query will aggregate loads based on the aggregations and
    weights defined in the user_defined_load_zone_units
    table.
    """

    query = f"""
        SELECT load_zone, year AS weather_iteration, {stage_id} as stage_id, 
        hour_of_year as timepoint, 
        '{load_component_name}' AS load_component, sum(weighted_load_mw) as 
        load_mw
        FROM (
        SELECT year, month, day_of_month, hour_of_day, load_zone_unit, load_zone, unit_weight, load_mw, unit_weight * load_mw as weighted_load_mw,
            (CAST(
                strftime('%j',
                    year || '-' || 
                    CASE
                    WHEN month > 9 THEN month
                    ELSE '0' || month END
                    || '-' || 
                    CASE
                    WHEN day_of_month > 9 THEN day_of_month
                    ELSE '0' || day_of_month END
                    ) AS DECIMAL
                ) - 1) * 24 + hour_of_day AS hour_of_year
        FROM raw_data_system_load
        JOIN user_defined_load_zone_units
        USING (load_zone_unit)
        )
    GROUP BY load_zone, year, hour_of_year
    """

    # Put into a dataframe and add to file
    df = pd.read_sql(query, con=conn)

    filename = os.path.join(
        output_directory,
        "load_levels",
        f"{load_levels_scenario_id}_{load_levels_scenario_name}.csv",
    )
    if overwrite:
        mode = "w"
        write_header = True
    else:
        mode = "a"
        write_header = not os.path.exists(filename)

    df.to_csv(
        filename,
        mode=mode,
        header=write_header,
        index=False,
    )


def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parsed_args = parse_arguments(args=args)

    if not parsed_args.quiet:
        print("Creating sync load profile CSVs...")

    os.makedirs(parsed_args.output_directory, exist_ok=True)
    os.makedirs(
        os.path.join(parsed_args.output_directory, "load_components"), exist_ok=True
    )
    os.makedirs(
        os.path.join(parsed_args.output_directory, "load_levels"), exist_ok=True
    )

    conn = connect_to_database(db_path=parsed_args.database)

    if not parsed_args.skip_load_scenario:
        create_load_scenario_csv(
            output_directory=parsed_args.output_directory,
            load_scenario_id=parsed_args.load_scenario_id,
            load_scenario_name=parsed_args.load_scenario_name,
            load_components_scenario_id=parsed_args.load_components_scenario_id,
            load_levels_scenario_id=parsed_args.load_levels_scenario_id,
            overwrite_load_scenario_csv=parsed_args.load_scenario_overwrite,
        )

    if not parsed_args.skip_load_components:
        create_load_components_scenario_csv(
            conn=conn,
            output_directory=parsed_args.output_directory,
            load_component_name=parsed_args.load_component,
            load_components_scenario_id=parsed_args.load_components_scenario_id,
            load_components_scenario_name=parsed_args.load_components_scenario_name,
            overwrite_load_components_csv=parsed_args.load_components_overwrite,
        )

    if not parsed_args.skip_load_levels:
        create_load_levels_csv(
            conn=conn,
            output_directory=parsed_args.output_directory,
            load_levels_scenario_id=parsed_args.load_levels_scenario_id,
            load_levels_scenario_name=parsed_args.load_levels_scenario_name,
            stage_id=parsed_args.stage_id,
            load_component_name=parsed_args.load_component,
            overwrite=parsed_args.load_levels_overwrite,
        )


if __name__ == "__main__":
    main()

1	# Copyright 2016-2025 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	"""	1✔
16	Sync Loads
17	**********
18
19	Create GridPath sync load profile inputs.
20
21	=====
22	Usage
23	=====
24
25	>>> gridpath_run_data_toolkit --single_step create_sync_load_input_csvs --settings_csv PATH/TO/SETTINGS/CSV
26
27	===================
28	Input prerequisites
29	===================
30
31	This module assumes the following raw input database tables have been populated:
32	* raw_data_system_load
33	* user_defined_load_zone_units
34
35	=========
36	Settings
37	=========
38	* database
39	* output_directory
40	* load_scenario_id
41	* load_scenario_name
42	* overwrite
43
44	"""
45
46	import sys	3✔
47	from argparse import ArgumentParser	3✔
48	import os.path	3✔
49	import pandas as pd	3✔
50
51	from data_toolkit.system.common_methods import (	3✔
52	create_load_scenario_csv,
53	create_load_components_scenario_csv,
54	)
55	from db.common_functions import connect_to_database	3✔
56
57	LOAD_SCENARIO_ID_DEFAULT = 1 # it's 6 in the test examples	3✔
58	LOAD_SCENARIO_NAME_DEFAULT = "ra_toolkit"	3✔
59	LOAD_COMPONENTS_SCENARIO_ID_DEFAULT = 1 # it's 6 in the test examples	3✔
60	LOAD_COMPONENTS_SCENARIO_NAME_DEFAULT = "ra_toolkit"	3✔
61	LOAD_LEVELS_SCENARIO_ID_DEFAULT = 1 # it's 6 in the test examples	3✔
62	LOAD_LEVELS_SCENARIO_NAME_DEFAULT = "ra_toolkit"	3✔
63	STAGE_ID_DEFAULT = 1	3✔
64	LOAD_COMPONENT_NAME_DEFAULT = "all"	3✔
65
66
67	def parse_arguments(args):	3✔
68	"""
69	:param args: the script arguments specified by the user
70	:return: the parsed known argument values (<class 'argparse.Namespace'>
71	Python object)
72
73	Parse the known arguments.
74	"""
75	parser = ArgumentParser(add_help=True)	3✔
76
77	parser.add_argument("-db", "--database")	3✔
78
79	parser.add_argument(	3✔
80	"-out_dir",
81	"--output_directory",
82	help="""This will be the location of the load_scenario_id file. The
83	load components and load levels files are assumed to be in the
84	'load_components' and 'load_levels' subdirectories respectively.""",
85	)
86	parser.add_argument(	3✔
87	"-id",
88	"--load_scenario_id",
89	default=LOAD_SCENARIO_ID_DEFAULT,
90	help=f"Defaults to {LOAD_SCENARIO_ID_DEFAULT}.",
91	)
92	parser.add_argument(	3✔
93	"-name",
94	"--load_scenario_name",
95	default=LOAD_SCENARIO_NAME_DEFAULT,
96	help=f"Defaults to '{LOAD_SCENARIO_NAME_DEFAULT}'.",
97	)
98	parser.add_argument(	3✔
99	"-lc_id",
100	"--load_components_scenario_id",
101	default=LOAD_COMPONENTS_SCENARIO_ID_DEFAULT,
102	help=f"Defaults to {LOAD_COMPONENTS_SCENARIO_ID_DEFAULT}.",
103	)
104	parser.add_argument(	3✔
105	"-lc_name",
106	"--load_components_scenario_name",
107	default=LOAD_COMPONENTS_SCENARIO_NAME_DEFAULT,
108	help=f"Defaults to '{LOAD_COMPONENTS_SCENARIO_NAME_DEFAULT}'.",
109	)
110	parser.add_argument(	3✔
111	"-ll_id",
112	"--load_levels_scenario_id",
113	default=LOAD_LEVELS_SCENARIO_ID_DEFAULT,
114	help=f"Defaults to {LOAD_LEVELS_SCENARIO_ID_DEFAULT}.",
115	)
116	parser.add_argument(	3✔
117	"-ll_name",
118	"--load_levels_scenario_name",
119	default=LOAD_LEVELS_SCENARIO_NAME_DEFAULT,
120	help=f"Defaults to '{LOAD_LEVELS_SCENARIO_NAME_DEFAULT}'.",
121	)
122
123	parser.add_argument(	3✔
124	"-stage",
125	"--stage_id",
126	default=STAGE_ID_DEFAULT,
127	help=f"Defaults to '{STAGE_ID_DEFAULT}",
128	)
129
130	parser.add_argument(	3✔
131	"-comp",
132	"--load_component",
133	default=LOAD_COMPONENT_NAME_DEFAULT,
134	help=f"Defaults to '{LOAD_COMPONENT_NAME_DEFAULT}",
135	)
136
137	parser.add_argument(	3✔
138	"-l_o",
139	"--load_scenario_overwrite",
140	default=False,
141	action="store_true",
142	help="Overwrite existing CSV files.",
143	)
144	parser.add_argument(	3✔
145	"-lc_o",
146	"--load_components_overwrite",
147	default=False,
148	action="store_true",
149	help="Overwrite existing CSV files.",
150	)
151	parser.add_argument(	3✔
152	"-ll_o",
153	"--load_levels_overwrite",
154	default=False,
155	action="store_true",
156	help="Overwrite existing CSV files.",
157	)
158
159	parser.add_argument(	3✔
160	"-skip_l",
161	"--skip_load_scenario",
162	default=False,
163	action="store_true",
164	help="Don't create load_scenario file.",
165	)
166	parser.add_argument(	3✔
167	"-skip_lc",
168	"--skip_load_components",
169	default=False,
170	action="store_true",
171	help="Don't create load components file.",
172	)
173	parser.add_argument(	3✔
174	"-skip_ll",
175	"--skip_load_levels",
176	default=False,
177	action="store_true",
178	help="Don't create load levels file.",
179	)
180
181	parser.add_argument("-q", "--quiet", default=False, action="store_true")	3✔
182
183	parsed_arguments = parser.parse_known_args(args=args)[0]	3✔
184
185	return parsed_arguments	3✔
186
187
188	def create_load_levels_csv(	3✔
189	conn,
190	output_directory,
191	load_levels_scenario_id,
192	load_levels_scenario_name,
193	stage_id,
194	load_component_name,
195	overwrite,
196	):
197	"""
198	This module currently assumes timepoint IDs will be 1 through 8760 for
199	each year. The query will aggregate loads based on the aggregations and
200	weights defined in the user_defined_load_zone_units
201	table.
202	"""
203
204	query = f"""	3✔
205	SELECT load_zone, year AS weather_iteration, {stage_id} as stage_id,
206	hour_of_year as timepoint,
207	'{load_component_name}' AS load_component, sum(weighted_load_mw) as
208	load_mw
209	FROM (
210	SELECT year, month, day_of_month, hour_of_day, load_zone_unit, load_zone, unit_weight, load_mw, unit_weight * load_mw as weighted_load_mw,
211	(CAST(
212	strftime('%j',
213	year \|\| '-' \|\|
214	CASE
215	WHEN month > 9 THEN month
216	ELSE '0' \|\| month END
217	\|\| '-' \|\|
218	CASE
219	WHEN day_of_month > 9 THEN day_of_month
220	ELSE '0' \|\| day_of_month END
221	) AS DECIMAL
222	) - 1) * 24 + hour_of_day AS hour_of_year
223	FROM raw_data_system_load
224	JOIN user_defined_load_zone_units
225	USING (load_zone_unit)
226	)
227	GROUP BY load_zone, year, hour_of_year
228	"""
229
230	# Put into a dataframe and add to file
231	df = pd.read_sql(query, con=conn)	3✔
232
233	filename = os.path.join(	3✔
234	output_directory,
235	"load_levels",
236	f"{load_levels_scenario_id}_{load_levels_scenario_name}.csv",
237	)
238	if overwrite:	3✔
239	mode = "w"	3✔
240	write_header = True	3✔
241	else:
242	mode = "a"	×
243	write_header = not os.path.exists(filename)	×
244
245	df.to_csv(	3✔
246	filename,
247	mode=mode,
248	header=write_header,
249	index=False,
250	)
251
252
253	def main(args=None):	3✔
254	if args is None:	3✔
255	args = sys.argv[1:]	×
256
257	parsed_args = parse_arguments(args=args)	3✔
258
259	if not parsed_args.quiet:	3✔
260	print("Creating sync load profile CSVs...")	×
261
262	os.makedirs(parsed_args.output_directory, exist_ok=True)	3✔
263	os.makedirs(	3✔
264	os.path.join(parsed_args.output_directory, "load_components"), exist_ok=True
265	)
266	os.makedirs(	3✔
267	os.path.join(parsed_args.output_directory, "load_levels"), exist_ok=True
268	)
269
270	conn = connect_to_database(db_path=parsed_args.database)	3✔
271
272	if not parsed_args.skip_load_scenario:	3✔
273	create_load_scenario_csv(	3✔
274	output_directory=parsed_args.output_directory,
275	load_scenario_id=parsed_args.load_scenario_id,
276	load_scenario_name=parsed_args.load_scenario_name,
277	load_components_scenario_id=parsed_args.load_components_scenario_id,
278	load_levels_scenario_id=parsed_args.load_levels_scenario_id,
279	overwrite_load_scenario_csv=parsed_args.load_scenario_overwrite,
280	)
281
282	if not parsed_args.skip_load_components:	3✔
283	create_load_components_scenario_csv(	3✔
284	conn=conn,
285	output_directory=parsed_args.output_directory,
286	load_component_name=parsed_args.load_component,
287	load_components_scenario_id=parsed_args.load_components_scenario_id,
288	load_components_scenario_name=parsed_args.load_components_scenario_name,
289	overwrite_load_components_csv=parsed_args.load_components_overwrite,
290	)
291
292	if not parsed_args.skip_load_levels:	3✔
293	create_load_levels_csv(	3✔
294	conn=conn,
295	output_directory=parsed_args.output_directory,
296	load_levels_scenario_id=parsed_args.load_levels_scenario_id,
297	load_levels_scenario_name=parsed_args.load_levels_scenario_name,
298	stage_id=parsed_args.stage_id,
299	load_component_name=parsed_args.load_component,
300	overwrite=parsed_args.load_levels_overwrite,
301	)
302
303
304	if __name__ == "__main__":	3✔
UNCOV 305	main()	×

blue-marble / gridpath / 17811364834

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous