27789608046

Committed 18 Jun 2026 09:11PM UTC coverage: 88.65% (-0.02%) from 88.671%

Build # 27789608046

Build Type

push

github

Committed by

anamileva

Commit Message

Fix Read the Docs test setup (#1380)

Coverage Stats

28368 of 32000 relevant lines covered (88.65%)

0.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.57

/data_toolkit/load_raw_data.py

# Copyright 2016-2024 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Load data into the GridPath raw data database. See the documentation of each
GridPath Data Toolkit module for data prerequisites. Use the
``files_to_import.csv`` file to tell GridPath which CSV files should be loaded
into which database table.

==================
What this step does
==================

This module is a generic bulk loader for raw CSV data into the GridPath
database. It reads a  file named ``files_to_import.csv`` located in the
directory given by ``--csv_location``. Each row of that file describes one
CSV file: an import flag (whether the file should be loaded), the CSV
filename (relative to ``--csv_location``), and the database table the file
should be loaded into.

The loader iterates over the CSV file rows and, for each row whose import flag
is True, reads the corresponding CSV from ``--csv_location`` and appends its
contents to the named database table (existing rows are preserved; data is
inserted with ``if_exists="append"``). Rows whose import flag is False are
skipped.

This generic loader is used throughout the Data Toolkit workflow to populate
``raw_data`` tables (e.g., VER profiles and their unit mapping, hydro operating
characteristics) that later Data Toolkit steps depend on.

=====
Usage
=====

>>> python -m data_toolkit.load_raw_data --database PATH/TO/DATABASE --csv_location PATH/TO/CSV/DIRECTORY

=========
Settings
=========
    * database
    * csv_location

The ``--csv_location`` directory must contain a ``files_to_import.csv``
manifest with columns for the import flag, the CSV filename, and the
destination database table, in that order.

"""

import sys
from argparse import ArgumentParser
import os.path
from sqlite3 import Connection

import pandas as pd

from db.common_functions import spin_on_database_lock_generic, connect_to_database


def parse_arguments(args):
    """
    :param args: the script arguments specified by the user
    :return: the parsed known argument values (<class 'argparse.Namespace'>
    Python object)

    Parse the known arguments.
    """
    parser = ArgumentParser(add_help=True)

    parser.add_argument("-db", "--database")
    parser.add_argument("-csv", "--csv_location")
    parser.add_argument("-q", "--quiet", default=False, action="store_true")

    parsed_arguments = parser.parse_known_args(args=args)[0]

    return parsed_arguments


def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parsed_args = parse_arguments(args=args)

    if not parsed_args.quiet:
        print("Importing raw data...")

    conn = connect_to_database(db_path=parsed_args.database)

    files_to_import_df = pd.read_csv(
        os.path.join(parsed_args.csv_location, "files_to_import.csv")
    )
    for index, row in files_to_import_df.iterrows():
        import_bool, f, table = row

        if import_bool:
            if not parsed_args.quiet:
                print(f"... {f}...")
            f_path = str(os.path.join(parsed_args.csv_location, f))

            read_and_import_csv(conn, f_path, table)

    conn.commit()
    conn.close()


def read_and_import_csv(conn: Connection, f_path: str, table):
    # Set low_memory to False to avoid dtype warning
    # TODO: actually specify dtypes instead
    df = pd.read_csv(f_path, delimiter=",", low_memory=False, on_bad_lines="warn")

    # print(f_path)
    # print(df)
    spin_on_database_lock_generic(
        command=df.to_sql(
            name=table,
            con=conn,
            if_exists="append",
            index=False,
        )
    )


if __name__ == "__main__":
    main()

1	# Copyright 2016-2024 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	"""
16	Load data into the GridPath raw data database. See the documentation of each
17	GridPath Data Toolkit module for data prerequisites. Use the
18	``files_to_import.csv`` file to tell GridPath which CSV files should be loaded
19	into which database table.
20
21	==================
22	What this step does
23	==================
24
25	This module is a generic bulk loader for raw CSV data into the GridPath
26	database. It reads a file named ``files_to_import.csv`` located in the
27	directory given by ``--csv_location``. Each row of that file describes one
28	CSV file: an import flag (whether the file should be loaded), the CSV
29	filename (relative to ``--csv_location``), and the database table the file
30	should be loaded into.
31
32	The loader iterates over the CSV file rows and, for each row whose import flag
33	is True, reads the corresponding CSV from ``--csv_location`` and appends its
34	contents to the named database table (existing rows are preserved; data is
35	inserted with ``if_exists="append"``). Rows whose import flag is False are
36	skipped.
37
38	This generic loader is used throughout the Data Toolkit workflow to populate
39	``raw_data`` tables (e.g., VER profiles and their unit mapping, hydro operating
40	characteristics) that later Data Toolkit steps depend on.
41
42	=====
43	Usage
44	=====
45
46	>>> python -m data_toolkit.load_raw_data --database PATH/TO/DATABASE --csv_location PATH/TO/CSV/DIRECTORY
47
48	=========
49	Settings
50	=========
51	* database
52	* csv_location
53
54	The ``--csv_location`` directory must contain a ``files_to_import.csv``
55	manifest with columns for the import flag, the CSV filename, and the
56	destination database table, in that order.
57
58	"""
59
60	import sys	1✔
61	from argparse import ArgumentParser	1✔
62	import os.path	1✔
63	from sqlite3 import Connection	1✔
64
65	import pandas as pd	1✔
66
67	from db.common_functions import spin_on_database_lock_generic, connect_to_database	1✔
68
69
70	def parse_arguments(args):	1✔
71	"""
72	:param args: the script arguments specified by the user
73	:return: the parsed known argument values (<class 'argparse.Namespace'>
74	Python object)
75
76	Parse the known arguments.
77	"""
78	parser = ArgumentParser(add_help=True)	1✔
79
80	parser.add_argument("-db", "--database")	1✔
81	parser.add_argument("-csv", "--csv_location")	1✔
82	parser.add_argument("-q", "--quiet", default=False, action="store_true")	1✔
83
84	parsed_arguments = parser.parse_known_args(args=args)[0]	1✔
85
86	return parsed_arguments	1✔
87
88
89	def main(args=None):	1✔
90	if args is None:	1✔
91	args = sys.argv[1:]	×
92
93	parsed_args = parse_arguments(args=args)	1✔
94
95	if not parsed_args.quiet:	1✔
96	print("Importing raw data...")	×
97
98	conn = connect_to_database(db_path=parsed_args.database)	1✔
99
100	files_to_import_df = pd.read_csv(	1✔
101	os.path.join(parsed_args.csv_location, "files_to_import.csv")
102	)
103	for index, row in files_to_import_df.iterrows():	1✔
104	import_bool, f, table = row	1✔
105
106	if import_bool:	1✔
107	if not parsed_args.quiet:	1✔
108	print(f"... {f}...")	×
109	f_path = str(os.path.join(parsed_args.csv_location, f))	1✔
110
111	read_and_import_csv(conn, f_path, table)	1✔
112
113	conn.commit()	1✔
114	conn.close()	1✔
115
116
117	def read_and_import_csv(conn: Connection, f_path: str, table):	1✔
118	# Set low_memory to False to avoid dtype warning
119	# TODO: actually specify dtypes instead
120	df = pd.read_csv(f_path, delimiter=",", low_memory=False, on_bad_lines="warn")	1✔
121
122	# print(f_path)
123	# print(df)
124	spin_on_database_lock_generic(	1✔
125	command=df.to_sql(
126	name=table,
127	con=conn,
128	if_exists="append",
129	index=False,
130	)
131	)
132
133
134	if __name__ == "__main__":	1✔
135	main()	×

blue-marble / gridpath / 27789608046

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous