17844932599

Committed 19 Sep 2025 12:32AM UTC coverage: 89.043%. Remained the same

Build # 17844932599

Build Type

Pull #1292

github

Committed by

web-flow

Commit Message

Merge b023aa42a into 096d8811e

Pull Request Pull Request #1292: Test on Linux, Windows, and Mac

Coverage Stats

13 of 15 new or added lines in 4 files covered. (86.67%)

1 existing line in 1 file now uncovered.

27680 of 31086 relevant lines covered (89.04%)

1.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.5

/data_toolkit/load_raw_data.py

# Copyright 2016-2024 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Load data into the GridPath raw data database. See the documentation of each
GridPath Data Toolkit module for data prerequisites. Use the
files_to_import.csv file to tell GridPath which CSV files should be loaded
into which database table.

"""

import sys
from argparse import ArgumentParser
import os.path
import pandas as pd

from db.common_functions import spin_on_database_lock_generic, connect_to_database


def parse_arguments(args):
    """
    :param args: the script arguments specified by the user
    :return: the parsed known argument values (<class 'argparse.Namespace'>
    Python object)

    Parse the known arguments.
    """
    parser = ArgumentParser(add_help=True)

    parser.add_argument("-db", "--database")
    parser.add_argument("-csv", "--csv_location")
    parser.add_argument("-q", "--quiet", default=False, action="store_true")

    parsed_arguments = parser.parse_known_args(args=args)[0]

    return parsed_arguments


def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parsed_args = parse_arguments(args=args)

    if not parsed_args.quiet:
        print("Importing raw data...")

    conn = connect_to_database(db_path=parsed_args.database)

    files_to_import_df = pd.read_csv(
        os.path.join(parsed_args.csv_location, "files_to_import.csv")
    )
    for index, row in files_to_import_df.iterrows():
        import_bool, f, table = row

        if import_bool:
            if not parsed_args.quiet:
                print(f"... {f}...")
            f_path = os.path.join(parsed_args.csv_location, f)

            # Set low_memory to False to avoid dtype warning
            # TODO: actually specify dtypes instead
            df = pd.read_csv(f_path, delimiter=",", low_memory=False)

            spin_on_database_lock_generic(
                command=df.to_sql(
                    name=table,
                    con=conn,
                    if_exists="append",
                    index=False,
                )
            )

    conn.close()


if __name__ == "__main__":
    main()

1	# Copyright 2016-2024 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	"""	1✔
16	Load data into the GridPath raw data database. See the documentation of each
17	GridPath Data Toolkit module for data prerequisites. Use the
18	files_to_import.csv file to tell GridPath which CSV files should be loaded
19	into which database table.
20
21	"""
22
23	import sys	2✔
24	from argparse import ArgumentParser	2✔
25	import os.path	2✔
26	import pandas as pd	2✔
27
28	from db.common_functions import spin_on_database_lock_generic, connect_to_database	2✔
29
30
31	def parse_arguments(args):	2✔
32	"""
33	:param args: the script arguments specified by the user
34	:return: the parsed known argument values (<class 'argparse.Namespace'>
35	Python object)
36
37	Parse the known arguments.
38	"""
39	parser = ArgumentParser(add_help=True)	2✔
40
41	parser.add_argument("-db", "--database")	2✔
42	parser.add_argument("-csv", "--csv_location")	2✔
43	parser.add_argument("-q", "--quiet", default=False, action="store_true")	2✔
44
45	parsed_arguments = parser.parse_known_args(args=args)[0]	2✔
46
47	return parsed_arguments	2✔
48
49
50	def main(args=None):	2✔
51	if args is None:	2✔
52	args = sys.argv[1:]	×
53
54	parsed_args = parse_arguments(args=args)	2✔
55
56	if not parsed_args.quiet:	2✔
57	print("Importing raw data...")	×
58
59	conn = connect_to_database(db_path=parsed_args.database)	2✔
60
61	files_to_import_df = pd.read_csv(	2✔
62	os.path.join(parsed_args.csv_location, "files_to_import.csv")
63	)
64	for index, row in files_to_import_df.iterrows():	2✔
65	import_bool, f, table = row	2✔
66
67	if import_bool:	2✔
68	if not parsed_args.quiet:	2✔
69	print(f"... {f}...")	×
70	f_path = os.path.join(parsed_args.csv_location, f)	2✔
71
72	# Set low_memory to False to avoid dtype warning
73	# TODO: actually specify dtypes instead
74	df = pd.read_csv(f_path, delimiter=",", low_memory=False)	2✔
75
76	spin_on_database_lock_generic(	2✔
77	command=df.to_sql(
78	name=table,
79	con=conn,
80	if_exists="append",
81	index=False,
82	)
83	)
84
85	conn.close()	2✔
86
87
88	if __name__ == "__main__":	2✔
UNCOV 89	main()	×

blue-marble / gridpath / 17844932599

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous