17811364834

Committed 17 Sep 2025 09:37PM UTC coverage: 89.049% (+0.09%) from 88.959%

Build # 17811364834

Build Type

Pull #1289

github

Committed by

web-flow

Commit Message

Merge 1e7c06b61 into bdb0a2ac5

Pull Request Pull Request #1289: Maintenance dependency upgrades

Coverage Stats

11 of 16 new or added lines in 7 files covered. (68.75%)

141 existing lines in 44 files now uncovered.

27640 of 31039 relevant lines covered (89.05%)

2.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.1

/data_toolkit/load_raw_data.py

# Copyright 2016-2024 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Load data into the GridPath raw data database. See the documentation of each
GridPath Data Toolkit module for data prerequisites. Use the
files_to_import.csv file to tell GridPath which CSV files should be loaded
into which database table.

"""

import sys
from argparse import ArgumentParser
import os.path
import pandas as pd

from db.common_functions import spin_on_database_lock_generic, connect_to_database


def parse_arguments(args):
    """
    :param args: the script arguments specified by the user
    :return: the parsed known argument values (<class 'argparse.Namespace'>
    Python object)

    Parse the known arguments.
    """
    parser = ArgumentParser(add_help=True)

    parser.add_argument("-db", "--database")
    parser.add_argument("-csv", "--csv_location")
    parser.add_argument("-q", "--quiet", default=False, action="store_true")

    parsed_arguments = parser.parse_known_args(args=args)[0]

    return parsed_arguments


def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parsed_args = parse_arguments(args=args)

    if not parsed_args.quiet:
        print("Importing raw data...")

    conn = connect_to_database(db_path=parsed_args.database)

    files_to_import_df = pd.read_csv(
        os.path.join(parsed_args.csv_location, "files_to_import.csv")
    )
    for index, row in files_to_import_df.iterrows():
        import_bool, f, table = row

        if import_bool:
            if not parsed_args.quiet:
                print(f"... {f}...")
            f_path = os.path.join(parsed_args.csv_location, f)
            df = pd.read_csv(f_path, delimiter=",")

            spin_on_database_lock_generic(
                command=df.to_sql(
                    name=table,
                    con=conn,
                    if_exists="append",
                    index=False,
                )
            )


if __name__ == "__main__":
    main()

1	# Copyright 2016-2024 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	"""	1✔
16	Load data into the GridPath raw data database. See the documentation of each
17	GridPath Data Toolkit module for data prerequisites. Use the
18	files_to_import.csv file to tell GridPath which CSV files should be loaded
19	into which database table.
20
21	"""
22
23	import sys	3✔
24	from argparse import ArgumentParser	3✔
25	import os.path	3✔
26	import pandas as pd	3✔
27
28	from db.common_functions import spin_on_database_lock_generic, connect_to_database	3✔
29
30
31	def parse_arguments(args):	3✔
32	"""
33	:param args: the script arguments specified by the user
34	:return: the parsed known argument values (<class 'argparse.Namespace'>
35	Python object)
36
37	Parse the known arguments.
38	"""
39	parser = ArgumentParser(add_help=True)	3✔
40
41	parser.add_argument("-db", "--database")	3✔
42	parser.add_argument("-csv", "--csv_location")	3✔
43	parser.add_argument("-q", "--quiet", default=False, action="store_true")	3✔
44
45	parsed_arguments = parser.parse_known_args(args=args)[0]	3✔
46
47	return parsed_arguments	3✔
48
49
50	def main(args=None):	3✔
51	if args is None:	3✔
52	args = sys.argv[1:]	×
53
54	parsed_args = parse_arguments(args=args)	3✔
55
56	if not parsed_args.quiet:	3✔
57	print("Importing raw data...")	×
58
59	conn = connect_to_database(db_path=parsed_args.database)	3✔
60
61	files_to_import_df = pd.read_csv(	3✔
62	os.path.join(parsed_args.csv_location, "files_to_import.csv")
63	)
64	for index, row in files_to_import_df.iterrows():	3✔
65	import_bool, f, table = row	3✔
66
67	if import_bool:	3✔
68	if not parsed_args.quiet:	3✔
69	print(f"... {f}...")	×
70	f_path = os.path.join(parsed_args.csv_location, f)	3✔
71	df = pd.read_csv(f_path, delimiter=",")	3✔
72
73	spin_on_database_lock_generic(	3✔
74	command=df.to_sql(
75	name=table,
76	con=conn,
77	if_exists="append",
78	index=False,
79	)
80	)
81
82
83	if __name__ == "__main__":	3✔
UNCOV 84	main()	×

blue-marble / gridpath / 17811364834

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous