17812509597

Committed 17 Sep 2025 10:39PM UTC coverage: 88.946% (-0.01%) from 88.959%

Build # 17812509597

Build Type

Pull #1289

github

Committed by

web-flow

Commit Message

Merge 97fc18e23 into bdb0a2ac5

Pull Request Pull Request #1289: Maintenance dependency upgrades

Coverage Stats

11 of 17 new or added lines in 7 files covered. (64.71%)

141 existing lines in 44 files now uncovered.

27351 of 30750 relevant lines covered (88.95%)

1.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.67

/data_toolkit/load_raw_data.py

# Copyright 2016-2024 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Load data into the GridPath raw data database. See the documentation of each
GridPath Data Toolkit module for data prerequisites. Use the
files_to_import.csv file to tell GridPath which CSV files should be loaded
into which database table.

"""

import sys
from argparse import ArgumentParser
import os.path
import pandas as pd

from db.common_functions import spin_on_database_lock_generic, connect_to_database


def parse_arguments(args):
    """
    :param args: the script arguments specified by the user
    :return: the parsed known argument values (<class 'argparse.Namespace'>
    Python object)

    Parse the known arguments.
    """
    parser = ArgumentParser(add_help=True)

    parser.add_argument("-db", "--database")
    parser.add_argument("-csv", "--csv_location")
    parser.add_argument("-q", "--quiet", default=False, action="store_true")

    parsed_arguments = parser.parse_known_args(args=args)[0]

    return parsed_arguments


def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parsed_args = parse_arguments(args=args)

    if not parsed_args.quiet:
        print("Importing raw data...")

    conn = connect_to_database(db_path=parsed_args.database)

    files_to_import_df = pd.read_csv(
        os.path.join(parsed_args.csv_location, "files_to_import.csv")
    )
    for index, row in files_to_import_df.iterrows():
        import_bool, f, table = row

        if import_bool:
            if not parsed_args.quiet:
                print(f"... {f}...")
            f_path = os.path.join(parsed_args.csv_location, f)
            df = pd.read_csv(f_path, delimiter=",")

            spin_on_database_lock_generic(
                command=df.to_sql(
                    name=table,
                    con=conn,
                    if_exists="append",
                    index=False,
                )
            )


if __name__ == "__main__":
    main()

1	# Copyright 2016-2024 Blue Marble Analytics LLC.
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# http://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	"""
16	Load data into the GridPath raw data database. See the documentation of each
17	GridPath Data Toolkit module for data prerequisites. Use the
18	files_to_import.csv file to tell GridPath which CSV files should be loaded
19	into which database table.
20
21	"""
22
23	import sys	2✔
24	from argparse import ArgumentParser	2✔
25	import os.path	2✔
26	import pandas as pd	2✔
27
28	from db.common_functions import spin_on_database_lock_generic, connect_to_database	2✔
29
30
31	def parse_arguments(args):	2✔
32	"""
33	:param args: the script arguments specified by the user
34	:return: the parsed known argument values (<class 'argparse.Namespace'>
35	Python object)
36
37	Parse the known arguments.
38	"""
39	parser = ArgumentParser(add_help=True)	2✔
40
41	parser.add_argument("-db", "--database")	2✔
42	parser.add_argument("-csv", "--csv_location")	2✔
43	parser.add_argument("-q", "--quiet", default=False, action="store_true")	2✔
44
45	parsed_arguments = parser.parse_known_args(args=args)[0]	2✔
46
47	return parsed_arguments	2✔
48
49
50	def main(args=None):	2✔
51	if args is None:	2✔
52	args = sys.argv[1:]	×
53
54	parsed_args = parse_arguments(args=args)	2✔
55
56	if not parsed_args.quiet:	2✔
57	print("Importing raw data...")	×
58
59	conn = connect_to_database(db_path=parsed_args.database)	2✔
60
61	files_to_import_df = pd.read_csv(	2✔
62	os.path.join(parsed_args.csv_location, "files_to_import.csv")
63	)
64	for index, row in files_to_import_df.iterrows():	2✔
65	import_bool, f, table = row	2✔
66
67	if import_bool:	2✔
68	if not parsed_args.quiet:	2✔
69	print(f"... {f}...")	×
70	f_path = os.path.join(parsed_args.csv_location, f)	2✔
71	df = pd.read_csv(f_path, delimiter=",")	2✔
72
73	spin_on_database_lock_generic(	2✔
74	command=df.to_sql(
75	name=table,
76	con=conn,
77	if_exists="append",
78	index=False,
79	)
80	)
81
82
83	if __name__ == "__main__":	2✔
UNCOV 84	main()	×

blue-marble / gridpath / 17812509597

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous