15068824991

Committed 12 Mar 2025 12:22PM UTC coverage: 38.661%. Remained the same

Build # 15068824991

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #170 from IATI/develop

Put the merged dashboard work live

Run Details

778 of 1567 new or added lines in 25 files covered. (49.65%)

242 existing lines in 21 files now uncovered.

878 of 2271 relevant lines covered (38.66%)

0.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/statsrunner/gitaggregate.py

1	import decimal	×
2	import json	×
NEW 3	import os	×
4	import sys	×
5
NEW 6	from common import decimal_default	×
7
8	# Set value for the gitout directory
NEW 9	GITOUT_DIR = os.environ.get("GITOUT_DIR") or "gitout"	×
10
11	# Set bool if the 'dated' argument has been used in calling this script
NEW 12	dated = len(sys.argv) > 1 and sys.argv[1] == "dated"	×
13
NEW 14	git_out_dir = os.path.join(GITOUT_DIR, "gitaggregate-dated" if dated else "gitaggregate")	×
15
16	# Exclude some json stats files from being aggregated
17	# These are typically the largest stats files that would consume large amounts of
18	# memory/disk space if aggregated over time
19	whitelisted_stats_files = [	×
20	"activities",
21	"activity_files",
22	"file_size_bins",
23	"file_size",
24	"invalidxml",
25	"nonstandardroots",
26	"organisation_files",
27	"publisher_has_org_file",
28	"publishers_per_version",
29	"publishers",
30	"publishers_validation",
31	"unique_identifiers",
32	"validation",
33	"versions",
34	"teststat", # Extra 'stat' added as the test_gitaggregate.py assumes a file with this name is present
35	]
36
37	# Load the reference of commits to dates
38	if dated:	×
NEW 39	gitdates = json.load(open("gitdate.json"))	×
40
41	# Make the gitout directory
42	try:	×
43	os.makedirs(git_out_dir)	×
44	except OSError:	×
45	pass	×
46
47	# Get a list containing the names of the entries in the directory
48	git_out_files = os.listdir(git_out_dir)	×
49
50	# Loop over each commit in gitout/commits
NEW 51	for commit in os.listdir(os.path.join(GITOUT_DIR, "commits")):	×
NEW 52	print("Aggregating for commit: {}".format(commit))	×
53
NEW 54	for fname in os.listdir(os.path.join(GITOUT_DIR, "commits", commit, "aggregated")):	×
NEW 55	if not fname.endswith(".json"):	×
UNCOV 56	continue	×
57
NEW 58	k = fname[:-5] # remove '.json' from the filename	×
59	# Ignore certain files
60	if k not in whitelisted_stats_files:	×
NEW 61	continue	×
62
NEW 63	print("Adding to {} for file: {}".format("gitaggregate-dated" if dated else "gitaggregate", fname))	×
64
NEW 65	commit_json_fname = os.path.join(GITOUT_DIR, "commits", commit, "aggregated", fname)	×
66
67	# Load the current file conents to memory, or set as an empty dictionary
UNCOV 68	if fname in git_out_files:	×
69	# FIXME: This is a possible cause of a memory issue in future, as the size of the aggregate file
70	# increases each time there is a new commit
71	with open(os.path.join(git_out_dir, fname)) as fp:	×
72	v = json.load(fp, parse_float=decimal.Decimal)	×
73	else:
74	v = {}	×
75
76	# If the commit that we are looping over is not already in the data for this file, then add it to the output
NEW 77	if commit not in v:	×
78	with open(commit_json_fname) as fp2:	×
79	v2 = json.load(fp2, parse_float=decimal.Decimal)	×
80	if dated:	×
81	if commit in gitdates:	×
82	v[gitdates[commit]] = v2	×
83	else:
84	v[commit] = v2	×
85
86	# Write output to a temporary file, then rename
NEW 87	with open(os.path.join(git_out_dir, k + ".json.new"), "w") as fp:	×
NEW 88	print("Writing data to {}".format(k))	×
89	json.dump(v, fp, sort_keys=True, indent=2, default=decimal_default)	×
NEW 90	print("Renaming file {} to {}".format(k + ".json.new", k + ".json"))	×
NEW 91	os.rename(os.path.join(git_out_dir, k + ".json.new"), os.path.join(git_out_dir, k + ".json"))	×

IATI / IATI-Stats / 15068824991

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous