23001673678

Committed 12 Mar 2026 12:20PM UTC coverage: 81.12% (-0.009%) from 81.129%

Build # 23001673678

Build Type

Pull #91

github

Committed by

web-flow

Commit Message

Merge 7017ee53b into 345f856fd

Pull Request Pull Request #91: apps/summarization/export_utils: reduce size of ratings in export

Run Details

0 of 4 new or added lines in 1 file covered. (0.0%)

22 existing lines in 1 file now uncovered.

7386 of 9105 relevant lines covered (81.12%)

0.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

5.97

/apps/summarization/export_utils/processing/cleaning.py

def clean_dict(data):
    """Remove empty/null values from a dictionary recursively."""
    if not isinstance(data, dict):
        return data

    cleaned = {}
    for key, value in data.items():
        # Skip None values
        if value is None:
            continue

        # Skip empty lists
        if isinstance(value, list) and len(value) == 0:
            continue

        # Skip empty strings
        if isinstance(value, str) and value == "":
            continue

        # Recursively clean nested dicts
        if isinstance(value, dict):
            cleaned_value = clean_dict(value)
            if cleaned_value:  # Only include non-empty dicts
                cleaned[key] = cleaned_value
        else:
            cleaned[key] = value

    return cleaned


def clean_comment(comment):
    """Clean a comment by removing redundant fields."""
    cleaned = {
        "id": comment["id"],
        "text": comment["text"],
        "created": comment["created"],
    }

    # Only include replies if they exist
    if comment.get("replies"):
        cleaned["replies"] = [clean_comment(r) for r in comment["replies"]]
        cleaned["reply_count"] = len(cleaned["replies"])

    # Only include ratings if they exist
    if comment.get("ratings"):
        cleaned["ratings"] = comment["ratings"]

    return cleaned


def clean_content_item(item):  # noqa: C901
    """Clean a content item (idea, poll, etc.) by removing empty fields."""
    cleaned = {}

    # Always include essential fields
    for field in ["id", "name", "description", "created", "reference_number"]:
        if field in item:
            cleaned[field] = item[field]

    # Optional fields only if they have values
    if item.get("category"):
        cleaned["category"] = item["category"]

    if item.get("labels"):
        cleaned["labels"] = item["labels"]

    if item.get("attachments"):
        cleaned["attachments"] = item["attachments"]

    if item.get("images") and item["images"] != [""]:
        cleaned["images"] = item["images"]

    # Clean comments
    if item.get("comments"):
        cleaned["comments"] = [clean_comment(c) for c in item["comments"]]

    # Clean ratings
    if item.get("ratings"):
        cleaned["ratings"] = item["ratings"]

    # Poll-specific fields
    if item.get("choices"):
        cleaned["choices"] = item["choices"]
    if item.get("answers"):
        cleaned["answers"] = item["answers"]
    if item.get("other_answers"):
        cleaned["other_answers"] = item["other_answers"]

    # Map-specific fields
    if item.get("point") and (item["point"].get("lat") or item["point"].get("lng")):
        cleaned["point"] = item["point"]
    if item.get("point_label"):
        cleaned["point_label"] = item["point_label"]

    return cleaned


def clean_export(export_data):
    """Clean the entire export by removing empty/null values."""
    cleaned = {
        "project": clean_dict(export_data["project"]),  # Keep this for project
        "phases": {},
        "offline_events": export_data.get("offline_events", []),
    }

    # Clean each phase
    for phase_name, phase_data in export_data["phases"].items():
        cleaned_phase = {"phase_status": phase_data["phase_status"], "modules": []}

        for module in phase_data["modules"]:
            cleaned_module = {
                "module_id": module["module_id"],
                "module_name": module["module_name"],
                "module_type": module["module_type"],
                "module_start": module["module_start"],
                "module_end": module["module_end"],
                "url": module["url"],
                "content": {},
            }

            # Only include description if it exists
            if module.get("description"):
                cleaned_module["description"] = module["description"]

            # Clean each content type
            for content_type, items in module["content"].items():
                if items:
                    cleaned_module["content"][content_type] = [
                        clean_content_item(item) for item in items
                    ]

            cleaned_phase["modules"].append(cleaned_module)  # Remove clean_dict

        cleaned["phases"][phase_name] = cleaned_phase

    return cleaned  # Remove final clean_dict

1	def clean_dict(data):	1✔
2	"""Remove empty/null values from a dictionary recursively."""
3	if not isinstance(data, dict):	×
4	return data	×
5
6	cleaned = {}	×
7	for key, value in data.items():	×
8	# Skip None values
9	if value is None:	×
10	continue	×
11
12	# Skip empty lists
13	if isinstance(value, list) and len(value) == 0:	×
14	continue	×
15
16	# Skip empty strings
17	if isinstance(value, str) and value == "":	×
18	continue	×
19
20	# Recursively clean nested dicts
21	if isinstance(value, dict):	×
22	cleaned_value = clean_dict(value)	×
23	if cleaned_value: # Only include non-empty dicts	×
24	cleaned[key] = cleaned_value	×
25	else:
26	cleaned[key] = value	×
27
28	return cleaned	×
29
30
31	def clean_comment(comment):	1✔
32	"""Clean a comment by removing redundant fields."""
33	cleaned = {	×
34	"id": comment["id"],
35	"text": comment["text"],
36	"created": comment["created"],
37	}
38
39	# Only include replies if they exist
40	if comment.get("replies"):	×
41	cleaned["replies"] = [clean_comment(r) for r in comment["replies"]]	×
UNCOV 42	cleaned["reply_count"] = len(cleaned["replies"])	×
43
44	# Only include ratings if they exist
45	if comment.get("ratings"):	×
UNCOV 46	cleaned["ratings"] = comment["ratings"]	×
47
UNCOV 48	return cleaned	×
49
50
51	def clean_content_item(item): # noqa: C901	1✔
52	"""Clean a content item (idea, poll, etc.) by removing empty fields."""
UNCOV 53	cleaned = {}	×
54
55	# Always include essential fields
56	for field in ["id", "name", "description", "created", "reference_number"]:	×
57	if field in item:	×
UNCOV 58	cleaned[field] = item[field]	×
59
60	# Optional fields only if they have values
61	if item.get("category"):	×
UNCOV 62	cleaned["category"] = item["category"]	×
63
64	if item.get("labels"):	×
UNCOV 65	cleaned["labels"] = item["labels"]	×
66
67	if item.get("attachments"):	×
UNCOV 68	cleaned["attachments"] = item["attachments"]	×
69
70	if item.get("images") and item["images"] != [""]:	×
UNCOV 71	cleaned["images"] = item["images"]	×
72
73	# Clean comments
74	if item.get("comments"):	×
UNCOV 75	cleaned["comments"] = [clean_comment(c) for c in item["comments"]]	×
76
77	# Clean ratings
78	if item.get("ratings"):	×
UNCOV 79	cleaned["ratings"] = item["ratings"]	×
80
81	# Poll-specific fields
82	if item.get("choices"):	×
83	cleaned["choices"] = item["choices"]	×
84	if item.get("answers"):	×
85	cleaned["answers"] = item["answers"]	×
86	if item.get("other_answers"):	×
UNCOV 87	cleaned["other_answers"] = item["other_answers"]	×
88
89	# Map-specific fields
90	if item.get("point") and (item["point"].get("lat") or item["point"].get("lng")):	×
91	cleaned["point"] = item["point"]	×
92	if item.get("point_label"):	×
UNCOV 93	cleaned["point_label"] = item["point_label"]	×
94
UNCOV 95	return cleaned	×
96
97
98	def clean_export(export_data):	1✔
99	"""Clean the entire export by removing empty/null values."""
UNCOV 100	cleaned = {	×
101	"project": clean_dict(export_data["project"]), # Keep this for project
102	"phases": {},
103	"offline_events": export_data.get("offline_events", []),
104	}
105
106	# Clean each phase
107	for phase_name, phase_data in export_data["phases"].items():	×
UNCOV 108	cleaned_phase = {"phase_status": phase_data["phase_status"], "modules": []}	×
109
110	for module in phase_data["modules"]:	×
UNCOV 111	cleaned_module = {	×
112	"module_id": module["module_id"],
113	"module_name": module["module_name"],
114	"module_type": module["module_type"],
115	"module_start": module["module_start"],
116	"module_end": module["module_end"],
117	"url": module["url"],
118	"content": {},
119	}
120
121	# Only include description if it exists
122	if module.get("description"):	×
UNCOV 123	cleaned_module["description"] = module["description"]	×
124
125	# Clean each content type
126	for content_type, items in module["content"].items():	×
127	if items:	×
UNCOV 128	cleaned_module["content"][content_type] = [	×
129	clean_content_item(item) for item in items
130	]
131
UNCOV 132	cleaned_phase["modules"].append(cleaned_module) # Remove clean_dict	×
133
UNCOV 134	cleaned["phases"][phase_name] = cleaned_phase	×
135
UNCOV 136	return cleaned # Remove final clean_dict	×

liqd / roots / 23001673678

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous