11332460857

Committed 14 Oct 2024 05:40PM UTC coverage: 59.134%. Remained the same

Build # 11332460857

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #274 from uw-it-aca/feature/new-career-outcomes

switching to admit.uw outcomes

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

628 of 1062 relevant lines covered (59.13%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/pathways/management/commands/import_data.py

# Copyright 2024 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

from django.core.management.base import BaseCommand
import json
import csv
from pathways.data_import import import_major_data, import_course_data, \
    import_curric_data, import_level_coi, import_coi_ranges, \
    import_gateway_courses, import_bottleneck_courses, \
    import_career_center_mapping
from logging import getLogger
import time
import hashlib
from pathways.models.data_import import DataImport


logger = getLogger(__name__)
COI_PATH = "pathways/data/coi_scores.csv"
MAJOR_PATH = "pathways/data/major_data.json"
COURSE_PATH = "pathways/data/course_data.json"
CURRIC_PATH = "pathways/data/curric_data.json"
BOTTLENECK_PATH = "pathways/data/bottleneck_courses.csv"
GATEWAY_PATH = "pathways/data/gateway_courses.csv"
CAREER_CENTER_PATH = "pathways/data/admit_major_mapping.csv"


class Command(BaseCommand):
    help = 'Run all unit tests'

    def handle(self, *args, **options):
        start = time.time()
        if not self.data_needs_update():
            return
        with open(COI_PATH) as coi_file:
            reader = csv.reader(coi_file)
            # skip headers
            next(reader)
            coi_data = []
            for row in reader:
                coi_data.append({"course_id": row[0],
                                 "coi_score": float(row[1])})
            import_level_coi(coi_data)
            import_coi_ranges(coi_data)
            with open(MAJOR_PATH) as major_file:
                data = json.load(major_file)
                import_major_data(data)
            with open(COURSE_PATH) as course_file:
                data = json.load(course_file)
                import_course_data(data, coi_data)
            with open(CURRIC_PATH) as curric_file:
                data = json.load(curric_file)
                import_curric_data(data, coi_data)
            with open(GATEWAY_PATH) as gateway_file:
                data = csv.reader(gateway_file)
                import_gateway_courses(data)
            with open(BOTTLENECK_PATH) as bottleneck_file:
                data = csv.reader(bottleneck_file)
                import_bottleneck_courses(data)
            with open(CAREER_CENTER_PATH) as career_major_file:
                data = csv.reader(career_major_file)
                import_career_center_mapping(data)

        total_time = time.time() - start
        logger.info("Imported data in: %s" % total_time)

    def data_needs_update(self):
        needs_update = False
        coi_hash = self._get_hash_by_path(COI_PATH)
        if DataImport.needs_import('coi', coi_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='coi',
                                  defaults={'hash': coi_hash})
        major_hash = self._get_hash_by_path(MAJOR_PATH)
        if DataImport.needs_import('major', major_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='major',
                                  defaults={'hash': major_hash})
        course_hash = self._get_hash_by_path(COURSE_PATH)
        if DataImport.needs_import('course', course_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='course',
                                  defaults={'hash': course_hash})
        curric_hash = self._get_hash_by_path(CURRIC_PATH)
        if DataImport.needs_import('curric', curric_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='curric',
                                  defaults={'hash': curric_hash})
        gateway_hash = self._get_hash_by_path(GATEWAY_PATH)
        if DataImport.needs_import('gateway', gateway_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='gateway',
                                  defaults={'hash': gateway_hash})
        bottleneck_hash = self._get_hash_by_path(BOTTLENECK_PATH)
        if DataImport.needs_import('bottleneck', bottleneck_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='bottleneck',
                                  defaults={'hash': bottleneck_hash})
        career_hash = self._get_hash_by_path(CAREER_CENTER_PATH)
        if DataImport.needs_import('cc_major', career_hash):
            needs_update = True
            DataImport.objects \
                .update_or_create(type='cc_major',
                                  defaults={'hash': career_hash})
        return needs_update

    def _get_hash_by_path(self, path):
        return hashlib.md5(open(path, 'rb').read()).hexdigest()

1	# Copyright 2024 UW-IT, University of Washington
2	# SPDX-License-Identifier: Apache-2.0
3
4	from django.core.management.base import BaseCommand	×
5	import json	×
6	import csv	×
7	from pathways.data_import import import_major_data, import_course_data, \	×
8	import_curric_data, import_level_coi, import_coi_ranges, \
9	import_gateway_courses, import_bottleneck_courses, \
10	import_career_center_mapping
11	from logging import getLogger	×
12	import time	×
13	import hashlib	×
14	from pathways.models.data_import import DataImport	×
15
16
17	logger = getLogger(__name__)	×
18	COI_PATH = "pathways/data/coi_scores.csv"	×
19	MAJOR_PATH = "pathways/data/major_data.json"	×
20	COURSE_PATH = "pathways/data/course_data.json"	×
21	CURRIC_PATH = "pathways/data/curric_data.json"	×
22	BOTTLENECK_PATH = "pathways/data/bottleneck_courses.csv"	×
23	GATEWAY_PATH = "pathways/data/gateway_courses.csv"	×
NEW 24	CAREER_CENTER_PATH = "pathways/data/admit_major_mapping.csv"	×
25
26
27	class Command(BaseCommand):	×
28	help = 'Run all unit tests'	×
29
30	def handle(self, args, *options):	×
31	start = time.time()	×
32	if not self.data_needs_update():	×
33	return	×
34	with open(COI_PATH) as coi_file:	×
35	reader = csv.reader(coi_file)	×
36	# skip headers
37	next(reader)	×
38	coi_data = []	×
39	for row in reader:	×
40	coi_data.append({"course_id": row[0],	×
41	"coi_score": float(row[1])})
42	import_level_coi(coi_data)	×
43	import_coi_ranges(coi_data)	×
44	with open(MAJOR_PATH) as major_file:	×
45	data = json.load(major_file)	×
46	import_major_data(data)	×
47	with open(COURSE_PATH) as course_file:	×
48	data = json.load(course_file)	×
49	import_course_data(data, coi_data)	×
50	with open(CURRIC_PATH) as curric_file:	×
51	data = json.load(curric_file)	×
52	import_curric_data(data, coi_data)	×
53	with open(GATEWAY_PATH) as gateway_file:	×
54	data = csv.reader(gateway_file)	×
55	import_gateway_courses(data)	×
56	with open(BOTTLENECK_PATH) as bottleneck_file:	×
57	data = csv.reader(bottleneck_file)	×
58	import_bottleneck_courses(data)	×
59	with open(CAREER_CENTER_PATH) as career_major_file:	×
60	data = csv.reader(career_major_file)	×
61	import_career_center_mapping(data)	×
62
63	total_time = time.time() - start	×
64	logger.info("Imported data in: %s" % total_time)	×
65
66	def data_needs_update(self):	×
67	needs_update = False	×
68	coi_hash = self._get_hash_by_path(COI_PATH)	×
69	if DataImport.needs_import('coi', coi_hash):	×
70	needs_update = True	×
71	DataImport.objects \	×
72	.update_or_create(type='coi',
73	defaults={'hash': coi_hash})
74	major_hash = self._get_hash_by_path(MAJOR_PATH)	×
75	if DataImport.needs_import('major', major_hash):	×
76	needs_update = True	×
77	DataImport.objects \	×
78	.update_or_create(type='major',
79	defaults={'hash': major_hash})
80	course_hash = self._get_hash_by_path(COURSE_PATH)	×
81	if DataImport.needs_import('course', course_hash):	×
82	needs_update = True	×
83	DataImport.objects \	×
84	.update_or_create(type='course',
85	defaults={'hash': course_hash})
86	curric_hash = self._get_hash_by_path(CURRIC_PATH)	×
87	if DataImport.needs_import('curric', curric_hash):	×
88	needs_update = True	×
89	DataImport.objects \	×
90	.update_or_create(type='curric',
91	defaults={'hash': curric_hash})
92	gateway_hash = self._get_hash_by_path(GATEWAY_PATH)	×
93	if DataImport.needs_import('gateway', gateway_hash):	×
94	needs_update = True	×
95	DataImport.objects \	×
96	.update_or_create(type='gateway',
97	defaults={'hash': gateway_hash})
98	bottleneck_hash = self._get_hash_by_path(BOTTLENECK_PATH)	×
99	if DataImport.needs_import('bottleneck', bottleneck_hash):	×
100	needs_update = True	×
101	DataImport.objects \	×
102	.update_or_create(type='bottleneck',
103	defaults={'hash': bottleneck_hash})
104	career_hash = self._get_hash_by_path(CAREER_CENTER_PATH)	×
105	if DataImport.needs_import('cc_major', career_hash):	×
106	needs_update = True	×
107	DataImport.objects \	×
108	.update_or_create(type='cc_major',
109	defaults={'hash': career_hash})
110	return needs_update	×
111
112	def _get_hash_by_path(self, path):	×
113	return hashlib.md5(open(path, 'rb').read()).hexdigest()	×

uw-it-aca / pathways / 11332460857

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous