11896224761

Committed 18 Nov 2024 03:59PM UTC coverage: 93.806% (-0.08%) from 93.887%

Build # 11896224761

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #265 from uw-it-aca/task/aws-upload

set up secrets for aws access

Run Details

12 of 14 new or added lines in 4 files covered. (85.71%)

3 existing lines in 1 file now uncovered.

3801 of 4052 relevant lines covered (93.81%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.65

/data_aggregator/report_builder.py

# Copyright 2024 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0


import io
import csv
from boto3 import client
from logging import getLogger
from django.conf import settings
from django.db import transaction
from django.test import override_settings
from data_aggregator.models import Report, SubaccountActivity
from data_aggregator.utilities import set_gcs_base_path
from data_aggregator.exceptions import TermNotStarted
from restclients_core.util.retry import retry
from restclients_core.exceptions import DataFailureException
from uw_canvas.accounts import Accounts as CanvasAccounts
from uw_canvas.analytics import Analytics as CanvasAnalytics
from uw_canvas.reports import Reports as CanvasReports
from uw_canvas.terms import Terms as CanvasTerms

logger = getLogger(__name__)

RETRY_STATUS_CODES = [0, 408, 500, 502, 503, 504]
RETRY_MAX = 5
RETRY_DELAY = 5


class ReportBuilder():
    def __init__(self):
        self._accounts = CanvasAccounts(per_page=100)
        self._analytics = CanvasAnalytics()
        self._reports = CanvasReports()
        self._terms = CanvasTerms()

    @retry(DataFailureException, status_codes=RETRY_STATUS_CODES,
           tries=RETRY_MAX, delay=RETRY_DELAY, logger=logger)
    def get_statistics_by_account(self, sis_account_id, sis_term_id):
        return self._analytics.get_statistics_by_account(
            sis_account_id, sis_term_id)

    @retry(DataFailureException, status_codes=RETRY_STATUS_CODES,
           tries=RETRY_MAX, delay=RETRY_DELAY, logger=logger)
    def get_activity_by_account(self, sis_account_id, sis_term_id):
        return self._analytics.get_activity_by_account(
            sis_account_id, sis_term_id)

    @retry(DataFailureException, status_codes=RETRY_STATUS_CODES,
           tries=RETRY_MAX, delay=RETRY_DELAY, logger=logger)
    def get_account_activities_data(self, root_account, sis_term_id):
        activities = []
        accounts = []
        accounts.append(root_account)
        accounts.extend(
            self._accounts.get_all_sub_accounts_by_sis_id(
                root_account.sis_account_id))
        activities = []
        for account in accounts:
            sis_account_id = account.sis_account_id
            if sis_account_id is None:
                continue
            activity = SubaccountActivity(term_id=sis_term_id,
                                          subaccount_id=sis_account_id,
                                          subaccount_name=account.name)

            data = self.get_statistics_by_account(sis_account_id, sis_term_id)

            for key, val in data.items():
                if key == "courses":
                    continue
                setattr(activity, key, val)
            try:
                data = self.get_activity_by_account(sis_account_id,
                                                    sis_term_id)
                for item in data["by_category"]:
                    setattr(activity,
                            "{}_views".format(item["category"]),
                            item["views"])
            except DataFailureException as ex:
                if ex.status != 504:
                    raise
            activities.append(activity)
        return activities

    def get_xlist_courses(self, root_account, sis_term_id):
        # create xlist lookup
        term = self._terms.get_term_by_sis_id(sis_term_id)
        xlist_courses = set()
        xlist_prov_report = self._reports.create_xlist_provisioning_report(
            root_account.account_id, term.term_id,
            params={"include_deleted": True})

        xlist_data_file = self._reports.get_report_data(xlist_prov_report)
        reader = csv.reader(xlist_data_file)
        next(reader, None)  # skip the headers
        for row in reader:
            if not len(row):
                continue
            sis_course_id = row[6]
            if sis_course_id:
                xlist_courses.add(sis_course_id)
        self._reports.delete_report(xlist_prov_report)
        return xlist_courses

    def get_course_data(self, root_account, sis_term_id):
        # create course totals lookup
        term = self._terms.get_term_by_sis_id(sis_term_id)
        course_prov_report = self._reports.create_course_provisioning_report(
            root_account.account_id, term.term_id,
            params={"include_deleted": True})
        course_data_file = self._reports.get_report_data(course_prov_report)
        course_data = []
        reader = csv.reader(course_data_file)
        next(reader, None)  # skip the headers
        for row in reader:
            if not len(row):
                continue
            course_data.append(row)
        self._reports.delete_report(course_prov_report)
        return course_data

    @transaction.atomic
    @override_settings(RESTCLIENTS_CANVAS_TIMEOUT=90)
    def build_subaccount_activity_report(self, root_account_id,
                                         sis_term_id=None, week_num=None):
        try:
            report = Report.objects.get_or_create_report(
                Report.SUBACCOUNT_ACTIVITY,
                sis_term_id=sis_term_id,
                week_num=week_num)
        except TermNotStarted as ex:
            logger.info("Term {} not started".format(ex))
            return

        set_gcs_base_path(report.term_id, report.term_week)

        root_account = self._accounts.get_account_by_sis_id(root_account_id)

        account_courses = {}

        # save activities and initialize course totals
        activity_data = self.get_account_activities_data(root_account,
                                                         report.term_id)
        for activity in activity_data:
            account_courses[activity.subaccount_id] = {
                "courses": 0,
                "active_courses": 0,
                "ind_study_courses": 0,
                "active_ind_study_courses": 0,
                "xlist_courses": 0,
                "xlist_ind_study_courses": 0
            }

            activity.report = report
            activity.save()

        # calculate course totals
        xlist_courses = self.get_xlist_courses(root_account, report.term_id)
        course_data = self.get_course_data(root_account, report.term_id)
        for row in course_data:
            if not len(row):
                continue

            sis_course_id = row[1]
            sis_account_id = row[6]
            if (sis_course_id is None or sis_account_id is None or
                    sis_account_id not in account_courses):
                continue

            status = row[9]
            ind_study = (len(sis_course_id.split("-")) == 6)
            is_xlist = (sis_course_id in xlist_courses)
            is_active = (status == "active")
            for sis_id in account_courses.keys():
                if sis_account_id.find(sis_id) == 0:
                    account_courses[sis_id]["courses"] += 1
                    if is_xlist:
                        account_courses[sis_id]["xlist_courses"] += 1
                    elif is_active:
                        account_courses[sis_id]["active_courses"] += 1

                    if ind_study:
                        account_courses[sis_id][
                            "ind_study_courses"] += 1
                        if is_xlist:
                            account_courses[sis_id][
                                "xlist_ind_study_courses"] += 1
                        elif is_active:
                            account_courses[sis_id][
                                "active_ind_study_courses"] += 1

        # save course totals
        for sis_account_id in account_courses:
            try:
                totals = account_courses[sis_account_id]
                activity = SubaccountActivity.objects.get(
                    report=report, term_id=report.term_id,
                    subaccount_id=sis_account_id)
                activity.courses = totals["courses"]
                activity.active_courses = totals["active_courses"]
                activity.ind_study_courses = totals["ind_study_courses"]
                activity.active_ind_study_courses = \
                    totals["active_ind_study_courses"]
                activity.xlist_courses = totals["xlist_courses"]
                activity.xlist_ind_study_courses = \
                    totals["xlist_ind_study_courses"]
                activity.save()
            except SubaccountActivity.DoesNotExist:
                continue

        report.finished()

    def export_subaccount_activity_report(
            self, sis_term_id=None, week_num=None):

        reports = Report.objects.get_subaccount_activity(
            sis_term_id=sis_term_id, week_num=week_num)

        if not len(reports):
            logger.info(f"No export data for {sis_term_id} week {week_num}")
            return

        self.upload_csv_file(self.generate_report_csv(reports))

    def generate_report_csv(self, reports):
        fileobj = io.StringIO()
        csv.register_dialect("unix_newline", lineterminator="\n")
        writer = csv.writer(fileobj, dialect="unix_newline")

        seen_terms = set()
        for index, report in enumerate(reports):
            if index == 0:
                writer.writerow(report.subaccount_activity_header())

            if report.term_id not in seen_terms:
                for subaccount in report.subaccounts:
                    writer.writerow(subaccount.csv_export_data())
                seen_terms.add(report.term_id)

        return fileobj.getvalue()

    def upload_csv_file(self, csv_data):
        s3client = client(
            "s3",
            aws_access_key_id=settings.EXPORT_AWS_ACCESS_KEY_ID,
            aws_secret_access_key=settings.EXPORT_AWS_SECRET_ACCESS_KEY)

        try:
            s3client.put_object(
                Body=csv_data,
                Bucket=settings.EXPORT_AWS_STORAGE_BUCKET_NAME,
                Key=settings.EXPORT_AWS_DEFAIULT_FILE_NAME,
                ContentType="text/csv",
            )
        except Exception as ex:
            logger.error(f"CSV write failed: {ex}")

1	# Copyright 2024 UW-IT, University of Washington
2	# SPDX-License-Identifier: Apache-2.0
3
4
5	import io	1✔
6	import csv	1✔
7	from boto3 import client	1✔
8	from logging import getLogger	1✔
9	from django.conf import settings	1✔
10	from django.db import transaction	1✔
11	from django.test import override_settings	1✔
12	from data_aggregator.models import Report, SubaccountActivity	1✔
13	from data_aggregator.utilities import set_gcs_base_path	1✔
14	from data_aggregator.exceptions import TermNotStarted	1✔
15	from restclients_core.util.retry import retry	1✔
16	from restclients_core.exceptions import DataFailureException	1✔
17	from uw_canvas.accounts import Accounts as CanvasAccounts	1✔
18	from uw_canvas.analytics import Analytics as CanvasAnalytics	1✔
19	from uw_canvas.reports import Reports as CanvasReports	1✔
20	from uw_canvas.terms import Terms as CanvasTerms	1✔
21
22	logger = getLogger(__name__)	1✔
23
24	RETRY_STATUS_CODES = [0, 408, 500, 502, 503, 504]	1✔
25	RETRY_MAX = 5	1✔
26	RETRY_DELAY = 5	1✔
27
28
29	class ReportBuilder():	1✔
30	def __init__(self):	1✔
31	self._accounts = CanvasAccounts(per_page=100)	1✔
32	self._analytics = CanvasAnalytics()	1✔
33	self._reports = CanvasReports()	1✔
34	self._terms = CanvasTerms()	1✔
35
36	@retry(DataFailureException, status_codes=RETRY_STATUS_CODES,	1✔
37	tries=RETRY_MAX, delay=RETRY_DELAY, logger=logger)
38	def get_statistics_by_account(self, sis_account_id, sis_term_id):	1✔
39	return self._analytics.get_statistics_by_account(	×
40	sis_account_id, sis_term_id)
41
42	@retry(DataFailureException, status_codes=RETRY_STATUS_CODES,	1✔
43	tries=RETRY_MAX, delay=RETRY_DELAY, logger=logger)
44	def get_activity_by_account(self, sis_account_id, sis_term_id):	1✔
45	return self._analytics.get_activity_by_account(	×
46	sis_account_id, sis_term_id)
47
48	@retry(DataFailureException, status_codes=RETRY_STATUS_CODES,	1✔
49	tries=RETRY_MAX, delay=RETRY_DELAY, logger=logger)
50	def get_account_activities_data(self, root_account, sis_term_id):	1✔
51	activities = []	1✔
52	accounts = []	1✔
53	accounts.append(root_account)	1✔
54	accounts.extend(	1✔
55	self._accounts.get_all_sub_accounts_by_sis_id(
56	root_account.sis_account_id))
57	activities = []	1✔
58	for account in accounts:	1✔
59	sis_account_id = account.sis_account_id	1✔
60	if sis_account_id is None:	1✔
61	continue	×
62	activity = SubaccountActivity(term_id=sis_term_id,	1✔
63	subaccount_id=sis_account_id,
64	subaccount_name=account.name)
65
66	data = self.get_statistics_by_account(sis_account_id, sis_term_id)	1✔
67
68	for key, val in data.items():	1✔
69	if key == "courses":	1✔
70	continue	×
71	setattr(activity, key, val)	1✔
72	try:	1✔
73	data = self.get_activity_by_account(sis_account_id,	1✔
74	sis_term_id)
75	for item in data["by_category"]:	1✔
76	setattr(activity,	1✔
77	"{}_views".format(item["category"]),
78	item["views"])
79	except DataFailureException as ex:	×
80	if ex.status != 504:	×
81	raise	×
82	activities.append(activity)	1✔
83	return activities	1✔
84
85	def get_xlist_courses(self, root_account, sis_term_id):	1✔
86	# create xlist lookup
87	term = self._terms.get_term_by_sis_id(sis_term_id)	1✔
88	xlist_courses = set()	1✔
89	xlist_prov_report = self._reports.create_xlist_provisioning_report(	1✔
90	root_account.account_id, term.term_id,
91	params={"include_deleted": True})
92
93	xlist_data_file = self._reports.get_report_data(xlist_prov_report)	1✔
94	reader = csv.reader(xlist_data_file)	1✔
95	next(reader, None) # skip the headers	1✔
96	for row in reader:	1✔
97	if not len(row):	1✔
98	continue	×
99	sis_course_id = row[6]	1✔
100	if sis_course_id:	1✔
101	xlist_courses.add(sis_course_id)	1✔
102	self._reports.delete_report(xlist_prov_report)	1✔
103	return xlist_courses	1✔
104
105	def get_course_data(self, root_account, sis_term_id):	1✔
106	# create course totals lookup
107	term = self._terms.get_term_by_sis_id(sis_term_id)	1✔
108	course_prov_report = self._reports.create_course_provisioning_report(	1✔
109	root_account.account_id, term.term_id,
110	params={"include_deleted": True})
111	course_data_file = self._reports.get_report_data(course_prov_report)	1✔
112	course_data = []	1✔
113	reader = csv.reader(course_data_file)	1✔
114	next(reader, None) # skip the headers	1✔
115	for row in reader:	1✔
116	if not len(row):	1✔
117	continue	×
118	course_data.append(row)	1✔
119	self._reports.delete_report(course_prov_report)	1✔
120	return course_data	1✔
121
122	@transaction.atomic	1✔
123	@override_settings(RESTCLIENTS_CANVAS_TIMEOUT=90)	1✔
124	def build_subaccount_activity_report(self, root_account_id,	1✔
125	sis_term_id=None, week_num=None):
126	try:	1✔
127	report = Report.objects.get_or_create_report(	1✔
128	Report.SUBACCOUNT_ACTIVITY,
129	sis_term_id=sis_term_id,
130	week_num=week_num)
131	except TermNotStarted as ex:	×
132	logger.info("Term {} not started".format(ex))	×
133	return	×
134
135	set_gcs_base_path(report.term_id, report.term_week)	1✔
136
137	root_account = self._accounts.get_account_by_sis_id(root_account_id)	1✔
138
139	account_courses = {}	1✔
140
141	# save activities and initialize course totals
142	activity_data = self.get_account_activities_data(root_account,	1✔
143	report.term_id)
144	for activity in activity_data:	1✔
145	account_courses[activity.subaccount_id] = {	1✔
146	"courses": 0,
147	"active_courses": 0,
148	"ind_study_courses": 0,
149	"active_ind_study_courses": 0,
150	"xlist_courses": 0,
151	"xlist_ind_study_courses": 0
152	}
153
154	activity.report = report	1✔
155	activity.save()	1✔
156
157	# calculate course totals
158	xlist_courses = self.get_xlist_courses(root_account, report.term_id)	1✔
159	course_data = self.get_course_data(root_account, report.term_id)	1✔
160	for row in course_data:	1✔
161	if not len(row):	1✔
162	continue	×
163
164	sis_course_id = row[1]	1✔
165	sis_account_id = row[6]	1✔
166	if (sis_course_id is None or sis_account_id is None or	1✔
167	sis_account_id not in account_courses):
168	continue	1✔
169
170	status = row[9]	×
171	ind_study = (len(sis_course_id.split("-")) == 6)	×
172	is_xlist = (sis_course_id in xlist_courses)	×
173	is_active = (status == "active")	×
174	for sis_id in account_courses.keys():	×
175	if sis_account_id.find(sis_id) == 0:	×
176	account_courses[sis_id]["courses"] += 1	×
177	if is_xlist:	×
178	account_courses[sis_id]["xlist_courses"] += 1	×
179	elif is_active:	×
180	account_courses[sis_id]["active_courses"] += 1	×
181
182	if ind_study:	×
183	account_courses[sis_id][	×
184	"ind_study_courses"] += 1
185	if is_xlist:	×
186	account_courses[sis_id][	×
187	"xlist_ind_study_courses"] += 1
188	elif is_active:	×
189	account_courses[sis_id][	×
190	"active_ind_study_courses"] += 1
191
192	# save course totals
193	for sis_account_id in account_courses:	1✔
194	try:	1✔
195	totals = account_courses[sis_account_id]	1✔
196	activity = SubaccountActivity.objects.get(	1✔
197	report=report, term_id=report.term_id,
198	subaccount_id=sis_account_id)
199	activity.courses = totals["courses"]	1✔
200	activity.active_courses = totals["active_courses"]	1✔
201	activity.ind_study_courses = totals["ind_study_courses"]	1✔
202	activity.active_ind_study_courses = \	1✔
203	totals["active_ind_study_courses"]
204	activity.xlist_courses = totals["xlist_courses"]	1✔
205	activity.xlist_ind_study_courses = \	1✔
206	totals["xlist_ind_study_courses"]
207	activity.save()	1✔
208	except SubaccountActivity.DoesNotExist:	×
209	continue	×
210
211	report.finished()	1✔
212
213	def export_subaccount_activity_report(	1✔
214	self, sis_term_id=None, week_num=None):
215
216	reports = Report.objects.get_subaccount_activity(	1✔
217	sis_term_id=sis_term_id, week_num=week_num)
218
219	if not len(reports):	1✔
220	logger.info(f"No export data for {sis_term_id} week {week_num}")	×
221	return	×
222
223	self.upload_csv_file(self.generate_report_csv(reports))	1✔
224
225	def generate_report_csv(self, reports):	1✔
226	fileobj = io.StringIO()	1✔
227	csv.register_dialect("unix_newline", lineterminator="\n")	1✔
228	writer = csv.writer(fileobj, dialect="unix_newline")	1✔
229
230	seen_terms = set()	1✔
231	for index, report in enumerate(reports):	1✔
232	if index == 0:	1✔
233	writer.writerow(report.subaccount_activity_header())	1✔
234
235	if report.term_id not in seen_terms:	1✔
236	for subaccount in report.subaccounts:	1✔
237	writer.writerow(subaccount.csv_export_data())	1✔
238	seen_terms.add(report.term_id)	1✔
239
240	return fileobj.getvalue()	1✔
241
242	def upload_csv_file(self, csv_data):	1✔
NEW 243	s3client = client(	×
244	"s3",
245	aws_access_key_id=settings.EXPORT_AWS_ACCESS_KEY_ID,
246	aws_secret_access_key=settings.EXPORT_AWS_SECRET_ACCESS_KEY)
247
UNCOV 248	try:	×
NEW 249	s3client.put_object(	×
250	Body=csv_data,
251	Bucket=settings.EXPORT_AWS_STORAGE_BUCKET_NAME,
252	Key=settings.EXPORT_AWS_DEFAIULT_FILE_NAME,
253	ContentType="text/csv",
254	)
UNCOV 255	except Exception as ex:	×
UNCOV 256	logger.error(f"CSV write failed: {ex}")	×

uw-it-aca / canvas-analytics / 11896224761

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous