06fffc96-31fa-407b-8443-1731e4db6cf1

Committed 04 Apr 2024 08:24AM CUT coverage: 68.368% (+1.1%) from 67.282%

Build # 06fffc96-31fa-407b-8443-1731e4db6cf1

Build Type

Pull #2269

circleci

Committed by

symroe

Commit Message

Clean up some testsing code

Misc fixes to the test code

Pull Request Pull Request #2269: Redesign SOPN models

Run Details

1749 of 2922 branches covered (59.86%)

Branch coverage included in aggregate %.

415 of 488 new or added lines in 25 files covered. (85.04%)

18 existing lines in 4 files now uncovered.

7093 of 10011 relevant lines covered (70.85%)

0.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/ynr/apps/sopn_parsing/management/commands/sopn_parsing_aws_textract.py

from datetime import timedelta
from time import sleep

from django.conf import settings
from django.utils import timezone
from official_documents.models import OfficialDocument
from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand
from sopn_parsing.helpers.textract_helpers import (
    TextractSOPNHelper,
    TextractSOPNParsingHelper,
)
from sopn_parsing.models import AWSTextractParsedSOPN


class Command(BaseSOPNParsingCommand):
    def add_arguments(self, parser):
        super().add_arguments(parser)
        parser.add_argument(
            "--blocking",
            action="store_true",
            help="Wait for AWS to parse each SOPN",
        )
        parser.add_argument(
            "--start-analysis",
            action="store_true",
            help="Start AWS Textract analysis for each SOPN",
        )
        parser.add_argument(
            "--get-results",
            action="store_true",
            help="Get AWS Textract results for each SOPN",
        )
        parser.add_argument(
            "--upload-path",
            action="store",
            help="For texting only: the S3 bucket path to upload local SOPNs to, in the form of s3://[bucket]/[prefix]/",
        )

    def queue_full(self):
        time_window = timezone.now() - timedelta(hours=1)
        processing = AWSTextractParsedSOPN.objects.filter(
            created__gt=time_window,
            status__in=["NOT_STARTED", "IN_PROGRESS"],
        )

        if count := processing.count() > settings.TEXTRACT_CONCURRENT_QUOTA:
            print(f"Processing: {count}")
            return True
        return False

    def get_queryset(self, options):
        return super().get_queryset(options)

    def check_all_documents(self, options, **kwargs):
        qs = self.get_queryset(options).filter(
            officialdocument__awstextractparsedsopn__status__in=[
                "NOT_STARTED",
                "IN_PROGRESS",
            ]
        )
        if qs:
            print(f"Checking {qs.count()} documents")
            for document in qs:
                textract_helper = TextractSOPNHelper(document.sopn)
                textract_helper.update_job_status(blocking=False)

    def handle(self, *args, **options):
        qs = self.get_queryset(options)
        # in start-analysis, we want the qs to be all documents
        # that don't have a textract result on the sopn
        # in get-results, we want the qs to be all documents
        # that have a textract result on the sopn
        # the following is repetitive but addresses both options
        if options["start_analysis"]:
            if not options["reparse"]:
                qs = qs.exclude(
                    officialdocument__awstextractparsedsopn__id=None
                )
            for ballot in qs:
                self.stdout.write(
                    f"Starting analysis for {ballot.ballot_paper_id}"
                )
                official_document: OfficialDocument = ballot.sopn
                if self.queue_full():
                    self.stdout.write(
                        f"Queue full, sleeping {settings.TEXTRACT_BACKOFF_TIME}"
                    )
                    self.check_all_documents(options)
                    sleep(settings.TEXTRACT_BACKOFF_TIME)
                textract_helper = TextractSOPNHelper(
                    official_document, upload_path=options["upload_path"]
                )
                # TO DO: add logging here
                if getattr(official_document, "textract_result", None):
                    continue
                sleep(settings.TEXTRACT_STAT_JOBS_PER_SECOND_QUOTA)
                textract_helper.start_detection(official_document)
        if options["get_results"]:
            qs = qs.filter(
                officialdocument__awstextractparsedsopn__isnull=False
            )
            for ballot in qs:
                official_document: OfficialDocument = ballot.sopn
                print(official_document)

                textract_helper = TextractSOPNHelper(official_document)
                textract_helper.update_job_status(
                    blocking=options["blocking"], reparse=options["reparse"]
                )

                textract_sopn_parsing_helper = TextractSOPNParsingHelper(
                    official_document
                )
                parsed = textract_sopn_parsing_helper.parse()
                print(parsed.as_pandas)
        self.check_all_documents(options)

1	from datetime import timedelta	×
2	from time import sleep	×
3
4	from django.conf import settings	×
5	from django.utils import timezone	×
6	from official_documents.models import OfficialDocument	×
7	from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand	×
NEW 8	from sopn_parsing.helpers.textract_helpers import (	×
9	TextractSOPNHelper,
10	TextractSOPNParsingHelper,
11	)
12	from sopn_parsing.models import AWSTextractParsedSOPN	×
13
14
15	class Command(BaseSOPNParsingCommand):	×
16	def add_arguments(self, parser):	×
17	super().add_arguments(parser)	×
18	parser.add_argument(	×
19	"--blocking",
20	action="store_true",
21	help="Wait for AWS to parse each SOPN",
22	)
23	parser.add_argument(	×
24	"--start-analysis",
25	action="store_true",
26	help="Start AWS Textract analysis for each SOPN",
27	)
28	parser.add_argument(	×
29	"--get-results",
30	action="store_true",
31	help="Get AWS Textract results for each SOPN",
32	)
33	parser.add_argument(	×
34	"--upload-path",
35	action="store",
36	help="For texting only: the S3 bucket path to upload local SOPNs to, in the form of s3://[bucket]/[prefix]/",
37	)
38
39	def queue_full(self):	×
40	time_window = timezone.now() - timedelta(hours=1)	×
41	processing = AWSTextractParsedSOPN.objects.filter(	×
42	created__gt=time_window,
43	status__in=["NOT_STARTED", "IN_PROGRESS"],
44	)
45
46	if count := processing.count() > settings.TEXTRACT_CONCURRENT_QUOTA:	×
47	print(f"Processing: {count}")	×
48	return True	×
49	return False	×
50
51	def get_queryset(self, options):	×
52	return super().get_queryset(options)	×
53
54	def check_all_documents(self, options, **kwargs):	×
55	qs = self.get_queryset(options).filter(	×
56	officialdocument__awstextractparsedsopn__status__in=[
57	"NOT_STARTED",
58	"IN_PROGRESS",
59	]
60	)
61	if qs:	×
62	print(f"Checking {qs.count()} documents")	×
63	for document in qs:	×
64	textract_helper = TextractSOPNHelper(document.sopn)	×
65	textract_helper.update_job_status(blocking=False)	×
66
67	def handle(self, args, *options):	×
68	qs = self.get_queryset(options)	×
69	# in start-analysis, we want the qs to be all documents
70	# that don't have a textract result on the sopn
71	# in get-results, we want the qs to be all documents
72	# that have a textract result on the sopn
73	# the following is repetitive but addresses both options
74	if options["start_analysis"]:	×
75	if not options["reparse"]:	×
76	qs = qs.exclude(	×
77	officialdocument__awstextractparsedsopn__id=None
78	)
79	for ballot in qs:	×
80	self.stdout.write(	×
81	f"Starting analysis for {ballot.ballot_paper_id}"
82	)
83	official_document: OfficialDocument = ballot.sopn	×
84	if self.queue_full():	×
85	self.stdout.write(	×
86	f"Queue full, sleeping {settings.TEXTRACT_BACKOFF_TIME}"
87	)
88	self.check_all_documents(options)	×
89	sleep(settings.TEXTRACT_BACKOFF_TIME)	×
90	textract_helper = TextractSOPNHelper(	×
91	official_document, upload_path=options["upload_path"]
92	)
93	# TO DO: add logging here
94	if getattr(official_document, "textract_result", None):	×
95	continue	×
96	sleep(settings.TEXTRACT_STAT_JOBS_PER_SECOND_QUOTA)	×
97	textract_helper.start_detection(official_document)	×
98	if options["get_results"]:	×
99	qs = qs.filter(	×
100	officialdocument__awstextractparsedsopn__isnull=False
101	)
102	for ballot in qs:	×
103	official_document: OfficialDocument = ballot.sopn	×
104	print(official_document)	×
105
106	textract_helper = TextractSOPNHelper(official_document)	×
107	textract_helper.update_job_status(	×
108	blocking=options["blocking"], reparse=options["reparse"]
109	)
110
111	textract_sopn_parsing_helper = TextractSOPNParsingHelper(	×
112	official_document
113	)
114	parsed = textract_sopn_parsing_helper.parse()	×
115	print(parsed.as_pandas)	×
116	self.check_all_documents(options)	×

DemocracyClub / yournextrepresentative / 06fffc96-31fa-407b-8443-1731e4db6cf1

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous