7fca60fc-eac6-49b9-b932-fa8b70f17437

Committed 12 Mar 2024 09:10AM UTC coverage: 67.226% (+0.4%) from 66.849%

Build # 7fca60fc-eac6-49b9-b932-fa8b70f17437

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2249 from DemocracyClub/feature/parse-textract-results

Parse TextractResult results

Run Details

1706 of 2894 branches covered (58.95%)

Branch coverage included in aggregate %.

114 of 181 new or added lines in 9 files covered. (62.98%)

3 existing lines in 3 files now uncovered.

6866 of 9857 relevant lines covered (69.66%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/ynr/apps/sopn_parsing/management/commands/sopn_parsing_aws_textract.py

from datetime import timedelta
from time import sleep

from django.conf import settings
from django.utils import timezone
from official_documents.models import OfficialDocument
from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand
from sopn_parsing.helpers.extract_pages import (
    TextractSOPNHelper,
    TextractSOPNParsingHelper,
)
from sopn_parsing.models import AWSTextractParsedSOPN


class Command(BaseSOPNParsingCommand):
    def add_arguments(self, parser):
        super().add_arguments(parser)
        parser.add_argument(
            "--blocking",
            action="store_true",
            help="Wait for AWS to parse each SOPN",
        )
        parser.add_argument(
            "--start-analysis",
            action="store_true",
            help="Start AWS Textract analysis for each SOPN",
        )
        parser.add_argument(
            "--get-results",
            action="store_true",
            help="Get AWS Textract results for each SOPN",
        )
        parser.add_argument(
            "--upload-path",
            action="store",
            help="For texting only: the S3 bucket path to upload local SOPNs to, in the form of s3://[bucket]/[prefix]/",
        )

    def queue_full(self):
        time_window = timezone.now() - timedelta(hours=1)
        processing = AWSTextractParsedSOPN.objects.filter(
            created__gt=time_window,
            status__in=["NOT_STARTED", "IN_PROGRESS"],
        )

        if count := processing.count() > settings.TEXTRACT_CONCURRENT_QUOTA:
            print(f"Processing: {count}")
            return True
        return False

    def get_queryset(self, options):
        return super().get_queryset(options)

    def check_all_documents(self, options, **kwargs):
        qs = self.get_queryset(options).filter(
            officialdocument__awstextractparsedsopn__status__in=[
                "NOT_STARTED",
                "IN_PROGRESS",
            ]
        )
        if qs:
            print(f"Checking {qs.count()} documents")
            for document in qs:
                textract_helper = TextractSOPNHelper(document.sopn)
                textract_helper.update_job_status(blocking=False)

    def handle(self, *args, **options):
        qs = self.get_queryset(options)
        # in start-analysis, we want the qs to be all documents
        # that don't have a textract result on the sopn
        # in get-results, we want the qs to be all documents
        # that have a textract result on the sopn
        # the following is repetitive but addresses both options
        if options["start_analysis"]:
            if not options["reparse"]:
                qs = qs.exclude(
                    officialdocument__awstextractparsedsopn__id=None
                )
            for ballot in qs:
                official_document: OfficialDocument = ballot.sopn
                if self.queue_full():
                    self.check_all_documents(options)
                    sleep(settings.TEXTRACT_BACKOFF_TIME)
                textract_helper = TextractSOPNHelper(
                    official_document, upload_path=options["upload_path"]
                )
                # TO DO: add logging here
                if getattr(official_document, "textract_result", None):
                    continue
                sleep(settings.TEXTRACT_STAT_JOBS_PER_SECOND_QUOTA)
                textract_helper.start_detection(official_document)
        if options["get_results"]:
            qs = qs.filter(
                officialdocument__awstextractparsedsopn__isnull=False
            )
            for ballot in qs:
                official_document: OfficialDocument = ballot.sopn
                print(official_document)

                textract_helper = TextractSOPNHelper(official_document)
                textract_helper.update_job_status(
                    blocking=options["blocking"], reparse=options["reparse"]
                )

                textract_sopn_parsing_helper = TextractSOPNParsingHelper(
                    official_document
                )
                parsed = textract_sopn_parsing_helper.parse()
                print(parsed.as_pandas)
        self.check_all_documents(options)

1	from datetime import timedelta	×
2	from time import sleep	×
3
NEW 4	from django.conf import settings	×
5	from django.utils import timezone	×
NEW 6	from official_documents.models import OfficialDocument	×
7	from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand	×
8	from sopn_parsing.helpers.extract_pages import (	×
9	TextractSOPNHelper,
10	TextractSOPNParsingHelper,
11	)
NEW 12	from sopn_parsing.models import AWSTextractParsedSOPN	×
13
14
15	class Command(BaseSOPNParsingCommand):	×
16	def add_arguments(self, parser):	×
17	super().add_arguments(parser)	×
18	parser.add_argument(	×
19	"--blocking",
20	action="store_true",
21	help="Wait for AWS to parse each SOPN",
22	)
23	parser.add_argument(	×
24	"--start-analysis",
25	action="store_true",
26	help="Start AWS Textract analysis for each SOPN",
27	)
28	parser.add_argument(	×
29	"--get-results",
30	action="store_true",
31	help="Get AWS Textract results for each SOPN",
32	)
NEW 33	parser.add_argument(	×
34	"--upload-path",
35	action="store",
36	help="For texting only: the S3 bucket path to upload local SOPNs to, in the form of s3://[bucket]/[prefix]/",
37	)
38
39	def queue_full(self):	×
40	time_window = timezone.now() - timedelta(hours=1)	×
NEW 41	processing = AWSTextractParsedSOPN.objects.filter(	×
42	created__gt=time_window,
43	status__in=["NOT_STARTED", "IN_PROGRESS"],
44	)
45
NEW 46	if count := processing.count() > settings.TEXTRACT_CONCURRENT_QUOTA:	×
47	print(f"Processing: {count}")	×
48	return True	×
49	return False	×
50
51	def get_queryset(self, options):	×
NEW 52	return super().get_queryset(options)	×
53
54	def check_all_documents(self, options, **kwargs):	×
55	qs = self.get_queryset(options).filter(	×
56	officialdocument__awstextractparsedsopn__status__in=[
57	"NOT_STARTED",
58	"IN_PROGRESS",
59	]
60	)
61	if qs:	×
62	print(f"Checking {qs.count()} documents")	×
63	for document in qs:	×
NEW 64	textract_helper = TextractSOPNHelper(document.sopn)	×
65	textract_helper.update_job_status(blocking=False)	×
66
67	def handle(self, args, *options):	×
NEW 68	qs = self.get_queryset(options)	×
69	# in start-analysis, we want the qs to be all documents
70	# that don't have a textract result on the sopn
71	# in get-results, we want the qs to be all documents
72	# that have a textract result on the sopn
73	# the following is repetitive but addresses both options
NEW 74	if options["start_analysis"]:	×
NEW 75	if not options["reparse"]:	×
NEW 76	qs = qs.exclude(	×
77	officialdocument__awstextractparsedsopn__id=None
78	)
NEW 79	for ballot in qs:	×
NEW 80	official_document: OfficialDocument = ballot.sopn	×
81	if self.queue_full():	×
82	self.check_all_documents(options)	×
NEW 83	sleep(settings.TEXTRACT_BACKOFF_TIME)	×
NEW 84	textract_helper = TextractSOPNHelper(	×
85	official_document, upload_path=options["upload_path"]
86	)
87	# TO DO: add logging here
88	if getattr(official_document, "textract_result", None):	×
89	continue	×
NEW 90	sleep(settings.TEXTRACT_STAT_JOBS_PER_SECOND_QUOTA)	×
91	textract_helper.start_detection(official_document)	×
NEW 92	if options["get_results"]:	×
NEW 93	qs = qs.filter(	×
94	officialdocument__awstextractparsedsopn__isnull=False
95	)
NEW 96	for ballot in qs:	×
NEW 97	official_document: OfficialDocument = ballot.sopn	×
NEW 98	print(official_document)	×
99
100	textract_helper = TextractSOPNHelper(official_document)	×
NEW 101	textract_helper.update_job_status(	×
102	blocking=options["blocking"], reparse=options["reparse"]
103	)
104
NEW 105	textract_sopn_parsing_helper = TextractSOPNParsingHelper(	×
106	official_document
107	)
NEW 108	parsed = textract_sopn_parsing_helper.parse()	×
NEW 109	print(parsed.as_pandas)	×
NEW 110	self.check_all_documents(options)	×

DemocracyClub / yournextrepresentative / 7fca60fc-eac6-49b9-b932-fa8b70f17437

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous