50cdb228-e78e-4d1f-9913-120901709e22

Committed 06 Nov 2023 01:56PM UTC coverage: 67.485% (-0.3%) from 67.801%

Build # 50cdb228-e78e-4d1f-9913-120901709e22

Build Type

Pull #2177

circleci

Committed by

VirginiaDooley

Commit Message

fixup! Extract tables with AWS Textract

Pull Request Pull Request #2177: Spike: AWS Textract

Run Details

1640 of 2760 branches covered (0.0%)

Branch coverage included in aggregate %.

12 of 69 new or added lines in 3 files covered. (17.39%)

110 existing lines in 10 files now uncovered.

6662 of 9542 relevant lines covered (69.82%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/ynr/apps/sopn_parsing/management/commands/sopn_parsing_parse_tables.py

from bulk_adding.models import RawPeople
from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand
from sopn_parsing.helpers.parse_tables import parse_raw_data_for_ballot


class Command(BaseSOPNParsingCommand):
    help = """
    Convert the raw extracted tables on the ParsedSOPN model to a parsed
    RawPeople model, and set the status as parsed.

    """

    def build_filter_kwargs(self, options):
        """
        Build kwargs used to filter the BallotQuerySet that is parsed
        - Always skip any ballots where we do not have a ParsedSOPN to try to
        extract candidates from
        - When test flag is used, dont make any changes
        - When parsing a single ballot, dont make any changes
        - When reparsing, only use ballots where we have previously created a
          RawPeople object from a ParsedSOPN
        - Otherwise filter by unparsed ParsedSOPN objects
        """
        # Always skip any ballots where we do not have a ParsedSOPN to try to
        # extract candidates from
        filter_kwargs = {"officialdocument__parsedsopn__isnull": False}
        if options.get("testing"):
            return filter_kwargs

        if options.get("ballot"):
            return filter_kwargs

        if options.get("reparse"):
            filter_kwargs[
                "rawpeople__source_type"
            ] = RawPeople.SOURCE_PARSED_PDF
            return filter_kwargs

        filter_kwargs["officialdocument__parsedsopn__parsed_data"] = None

        # Where the status is unparsed
        filter_kwargs["officialdocument__parsedsopn__status"] = "unparsed"

        return filter_kwargs

    def handle(self, *args, **options):
        # filters that we never change with args. These two would raise
        # ValueErrors in the parse_raw_data_for_ballot function
        base_qs = self.get_queryset(options)

        filter_kwargs = self.build_filter_kwargs(options)

        qs = base_qs.filter(**filter_kwargs)
        qs = qs.filter(
            candidates_locked=False,  # Never parse a locked ballot
            suggestedpostlock=None,  # Never parse a ballot with lock suggestions
        )

        if not qs.exists():
            msg = ["No ballots to parse found."]

            if options.get("ballot"):
                msg.append(
                    "This ballot might be locked or have lock suggestions"
                )

            self.stderr.write("\n".join(msg))

        for ballot in qs:
            parse_raw_data_for_ballot(ballot)

1	from bulk_adding.models import RawPeople	×
2	from sopn_parsing.helpers.command_helpers import BaseSOPNParsingCommand	×
3	from sopn_parsing.helpers.parse_tables import parse_raw_data_for_ballot	×
4
5
6	class Command(BaseSOPNParsingCommand):	×
7	help = """	×
8	Convert the raw extracted tables on the ParsedSOPN model to a parsed
9	RawPeople model, and set the status as parsed.
10
11	"""
12
13	def build_filter_kwargs(self, options):	×
14	"""
15	Build kwargs used to filter the BallotQuerySet that is parsed
16	- Always skip any ballots where we do not have a ParsedSOPN to try to
17	extract candidates from
18	- When test flag is used, dont make any changes
19	- When parsing a single ballot, dont make any changes
20	- When reparsing, only use ballots where we have previously created a
21	RawPeople object from a ParsedSOPN
22	- Otherwise filter by unparsed ParsedSOPN objects
23	"""
24	# Always skip any ballots where we do not have a ParsedSOPN to try to
25	# extract candidates from
26	filter_kwargs = {"officialdocument__parsedsopn__isnull": False}	×
27	if options.get("testing"):	×
28	return filter_kwargs	×
29
30	if options.get("ballot"):	×
31	return filter_kwargs	×
32
33	if options.get("reparse"):	×
34	filter_kwargs[	×
35	"rawpeople__source_type"
36	] = RawPeople.SOURCE_PARSED_PDF
37	return filter_kwargs	×
38
39	filter_kwargs["officialdocument__parsedsopn__parsed_data"] = None	×
40
41	# Where the status is unparsed
42	filter_kwargs["officialdocument__parsedsopn__status"] = "unparsed"	×
43
44	return filter_kwargs	×
45
46	def handle(self, args, *options):	×
47	# filters that we never change with args. These two would raise
48	# ValueErrors in the parse_raw_data_for_ballot function
UNCOV 49	base_qs = self.get_queryset(options)	×
50
51	filter_kwargs = self.build_filter_kwargs(options)	×
52
53	qs = base_qs.filter(**filter_kwargs)	×
UNCOV 54	qs = qs.filter(	×
55	candidates_locked=False, # Never parse a locked ballot
56	suggestedpostlock=None, # Never parse a ballot with lock suggestions
57	)
58
UNCOV 59	if not qs.exists():	×
UNCOV 60	msg = ["No ballots to parse found."]	×
61
62	if options.get("ballot"):	×
UNCOV 63	msg.append(	×
64	"This ballot might be locked or have lock suggestions"
65	)
66
UNCOV 67	self.stderr.write("\n".join(msg))	×
68
69	for ballot in qs:	×
UNCOV 70	parse_raw_data_for_ballot(ballot)	×

DemocracyClub / yournextrepresentative / 50cdb228-e78e-4d1f-9913-120901709e22

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous