7b7037ed-8eaf-4ec1-aa09-efaec9dc9039

Committed 01 Apr 2024 02:06PM UTC coverage: 67.088% (-0.2%) from 67.282%

Build # 7b7037ed-8eaf-4ec1-aa09-efaec9dc9039

Build Type

Pull #2269

circleci

Committed by

symroe

Commit Message

Add initial SOPN Matching tooling

Pull Request Pull Request #2269: Redesign SOPN models

Run Details

1712 of 2916 branches covered (58.71%)

Branch coverage included in aggregate %.

245 of 287 new or added lines in 16 files covered. (85.37%)

31 existing lines in 6 files now uncovered.

6941 of 9982 relevant lines covered (69.54%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.81

/ynr/apps/official_documents/models.py

import os
from pathlib import Path
from typing import List

from django.core.validators import FileExtensionValidator
from django.db import models
from django.urls import reverse
from django.utils import timezone
from django_extensions.db.models import TimeStampedModel
from sopn_parsing.models import AWSTextractParsedSOPN

DOCUMENT_UPLOADERS_GROUP_NAME = "Document Uploaders"


def document_file_name(instance, filename):
    return os.path.join(
        "official_documents", str(instance.ballot.ballot_paper_id), filename
    )


class OfficialDocument(TimeStampedModel):
    NOMINATION_PAPER = "Nomination paper"

    DOCUMENT_TYPES = (
        (NOMINATION_PAPER, "Nomination paper", "Nomination papers"),
    )

    document_type = models.CharField(
        blank=False,
        choices=[(d[0], d[1]) for d in DOCUMENT_TYPES],
        max_length=100,
    )
    uploaded_file = models.FileField(
        upload_to=document_file_name,
        max_length=800,
        validators=[FileExtensionValidator(allowed_extensions=["pdf", "docx"])],
    )
    ballot = models.ForeignKey(
        "candidates.Ballot", null=False, on_delete=models.CASCADE
    )
    source_url = models.URLField(
        help_text="The URL of this document", max_length=1000
    )
    relevant_pages = models.CharField(
        "The pages containing information about this ballot",
        max_length=50,
        default="",
    )

    class Meta:
        get_latest_by = "modified"

    def __str__(self):
        return "{} ({})".format(self.ballot.ballot_paper_id, self.source_url)

    def get_absolute_url(self):
        return reverse(
            "ballot_paper_sopn",
            kwargs={"ballot_id": self.ballot.ballot_paper_id},
        )

    @property
    def locked(self):
        """
        Is this post election locked?
        """
        return self.ballot.candidates_locked

    @property
    def lock_suggested(self):
        """
        Is there a suggested lock for this document?
        """
        return self.ballot.suggestedpostlock_set.exists()

    def get_pages(self):
        if self.relevant_pages and self.relevant_pages != "all":
            pages = self.relevant_pages.split(",")
            return sorted(int(p) for p in pages)
        return None

    @property
    def get_aws_parsed_sopn(self):
        """given an instance of an official document, return the associated aws parsed sopn"""
        return AWSTextractParsedSOPN.objects.get(sopn=self)

    @property
    def first_page_number(self):
        if self.get_pages():
            return self.get_pages()[0]
        return None

    @property
    def last_page_number(self):
        if self.get_pages():
            return self.get_pages()[-1]
        return None


def election_sopn_file_name(instance: "ElectionSOPN", filename):
    return (
        Path("official_documents")
        / instance.election.slug
        / timezone.now().isoformat()
        / filename
    )


class PageMatchingMethods(models.TextChoices):
    AUTO_MATCHED = "Automatically matched"
    MANUAL_MATCHED = "Manually matched"


class ElectionSOPN(TimeStampedModel):
    """
    Stores SOPNs that contain candidate information for each ballot in an election.

    We don't use this directly, rather we use this as a donor document for populating BallotSOPN instances.

    """

    election = models.OneToOneField(
        "elections.Election", on_delete=models.CASCADE, blank=True
    )

    uploaded_file = models.FileField(
        upload_to=election_sopn_file_name,
        max_length=1200,
        validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
    )

    source_url = models.URLField(
        help_text="The URL of this document", max_length=1200
    )

    page_matching_method = models.CharField(
        max_length=255, choices=PageMatchingMethods.choices, null=True
    )

    class Meta:
        get_latest_by = "modified"

    def __str__(self):
        return "{} ({})".format(self.election.slug, self.source_url)

    def get_absolute_url(self):
        return reverse(
            "election_sopn",
            kwargs={"election_id": self.election.slug},
        )

    @property
    def pages_matched(self):
        return (
            bool(self.page_matching_method)
            and not self.election.ballot_set.filter(
                sopn__relevant_pages__isnull=True
            ).exists()
        )


def ballot_sopn_file_name(instance: "BaseBallotSOPN", filename):
    return (
        Path("official_documents")
        / instance.ballot.ballot_paper_id
        / "sopn"
        / timezone.now().isoformat()
        / filename
    )


class BaseBallotSOPN(TimeStampedModel):
    """
    An abstract class used by BallotSOPN and BallotSOPNHistory. Used to keep both models in sync.

    """

    ballot = models.OneToOneField(
        "candidates.Ballot", on_delete=models.CASCADE, related_name="sopn"
    )

    # Don't add `ballot` here as we want different related names for each model
    uploaded_file = models.FileField(
        upload_to=ballot_sopn_file_name,
        max_length=1200,
        validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
    )

    source_url = models.URLField(
        help_text="The URL of this document", max_length=1200
    )

    relevant_pages = models.CharField(
        max_length=20,
        help_text="The pages in the ElectionSOPN that relate to this Ballot",
        default="all",
    )

    class Meta:
        get_latest_by = "modified"
        abstract = True

    def __str__(self):
        return "{} ({})".format(self.ballot.ballot_paper_id, self.source_url)

    @property
    def first_page_int(self):
        return min(self.page_number_list)

    @property
    def page_number_list(self) -> List[int]:
        return [int(i) for i in self.relevant_pages.split(",") if i != "all"]

    @property
    def one_based_relevant_pages(self):
        return ", ".join([str(i + 1) for i in self.page_number_list])


class BallotSOPN(BaseBallotSOPN):
    ...

    def get_absolute_url(self):
        return reverse(
            "ballot_paper_sopn",
            kwargs={"ballot_id": self.ballot.ballot_paper_id},
        )


class BallotSOPNHistory(BaseBallotSOPN):
    ballot = models.ForeignKey(
        "candidates.Ballot",
        on_delete=models.CASCADE,
        related_name="sopn_history",
    )

1	import os	1✔
2	from pathlib import Path	1✔
3	from typing import List	1✔
4
5	from django.core.validators import FileExtensionValidator	1✔
6	from django.db import models	1✔
7	from django.urls import reverse	1✔
8	from django.utils import timezone	1✔
9	from django_extensions.db.models import TimeStampedModel	1✔
10	from sopn_parsing.models import AWSTextractParsedSOPN	1✔
11
12	DOCUMENT_UPLOADERS_GROUP_NAME = "Document Uploaders"	1✔
13
14
15	def document_file_name(instance, filename):	1✔
UNCOV 16	return os.path.join(	×
17	"official_documents", str(instance.ballot.ballot_paper_id), filename
18	)
19
20
21	class OfficialDocument(TimeStampedModel):	1✔
22	NOMINATION_PAPER = "Nomination paper"	1✔
23
24	DOCUMENT_TYPES = (	1✔
25	(NOMINATION_PAPER, "Nomination paper", "Nomination papers"),
26	)
27
28	document_type = models.CharField(	1✔
29	blank=False,
30	choices=[(d[0], d[1]) for d in DOCUMENT_TYPES],
31	max_length=100,
32	)
33	uploaded_file = models.FileField(	1✔
34	upload_to=document_file_name,
35	max_length=800,
36	validators=[FileExtensionValidator(allowed_extensions=["pdf", "docx"])],
37	)
38	ballot = models.ForeignKey(	1✔
39	"candidates.Ballot", null=False, on_delete=models.CASCADE
40	)
41	source_url = models.URLField(	1✔
42	help_text="The URL of this document", max_length=1000
43	)
44	relevant_pages = models.CharField(	1✔
45	"The pages containing information about this ballot",
46	max_length=50,
47	default="",
48	)
49
50	class Meta:	1✔
51	get_latest_by = "modified"	1✔
52
53	def __str__(self):	1✔
UNCOV 54	return "{} ({})".format(self.ballot.ballot_paper_id, self.source_url)	×
55
56	def get_absolute_url(self):	1✔
UNCOV 57	return reverse(	×
58	"ballot_paper_sopn",
59	kwargs={"ballot_id": self.ballot.ballot_paper_id},
60	)
61
62	@property	1✔
63	def locked(self):	1✔
64	"""
65	Is this post election locked?
66	"""
UNCOV 67	return self.ballot.candidates_locked	×
68
69	@property	1✔
70	def lock_suggested(self):	1✔
71	"""
72	Is there a suggested lock for this document?
73	"""
UNCOV 74	return self.ballot.suggestedpostlock_set.exists()	×
75
76	def get_pages(self):	1✔
UNCOV 77	if self.relevant_pages and self.relevant_pages != "all":	×
UNCOV 78	pages = self.relevant_pages.split(",")	×
UNCOV 79	return sorted(int(p) for p in pages)	×
UNCOV 80	return None	×
81
82	@property	1✔
83	def get_aws_parsed_sopn(self):	1✔
84	"""given an instance of an official document, return the associated aws parsed sopn"""
85	return AWSTextractParsedSOPN.objects.get(sopn=self)	×
86
87	@property	1✔
88	def first_page_number(self):	1✔
UNCOV 89	if self.get_pages():	×
UNCOV 90	return self.get_pages()[0]	×
UNCOV 91	return None	×
92
93	@property	1✔
94	def last_page_number(self):	1✔
UNCOV 95	if self.get_pages():	×
UNCOV 96	return self.get_pages()[-1]	×
UNCOV 97	return None	×
98
99
100	def election_sopn_file_name(instance: "ElectionSOPN", filename):	1✔
101	return (	1✔
102	Path("official_documents")
103	/ instance.election.slug
104	/ timezone.now().isoformat()
105	/ filename
106	)
107
108
109	class PageMatchingMethods(models.TextChoices):	1✔
110	AUTO_MATCHED = "Automatically matched"	1✔
111	MANUAL_MATCHED = "Manually matched"	1✔
112
113
114	class ElectionSOPN(TimeStampedModel):	1✔
115	"""
116	Stores SOPNs that contain candidate information for each ballot in an election.
117
118	We don't use this directly, rather we use this as a donor document for populating BallotSOPN instances.
119
120	"""
121
122	election = models.OneToOneField(	1✔
123	"elections.Election", on_delete=models.CASCADE, blank=True
124	)
125
126	uploaded_file = models.FileField(	1✔
127	upload_to=election_sopn_file_name,
128	max_length=1200,
129	validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
130	)
131
132	source_url = models.URLField(	1✔
133	help_text="The URL of this document", max_length=1200
134	)
135
136	page_matching_method = models.CharField(	1✔
137	max_length=255, choices=PageMatchingMethods.choices, null=True
138	)
139
140	class Meta:	1✔
141	get_latest_by = "modified"	1✔
142
143	def __str__(self):	1✔
NEW 144	return "{} ({})".format(self.election.slug, self.source_url)	×
145
146	def get_absolute_url(self):	1✔
NEW 147	return reverse(	×
148	"election_sopn",
149	kwargs={"election_id": self.election.slug},
150	)
151
152	@property	1✔
153	def pages_matched(self):	1✔
NEW 154	return (	×
155	bool(self.page_matching_method)
156	and not self.election.ballot_set.filter(
157	sopn__relevant_pages__isnull=True
158	).exists()
159	)
160
161
162	def ballot_sopn_file_name(instance: "BaseBallotSOPN", filename):	1✔
163	return (	1✔
164	Path("official_documents")
165	/ instance.ballot.ballot_paper_id
166	/ "sopn"
167	/ timezone.now().isoformat()
168	/ filename
169	)
170
171
172	class BaseBallotSOPN(TimeStampedModel):	1✔
173	"""
174	An abstract class used by BallotSOPN and BallotSOPNHistory. Used to keep both models in sync.
175
176	"""
177
178	ballot = models.OneToOneField(	1✔
179	"candidates.Ballot", on_delete=models.CASCADE, related_name="sopn"
180	)
181
182	# Don't add `ballot` here as we want different related names for each model
183	uploaded_file = models.FileField(	1✔
184	upload_to=ballot_sopn_file_name,
185	max_length=1200,
186	validators=[FileExtensionValidator(allowed_extensions=["pdf"])],
187	)
188
189	source_url = models.URLField(	1✔
190	help_text="The URL of this document", max_length=1200
191	)
192
193	relevant_pages = models.CharField(	1✔
194	max_length=20,
195	help_text="The pages in the ElectionSOPN that relate to this Ballot",
196	default="all",
197	)
198
199	class Meta:	1✔
200	get_latest_by = "modified"	1✔
201	abstract = True	1✔
202
203	def __str__(self):	1✔
204	return "{} ({})".format(self.ballot.ballot_paper_id, self.source_url)	1✔
205
206	@property	1✔
207	def first_page_int(self):	1✔
NEW 208	return min(self.page_number_list)	×
209
210	@property	1✔
211	def page_number_list(self) -> List[int]:	1✔
NEW 212	return [int(i) for i in self.relevant_pages.split(",") if i != "all"]	×
213
214	@property	1✔
215	def one_based_relevant_pages(self):	1✔
NEW 216	return ", ".join([str(i + 1) for i in self.page_number_list])	×
217
218
219	class BallotSOPN(BaseBallotSOPN):	1✔
220	...
221
222	def get_absolute_url(self):	1✔
223	return reverse(	1✔
224	"ballot_paper_sopn",
225	kwargs={"ballot_id": self.ballot.ballot_paper_id},
226	)
227
228
229	class BallotSOPNHistory(BaseBallotSOPN):	1✔
230	ballot = models.ForeignKey(	1✔
231	"candidates.Ballot",
232	on_delete=models.CASCADE,
233	related_name="sopn_history",
234	)

DemocracyClub / yournextrepresentative / 7b7037ed-8eaf-4ec1-aa09-efaec9dc9039

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous