756047b3-833c-4f2e-bcef-27a788f4472f

Committed 07 Mar 2024 05:00PM UTC coverage: 67.566% (+0.4%) from 67.183%

Build # 756047b3-833c-4f2e-bcef-27a788f4472f

Build Type

Pull #2249

circleci

Committed by

symroe

Commit Message

Deal with not being in AWS better

Pull Request Pull Request #2249: Parse TextractResult results

Run Details

1695 of 2842 branches covered (59.64%)

Branch coverage included in aggregate %.

114 of 181 new or added lines in 9 files covered. (62.98%)

3 existing lines in 3 files now uncovered.

6846 of 9799 relevant lines covered (69.86%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.0

/ynr/apps/sopn_parsing/models.py

import json
from io import BytesIO

from django.core.files.images import ImageFile
from django.db import models
from model_utils.models import TimeStampedModel
from pandas import concat
from textractor.parsers import response_parser
from textractor.parsers.response_parser import parse


class CamelotParsedSOPN(TimeStampedModel):
    """
    A model for storing the parsed data out of a PDF
    where data has been extracted using Camelot.

    """

    sopn = models.OneToOneField(
        "official_documents.OfficialDocument", on_delete=models.CASCADE
    )
    raw_data = models.TextField()
    raw_data_type = models.CharField(max_length=255, default="pandas")
    parsed_data = models.TextField(null=True)
    status = models.CharField(max_length=255, default="unparsed")

    @property
    def as_pandas(self):
        import pandas

        pandas.set_option("display.max_colwidth", None)
        return pandas.DataFrame.from_dict(json.loads(self.raw_data))

    @property
    def data_as_html(self):
        if self.raw_data_type == "pandas":
            data = self.as_pandas
            header = data.iloc[0]
            data = data[1:]
            data.columns = header
            return data.to_html(index=False, escape=False).replace(
                "\\n", "<br>"
            )
        return None


class AWSTextractParsedSOPNStatus(models.TextChoices):
    NOT_STARTED = "NOT_STARTED", "Not Started"
    SUCCEEDED = "SUCCEEDED", "Succeeded"
    IN_PROGRESS = "IN_PROGRESS", "In Progress"
    FAILED = "FAILED", "Failed"
    PARTIAL_SUCCESS = "PARTIAL_SUCCESS", "Partial Success"


def AWSTextractParsedSOPNImage_upload_path(instance, filename):
    return f"AWSTextractParsedSOPNImages/{instance.pk}/{filename}"


class AWSTextractParsedSOPNImage(models.Model):
    image = models.ImageField(upload_to=AWSTextractParsedSOPNImage_upload_path)
    parsed_sopn = models.ForeignKey(
        "AWSTextractParsedSOPN", on_delete=models.CASCADE, related_name="images"
    )

    @staticmethod
    def pil_to_content_image(pil_image, filename):
        """
        Takes a PIL image and returns an ImageFile
        """
        image_io = BytesIO()
        pil_image.save(image_io, format="PNG")
        return ImageFile(image_io, name=filename)


class AWSTextractParsedSOPN(TimeStampedModel):
    """
    A model for storing the parsed data out of a PDF
    where data has been extracted using AWS Textract.

    """

    sopn = models.OneToOneField(
        "official_documents.OfficialDocument", on_delete=models.CASCADE
    )
    job_id = models.CharField(max_length=100)
    raw_data = models.TextField()
    raw_data_type = models.CharField(max_length=255, default="pandas")
    parsed_data = models.TextField(null=True)
    status = models.CharField(
        max_length=255,
        choices=AWSTextractParsedSOPNStatus.choices,
        default=AWSTextractParsedSOPNStatus.NOT_STARTED,
    )

    @property
    def as_pandas(self):
        import pandas

        pandas.set_option("display.max_colwidth", None)
        return pandas.DataFrame.from_dict(json.loads(self.parsed_data))

    def parse_raw_data(self):
        """
        Use AWS Textractor to convert the raw JSON response
        to a Pandas dataframe (and then to JSON for storing
        on the model).

        It's possible for Textract to detect more than one
        table per document. Default to merging them all
        into one table.
        :return:
        """
        # User Textractor to parse the raw JSON
        parsed = parse(json.loads(self.raw_data))
        # Store all data frames in a list
        frames = []

        # Table headers that we've seen
        for table in parsed.tables:
            # Get the pandas version of the table
            df = table.to_pandas()
            frames.append(df)

        # Merge all the dataframes
        df = concat(
            frames,
            ignore_index=True,
        )
        self.parsed_data = df.to_json()

    def as_textractor_document(self):
        return response_parser.parse(json.loads(self.raw_data))

1	import json	1✔
2	from io import BytesIO	1✔
3
4	from django.core.files.images import ImageFile	1✔
5	from django.db import models	1✔
6	from model_utils.models import TimeStampedModel	1✔
7	from pandas import concat	1✔
8	from textractor.parsers import response_parser	1✔
9	from textractor.parsers.response_parser import parse	1✔
10
11
12	class CamelotParsedSOPN(TimeStampedModel):	1✔
13	"""
14	A model for storing the parsed data out of a PDF
15	where data has been extracted using Camelot.
16
17	"""
18
19	sopn = models.OneToOneField(	1✔
20	"official_documents.OfficialDocument", on_delete=models.CASCADE
21	)
22	raw_data = models.TextField()	1✔
23	raw_data_type = models.CharField(max_length=255, default="pandas")	1✔
24	parsed_data = models.TextField(null=True)	1✔
25	status = models.CharField(max_length=255, default="unparsed")	1✔
26
27	@property	1✔
28	def as_pandas(self):	1✔
29	import pandas	1✔
30
31	pandas.set_option("display.max_colwidth", None)	1✔
32	return pandas.DataFrame.from_dict(json.loads(self.raw_data))	1✔
33
34	@property	1✔
35	def data_as_html(self):	1✔
36	if self.raw_data_type == "pandas":	×
37	data = self.as_pandas	×
38	header = data.iloc[0]	×
39	data = data[1:]	×
40	data.columns = header	×
41	return data.to_html(index=False, escape=False).replace(	×
42	"\\n", "<br>"
43	)
44	return None	×
45
46
47	class AWSTextractParsedSOPNStatus(models.TextChoices):	1✔
48	NOT_STARTED = "NOT_STARTED", "Not Started"	1✔
49	SUCCEEDED = "SUCCEEDED", "Succeeded"	1✔
50	IN_PROGRESS = "IN_PROGRESS", "In Progress"	1✔
51	FAILED = "FAILED", "Failed"	1✔
52	PARTIAL_SUCCESS = "PARTIAL_SUCCESS", "Partial Success"	1✔
53
54
55	def AWSTextractParsedSOPNImage_upload_path(instance, filename):	1✔
NEW 56	return f"AWSTextractParsedSOPNImages/{instance.pk}/{filename}"	×
57
58
59	class AWSTextractParsedSOPNImage(models.Model):	1✔
60	image = models.ImageField(upload_to=AWSTextractParsedSOPNImage_upload_path)	1✔
61	parsed_sopn = models.ForeignKey(	1✔
62	"AWSTextractParsedSOPN", on_delete=models.CASCADE, related_name="images"
63	)
64
65	@staticmethod	1✔
66	def pil_to_content_image(pil_image, filename):	1✔
67	"""
68	Takes a PIL image and returns an ImageFile
69	"""
NEW 70	image_io = BytesIO()	×
NEW 71	pil_image.save(image_io, format="PNG")	×
NEW 72	return ImageFile(image_io, name=filename)	×
73
74
75	class AWSTextractParsedSOPN(TimeStampedModel):	1✔
76	"""
77	A model for storing the parsed data out of a PDF
78	where data has been extracted using AWS Textract.
79
80	"""
81
82	sopn = models.OneToOneField(	1✔
83	"official_documents.OfficialDocument", on_delete=models.CASCADE
84	)
85	job_id = models.CharField(max_length=100)	1✔
86	raw_data = models.TextField()	1✔
87	raw_data_type = models.CharField(max_length=255, default="pandas")	1✔
88	parsed_data = models.TextField(null=True)	1✔
89	status = models.CharField(	1✔
90	max_length=255,
91	choices=AWSTextractParsedSOPNStatus.choices,
92	default=AWSTextractParsedSOPNStatus.NOT_STARTED,
93	)
94
95	@property	1✔
96	def as_pandas(self):	1✔
97	import pandas	1✔
98
99	pandas.set_option("display.max_colwidth", None)	1✔
100	return pandas.DataFrame.from_dict(json.loads(self.parsed_data))	1✔
101
102	def parse_raw_data(self):	1✔
103	"""
104	Use AWS Textractor to convert the raw JSON response
105	to a Pandas dataframe (and then to JSON for storing
106	on the model).
107
108	It's possible for Textract to detect more than one
109	table per document. Default to merging them all
110	into one table.
111	:return:
112	"""
113	# User Textractor to parse the raw JSON
114	parsed = parse(json.loads(self.raw_data))	1✔
115	# Store all data frames in a list
116	frames = []	1✔
117
118	# Table headers that we've seen
119	for table in parsed.tables:	1✔
120	# Get the pandas version of the table
121	df = table.to_pandas()	1✔
122	frames.append(df)	1✔
123
124	# Merge all the dataframes
125	df = concat(	1✔
126	frames,
127	ignore_index=True,
128	)
129	self.parsed_data = df.to_json()	1✔
130
131	def as_textractor_document(self):	1✔
NEW 132	return response_parser.parse(json.loads(self.raw_data))	×

DemocracyClub / yournextrepresentative / 756047b3-833c-4f2e-bcef-27a788f4472f

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous