13215816783

Committed 08 Feb 2025 12:54PM UTC coverage: 83.824% (+0.2%) from 83.657%

Build # 13215816783

Build Type

push

github

Committed by

web-flow

Commit Message

CI test for makemessages (#557)

* CI test for makemessages

do we want something like this to check that if there are new messages that the po files are updated for translators?

this might create quite a bit of conflicts but then again those should be really easy to fix by just running makemessages again after rebasing...

* and now the ci should be green

* change how the message is built

* diff just po files

* fix stupid typo

* makemessages to get correct line numbers after master merge

* revert the test change for ci

* better way for the translation, fstrings would be nice but are not yet supported

* fixed a stupid typo

* also check for obsolete translations

* update readme to reflect that no obosolete translations should be left

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

27 existing lines in 2 files now uncovered.

2135 of 2547 relevant lines covered (83.82%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.94

/utils/dataimport.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import datetime
import re
from decimal import Decimal

from users.models import BankTransaction

from utils.businesslogic import BusinessLogic
from utils.holvitoolbox import HolviToolbox

logger = logging.getLogger(__name__)


class ParseError(Exception):
    """Raised when the data has invalid value"""

    pass


"""
This class contains data importers

 - Bank events. This is in form of bank transactions in and out of an
   account. Only one account is supported currently.
   Current implementation uses Nordea TITO format which seems to be
   some kind of standard.
   Use it if you can or write your own importer.

After data is imported, businesslogic to update all user data is
invoked. All functions return an object showing counts of imported,
already existing and failed items and array of failed rows. These
are displayed in the import UI.
"""


class DataImport:
    # Nordea TITO import. TITO spec here: https://www.nordea.fi/Images/146-84478/xml_tiliote.pdf
    # Note: this is the text-based TITO, not XML.
    # DEPRECATED: but the decorator is only available after python 3.13
    @staticmethod
    def import_tito(f):
        tito = f.read().decode("utf8")
        lines = tito.split("\n")
        imported = exists = error = 0
        failedrows = []
        for line in lines[1:]:
            logger.debug(f"import_tito - Processing line: {line}")
            try:
                if len(line) == 0 or line[0] != "T":
                    raise ParseError("Empty line or not starting with T")
                data_type = int(line[1:3])
                # Currently handle only type 10 (basic transaction data)
                if data_type == 10:
                    # normalize the line ending before checking the length
                    if int(line[3:6]) != 188 or len(line.rstrip("\r\n")) != 188:
                        raise ParseError("Length should be 188")
                    transaction_id = line[6:12]
                    archival_reference = line[12:30].strip()
                    if len(archival_reference) < 18:
                        raise ParseError(
                            "Archival reference number invalid: " + archival_reference
                        )
                    transaction_date = datetime.datetime.strptime(line[30:36], "%y%m%d")
                    transaction_type = int(line[48])
                    if transaction_type < 1 or transaction_type > 4:
                        raise ParseError(
                            "Transaction type should be between 1 and 4, not "
                            + str(transaction_type)
                        )
                    code = line[49:52]
                    message = line[52:87].strip()
                    if message == "Viitemaksu":
                        message = None
                    amount = int(line[87:106]) / 100
                    peer = line[108:143]
                    peer = peer.replace("[", "Ä")
                    peer = peer.replace("]", "Å")
                    peer = peer.replace("\\", "Ö")
                    # tito format has leading zeroes in reference number, strip them
                    reference = line[159:179].strip().lstrip("0")

                    # Done parsing, add the transaction

                    try:
                        # Archival reference with date is unique
                        BankTransaction.objects.get(
                            archival_reference=archival_reference, date=transaction_date
                        )
                        exists = exists + 1
                    except BankTransaction.DoesNotExist:
                        transaction = BankTransaction.objects.create(
                            archival_reference=archival_reference,
                            date=transaction_date,
                            amount=amount,
                            reference_number=reference,
                            sender=peer,
                            transaction_id=transaction_id,
                            code=code,
                        )
                        BusinessLogic.new_transaction(transaction)
                        imported = imported + 1
            except ParseError as err:
                logger.error(f"Error parsing data: {err}")
                error = error + 1
                failedrows.append(line + " (" + str(err) + ")")

        results = {
            "imported": imported,
            "exists": exists,
            "error": error,
            "failedrows": failedrows,
        }
        logger.info(f"Data imported: {results} - now updating all users..")

        BusinessLogic.update_all_users()

        return results

    # Holvi TITO import. TITO spec here: ???
    # Note: this is the XLSX-based TITO.
    @staticmethod
    def import_holvi(f):
        holvi = HolviToolbox.parse_account_statement(f)
        imported = exists = error = 0
        failedrows = []
        for line in holvi:
            logger.debug(f"import_holvi - Processing line: {line}")
            try:
                if len(line) == 0:
                    raise ParseError("Empty line")
                archival_reference = line["Filing ID"].strip()
                if len(archival_reference) < 32:
                    raise ParseError(
                        "Archival reference number invalid: " + archival_reference
                    )
                transaction_date = line["Date_parsed"]
                message = line["Message"].strip()
                if message in ["Viitemaksu", "None"]:
                    message = None
                amount = Decimal(line["Amount"])
                peer = line["Counterparty"]
                # holvi reference has leading zeroes, clean them up here also
                reference = line["Reference"].strip().lstrip("0")

                # Done parsing, add the transaction

                try:
                    # Archival reference with date is unique
                    BankTransaction.objects.get(
                        archival_reference=archival_reference, date=transaction_date
                    )
                    exists = exists + 1
                except BankTransaction.DoesNotExist:
                    transaction = BankTransaction.objects.create(
                        archival_reference=archival_reference,
                        date=transaction_date,
                        amount=amount,
                        reference_number=reference,
                        sender=peer,
                    )
                    BusinessLogic.new_transaction(transaction)
                    imported = imported + 1
            except ParseError as err:
                logger.error(f"Error parsing data: {err}")
                error = error + 1
                failedrows.append(line + " (" + str(err) + ")")

        results = {
            "imported": imported,
            "exists": exists,
            "error": error,
            "failedrows": failedrows,
        }
        logger.info(f"Data imported: {results} - now updating all users..")

        BusinessLogic.update_all_users()

        return results

    @staticmethod
    def import_nordigen(data):
        """
        Importer for nordigen api response https://ob.nordigen.com/api/docs#/accounts/accounts_transactions_retrieve

        Takes the whole nordigen response but parses only the "booked" section from it (pending transactions do not count)
        """
        booked = data["transactions"]["booked"]
        imported = exists = error = 0

        failedrows = []
        for one in booked:
            logger.debug(f"import_nordigen - Processing entry: {one}")

            try:
                # map nordigen fields to our fields
                # NOTE: it seems that some banks give different fields
                # see: https://nordigen.com/en/docs/account-information/output/transactions/
                archival_reference = one["transactionId"]
                transaction_date = datetime.datetime.strptime(
                    one["bookingDate"], "%Y-%m-%d"
                ).date()
                if one["transactionAmount"]["currency"] != "EUR":
                    raise Exception("Cannot handle different currencies")
                amount = one["transactionAmount"]["amount"]
                reference = one.get("entryReference") or ""
                # clear leading zeroes from real reference
                reference = reference.lstrip("0")
                sender = one.get("debtorName", "")
                message = one.get("additionalInformation", "")

                # If reference is empty, try extracting the first number from the message
                if not reference:
                    match = re.search(r"\b0*(\d+)\b", message)
                    if match:
                        reference = match.group(1)
                        message = message + " (reference extracted from message)"

                try:
                    # Archival reference with date is unique
                    BankTransaction.objects.get(
                        archival_reference=archival_reference, date=transaction_date
                    )
                    exists = exists + 1
                except BankTransaction.DoesNotExist:
                    transaction = BankTransaction.objects.create(
                        archival_reference=archival_reference,
                        date=transaction_date,
                        transaction_id=archival_reference,
                        amount=amount,
                        reference_number=reference,
                        sender=sender,
                        message=message,
                    )
                    BusinessLogic.new_transaction(transaction)
                    imported = imported + 1

            except Exception as e:
                logger.error(f"Error: {e}")
                error = error + 1
                failedrows.append(str(one) + " (" + str(e) + ")")

        results = {
            "imported": imported,
            "exists": exists,
            "error": error,
            "failedrows": failedrows,
        }

        logger.info(f"Data imported: {results} - now updating all users..")
        BusinessLogic.update_all_users()

        return results

1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	import logging	1✔
4	import datetime	1✔
5	import re	1✔
6	from decimal import Decimal	1✔
7
8	from users.models import BankTransaction	1✔
9
10	from utils.businesslogic import BusinessLogic	1✔
11	from utils.holvitoolbox import HolviToolbox	1✔
12
13	logger = logging.getLogger(__name__)	1✔
14
15
16	class ParseError(Exception):	1✔
17	"""Raised when the data has invalid value"""
18
19	pass	1✔
20
21
22	"""	1✔
23	This class contains data importers
24
25	- Bank events. This is in form of bank transactions in and out of an
26	account. Only one account is supported currently.
27	Current implementation uses Nordea TITO format which seems to be
28	some kind of standard.
29	Use it if you can or write your own importer.
30
31	After data is imported, businesslogic to update all user data is
32	invoked. All functions return an object showing counts of imported,
33	already existing and failed items and array of failed rows. These
34	are displayed in the import UI.
35	"""
36
37
38	class DataImport:	1✔
39	# Nordea TITO import. TITO spec here: https://www.nordea.fi/Images/146-84478/xml_tiliote.pdf
40	# Note: this is the text-based TITO, not XML.
41	# DEPRECATED: but the decorator is only available after python 3.13
42	@staticmethod	1✔
43	def import_tito(f):	1✔
44	tito = f.read().decode("utf8")	1✔
45	lines = tito.split("\n")	1✔
46	imported = exists = error = 0	1✔
47	failedrows = []	1✔
48	for line in lines[1:]:	1✔
49	logger.debug(f"import_tito - Processing line: {line}")	1✔
50	try:	1✔
51	if len(line) == 0 or line[0] != "T":	1✔
52	raise ParseError("Empty line or not starting with T")	1✔
53	data_type = int(line[1:3])	1✔
54	# Currently handle only type 10 (basic transaction data)
55	if data_type == 10:	1✔
56	# normalize the line ending before checking the length
57	if int(line[3:6]) != 188 or len(line.rstrip("\r\n")) != 188:	1✔
58	raise ParseError("Length should be 188")	1✔
59	transaction_id = line[6:12]	1✔
60	archival_reference = line[12:30].strip()	1✔
61	if len(archival_reference) < 18:	1✔
UNCOV 62	raise ParseError(	×
63	"Archival reference number invalid: " + archival_reference
64	)
65	transaction_date = datetime.datetime.strptime(line[30:36], "%y%m%d")	1✔
66	transaction_type = int(line[48])	1✔
67	if transaction_type < 1 or transaction_type > 4:	1✔
UNCOV 68	raise ParseError(	×
69	"Transaction type should be between 1 and 4, not "
70	+ str(transaction_type)
71	)
72	code = line[49:52]	1✔
73	message = line[52:87].strip()	1✔
74	if message == "Viitemaksu":	1✔
75	message = None	1✔
76	amount = int(line[87:106]) / 100	1✔
77	peer = line[108:143]	1✔
78	peer = peer.replace("[", "Ä")	1✔
79	peer = peer.replace("]", "Å")	1✔
80	peer = peer.replace("\\", "Ö")	1✔
81	# tito format has leading zeroes in reference number, strip them
82	reference = line[159:179].strip().lstrip("0")	1✔
83
84	# Done parsing, add the transaction
85
86	try:	1✔
87	# Archival reference with date is unique
88	BankTransaction.objects.get(	1✔
89	archival_reference=archival_reference, date=transaction_date
90	)
91	exists = exists + 1	1✔
92	except BankTransaction.DoesNotExist:	1✔
93	transaction = BankTransaction.objects.create(	1✔
94	archival_reference=archival_reference,
95	date=transaction_date,
96	amount=amount,
97	reference_number=reference,
98	sender=peer,
99	transaction_id=transaction_id,
100	code=code,
101	)
102	BusinessLogic.new_transaction(transaction)	1✔
103	imported = imported + 1	1✔
104	except ParseError as err:	1✔
105	logger.error(f"Error parsing data: {err}")	1✔
106	error = error + 1	1✔
107	failedrows.append(line + " (" + str(err) + ")")	1✔
108
109	results = {	1✔
110	"imported": imported,
111	"exists": exists,
112	"error": error,
113	"failedrows": failedrows,
114	}
115	logger.info(f"Data imported: {results} - now updating all users..")	1✔
116
117	BusinessLogic.update_all_users()	1✔
118
119	return results	1✔
120
121	# Holvi TITO import. TITO spec here: ???
122	# Note: this is the XLSX-based TITO.
123	@staticmethod	1✔
124	def import_holvi(f):	1✔
125	holvi = HolviToolbox.parse_account_statement(f)	1✔
126	imported = exists = error = 0	1✔
127	failedrows = []	1✔
128	for line in holvi:	1✔
129	logger.debug(f"import_holvi - Processing line: {line}")	1✔
130	try:	1✔
131	if len(line) == 0:	1✔
UNCOV 132	raise ParseError("Empty line")	×
133	archival_reference = line["Filing ID"].strip()	1✔
134	if len(archival_reference) < 32:	1✔
UNCOV 135	raise ParseError(	×
136	"Archival reference number invalid: " + archival_reference
137	)
138	transaction_date = line["Date_parsed"]	1✔
139	message = line["Message"].strip()	1✔
140	if message in ["Viitemaksu", "None"]:	1✔
141	message = None	1✔
142	amount = Decimal(line["Amount"])	1✔
143	peer = line["Counterparty"]	1✔
144	# holvi reference has leading zeroes, clean them up here also
145	reference = line["Reference"].strip().lstrip("0")	1✔
146
147	# Done parsing, add the transaction
148
149	try:	1✔
150	# Archival reference with date is unique
151	BankTransaction.objects.get(	1✔
152	archival_reference=archival_reference, date=transaction_date
153	)
154	exists = exists + 1	1✔
155	except BankTransaction.DoesNotExist:	1✔
156	transaction = BankTransaction.objects.create(	1✔
157	archival_reference=archival_reference,
158	date=transaction_date,
159	amount=amount,
160	reference_number=reference,
161	sender=peer,
162	)
163	BusinessLogic.new_transaction(transaction)	1✔
164	imported = imported + 1	1✔
165	except ParseError as err:	×
UNCOV 166	logger.error(f"Error parsing data: {err}")	×
UNCOV 167	error = error + 1	×
UNCOV 168	failedrows.append(line + " (" + str(err) + ")")	×
169
170	results = {	1✔
171	"imported": imported,
172	"exists": exists,
173	"error": error,
174	"failedrows": failedrows,
175	}
176	logger.info(f"Data imported: {results} - now updating all users..")	1✔
177
178	BusinessLogic.update_all_users()	1✔
179
180	return results	1✔
181
182	@staticmethod	1✔
183	def import_nordigen(data):	1✔
184	"""
185	Importer for nordigen api response https://ob.nordigen.com/api/docs#/accounts/accounts_transactions_retrieve
186
187	Takes the whole nordigen response but parses only the "booked" section from it (pending transactions do not count)
188	"""
189	booked = data["transactions"]["booked"]	1✔
190	imported = exists = error = 0	1✔
191
192	failedrows = []	1✔
193	for one in booked:	1✔
194	logger.debug(f"import_nordigen - Processing entry: {one}")	1✔
195
196	try:	1✔
197	# map nordigen fields to our fields
198	# NOTE: it seems that some banks give different fields
199	# see: https://nordigen.com/en/docs/account-information/output/transactions/
200	archival_reference = one["transactionId"]	1✔
201	transaction_date = datetime.datetime.strptime(	1✔
202	one["bookingDate"], "%Y-%m-%d"
203	).date()
204	if one["transactionAmount"]["currency"] != "EUR":	1✔
205	raise Exception("Cannot handle different currencies")	1✔
206	amount = one["transactionAmount"]["amount"]	1✔
207	reference = one.get("entryReference") or ""	1✔
208	# clear leading zeroes from real reference
209	reference = reference.lstrip("0")	1✔
210	sender = one.get("debtorName", "")	1✔
211	message = one.get("additionalInformation", "")	1✔
212
213	# If reference is empty, try extracting the first number from the message
214	if not reference:	1✔
215	match = re.search(r"\b0*(\d+)\b", message)	1✔
216	if match:	1✔
217	reference = match.group(1)	1✔
218	message = message + " (reference extracted from message)"	1✔
219
220	try:	1✔
221	# Archival reference with date is unique
222	BankTransaction.objects.get(	1✔
223	archival_reference=archival_reference, date=transaction_date
224	)
225	exists = exists + 1	1✔
226	except BankTransaction.DoesNotExist:	1✔
227	transaction = BankTransaction.objects.create(	1✔
228	archival_reference=archival_reference,
229	date=transaction_date,
230	transaction_id=archival_reference,
231	amount=amount,
232	reference_number=reference,
233	sender=sender,
234	message=message,
235	)
236	BusinessLogic.new_transaction(transaction)	1✔
237	imported = imported + 1	1✔
238
239	except Exception as e:	1✔
240	logger.error(f"Error: {e}")	1✔
241	error = error + 1	1✔
242	failedrows.append(str(one) + " (" + str(e) + ")")	1✔
243
244	results = {	1✔
245	"imported": imported,
246	"exists": exists,
247	"error": error,
248	"failedrows": failedrows,
249	}
250
251	logger.info(f"Data imported: {results} - now updating all users..")	1✔
252	BusinessLogic.update_all_users()	1✔
253
254	return results	1✔

TampereHacklab / mulysa / 13215816783

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous