49150ee2-a89b-4df8-9538-55cb06c06872

Committed 24 Aug 2025 11:07PM UTC coverage: 42.715% (+1.5%) from 41.169%

Build # 49150ee2-a89b-4df8-9538-55cb06c06872

Build Type

push

circleci

Committed by

mpasternak

Commit Message

Merge branch 'release/v202508.1207'

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

1119 existing lines in 82 files now uncovered.

16660 of 39003 relevant lines covered (42.71%)

1.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

36.24

src/import_common/normalization.py

from decimal import Decimal
from typing import Union

from django.core.exceptions import ValidationError
from numpy import isnan
from pathspec.util import normalize_file


def remove_trailing_interpunction(s: str) -> str:
    try:
        while s[-1] in ".,-":
            s = s[:-1]
    except IndexError:
        return ""

    return s


def remove_extra_spaces(s: str) -> str:
    while s.find("  ") >= 0:
        s = s.replace("  ", " ")
    s = s.strip()
    return s


def fix_spaces_before_dots_and_commas(s: str) -> str:
    while s.find(" .") >= 0 or s.find(" ,") >= 0:
        s = s.replace(" .", ". ").replace(" ,", ", ")
    return s


def normalize_first_name(s: str) -> str | None:
    if not isinstance(s, str) or s is None or s == "" or not s:
        return
    return remove_extra_spaces(s)


normalize_last_name = normalize_first_name
normalize_publisher = normalize_first_name


def normalize_boolean(s: Union[str, None, bool]) -> Union[bool, None]:
    if isinstance(s, bool):
        return s

    if not isinstance(s, str):
        return

    s = s.strip().lower()

    if s.lower() in ["tak", "prawda", "true", "t", "p"]:
        return True

    if s in ["false", "nie", "fałsz", "falsz", "f", "n"]:
        return False


def normalize_nullboleanfield(s: Union[str, None, bool]) -> Union[bool, None]:
    return normalize_boolean(s)


def normalize_skrot(s):
    if s is None:
        return
    return remove_extra_spaces(fix_spaces_before_dots_and_commas(s.lower()))


def normalize_tytul_naukowy(s):
    return normalize_skrot(s)


def normalize_title(s: str) -> str | None:
    if s is None or not s:
        return
    return remove_extra_spaces(s)


ONLINE_STR = "[online]"


def normalize_tytul_publikacji(s):
    if s is None:
        return
    ret = remove_trailing_interpunction(
        remove_extra_spaces(fix_spaces_before_dots_and_commas(s))
    )
    if ret.endswith(ONLINE_STR):
        ret = ret[: -len(ONLINE_STR)].strip()
    ret = ret.replace("\r", " ").replace("\n", " ")
    return remove_extra_spaces(ret)


def normalize_funkcja_autora(s: str) -> str:
    return normalize_skrot(s).lower()


def normalize_grupa_pracownicza(s: str):
    return normalize_skrot(s)


def normalize_wymiar_etatu(s: str):
    return normalize_skrot(s)


def normalize_nazwa_jednostki(s: str) -> str:
    return remove_extra_spaces(s.strip())


def normalize_nazwa_wydawcy(s: str) -> str:
    return remove_extra_spaces(s.strip())


def normalize_exception_reason(s: str) -> str:
    return remove_extra_spaces(s.replace("\n", " ").replace("\r\n", " "))


def normalize_tytul_zrodla(s):
    return remove_extra_spaces(fix_spaces_before_dots_and_commas(s))


def normalize_nazwa_dyscypliny(s):
    return remove_extra_spaces(fix_spaces_before_dots_and_commas(s))


def normalize_isbn(isbn: str) -> str | None:
    if isbn is None or not isinstance(isbn, str) or not isbn:
        return

    return isbn.replace(".", "").replace("-", "").replace(" ", "").strip()


normalize_issn = normalize_isbn


def normalize_kod_dyscypliny(k):
    if k is None:
        return None

    if k.endswith("_0"):
        k = k[:-2]
        return f"{k[0]}.{int(k[1:])}"
    if k.find(".") >= 0:
        return k

    if len(k) == 4 and k[0] in "123456789":
        # 1001 -> 10.1
        return f"{k[0]}{k[1]}.{int(k[2:])}"

    return f"{k[0]}.{int(k[1:])}"


def normalize_public_uri(public_uri):
    if public_uri is not None:
        return public_uri.strip()


def normalize_doi(s: str) -> None | str:
    """
    https://www.doi.org/doi_handbook/2_Numbering.html#2.4
    """

    if s is None:
        return

    s = s.strip()

    if not s:
        return

    return (
        s.lower()
        .replace("http://", "")
        .replace("https://", "")
        .replace("dx.doi.org/", "")
        .replace("doi.org/", "")
    )


def normalize_filename(s: str) -> str:
    s = normalize_file(s).replace(" ", "_").replace(",", "_")

    while s.find("__") != -1:
        s = s.replace("__", "_")
    return s


def normalize_orcid(s: str) -> str | None:
    if not isinstance(s, str) or s is None or s == "" or not s:
        return
    return (
        s.strip()
        .lower()
        .replace("http://orcid.org/", "")
        .replace("https://orcid.org/", "")
        .upper()
    )


def normalize_nulldecimalfield(probably_decimal):
    if probably_decimal is None:
        return

    try:
        if probably_decimal is not None and isnan(probably_decimal):
            return
    except TypeError:
        raise ValidationError(
            f"Nie mogę skonwertować liczby w formacie {probably_decimal} na typ Decimal"
        )

    try:
        # float() zeby pozbyc sie np numpy.int64
        return Decimal(float(probably_decimal))
    except TypeError:
        raise ValidationError(
            f"Nie mogę skonwertować liczby w formacie {probably_decimal} na typ Decimal"
        )


def normalize_oplaty_za_publikacje(
    rekord,
    opl_pub_cost_free,
    opl_pub_research_potential,
    opl_pub_research_or_development_projects,
    opl_pub_other,
    opl_pub_amount,
):
    # Publikacja bezkosztowa
    rekord.opl_pub_cost_free = False
    if normalize_boolean(opl_pub_cost_free):
        rekord.opl_pub_cost_free = True

    # Środki finansowe o których mowa w artykule 365
    rekord.opl_pub_research_potential = False
    if normalize_boolean(opl_pub_research_potential):
        rekord.opl_pub_research_potential = True

    # Środki finansowe na realizację projektu
    rekord.opl_pub_research_or_development_projects = False
    if normalize_boolean(opl_pub_research_or_development_projects):
        rekord.opl_pub_research_or_development_projects = True

    # Inne srodki finansowe
    rekord.opl_pub_other = False
    if normalize_boolean(opl_pub_other):
        rekord.opl_pub_other = True

    # Kwota
    rekord.opl_pub_amount = normalize_nulldecimalfield(opl_pub_amount)

    rekord.clean()


def normalize_rekord_id(rekord_id):
    """Normalizuje z postaci (43, 43) do {43,43} zeby mozna odpytac baze dancyh."""
    if not isinstance(rekord_id, str):
        return

    if rekord_id is None:
        return

    if not rekord_id:
        return

    ret = rekord_id.replace("(", "{").replace(")", "}").strip()
    if not ret.startswith("{"):
        ret = "{" + ret
    if not ret.endswith("}"):
        ret += "}"

    return ret

1	from decimal import Decimal	3✔
2	from typing import Union	3✔
3
4	from django.core.exceptions import ValidationError	3✔
5	from numpy import isnan	3✔
6	from pathspec.util import normalize_file	3✔
7
8
9	def remove_trailing_interpunction(s: str) -> str:	3✔
10	try:	×
11	while s[-1] in ".,-":	×
12	s = s[:-1]	×
13	except IndexError:	×
14	return ""	×
15
16	return s	×
17
18
19	def remove_extra_spaces(s: str) -> str:	3✔
20	while s.find(" ") >= 0:	1✔
21	s = s.replace(" ", " ")	×
22	s = s.strip()	1✔
23	return s	1✔
24
25
26	def fix_spaces_before_dots_and_commas(s: str) -> str:	3✔
27	while s.find(" .") >= 0 or s.find(" ,") >= 0:	×
28	s = s.replace(" .", ". ").replace(" ,", ", ")	×
29	return s	×
30
31
32	def normalize_first_name(s: str) -> str \| None:	3✔
33	if not isinstance(s, str) or s is None or s == "" or not s:	1✔
34	return	×
35	return remove_extra_spaces(s)	1✔
36
37
38	normalize_last_name = normalize_first_name	3✔
39	normalize_publisher = normalize_first_name	3✔
40
41
42	def normalize_boolean(s: Union[str, None, bool]) -> Union[bool, None]:	3✔
43	if isinstance(s, bool):	×
44	return s	×
45
46	if not isinstance(s, str):	×
47	return	×
48
49	s = s.strip().lower()	×
50
51	if s.lower() in ["tak", "prawda", "true", "t", "p"]:	×
52	return True	×
53
54	if s in ["false", "nie", "fałsz", "falsz", "f", "n"]:	×
55	return False	×
56
57
58	def normalize_nullboleanfield(s: Union[str, None, bool]) -> Union[bool, None]:	3✔
59	return normalize_boolean(s)	×
60
61
62	def normalize_skrot(s):	3✔
63	if s is None:	×
64	return	×
65	return remove_extra_spaces(fix_spaces_before_dots_and_commas(s.lower()))	×
66
67
68	def normalize_tytul_naukowy(s):	3✔
69	return normalize_skrot(s)	×
70
71
72	def normalize_title(s: str) -> str \| None:	3✔
73	if s is None or not s:	×
74	return	×
75	return remove_extra_spaces(s)	×
76
77
78	ONLINE_STR = "[online]"	3✔
79
80
81	def normalize_tytul_publikacji(s):	3✔
82	if s is None:	×
83	return	×
84	ret = remove_trailing_interpunction(	×
85	remove_extra_spaces(fix_spaces_before_dots_and_commas(s))
86	)
87	if ret.endswith(ONLINE_STR):	×
88	ret = ret[: -len(ONLINE_STR)].strip()	×
89	ret = ret.replace("\r", " ").replace("\n", " ")	×
90	return remove_extra_spaces(ret)	×
91
92
93	def normalize_funkcja_autora(s: str) -> str:	3✔
94	return normalize_skrot(s).lower()	×
95
96
97	def normalize_grupa_pracownicza(s: str):	3✔
98	return normalize_skrot(s)	×
99
100
101	def normalize_wymiar_etatu(s: str):	3✔
102	return normalize_skrot(s)	×
103
104
105	def normalize_nazwa_jednostki(s: str) -> str:	3✔
106	return remove_extra_spaces(s.strip())	×
107
108
109	def normalize_nazwa_wydawcy(s: str) -> str:	3✔
110	return remove_extra_spaces(s.strip())	1✔
111
112
113	def normalize_exception_reason(s: str) -> str:	3✔
114	return remove_extra_spaces(s.replace("\n", " ").replace("\r\n", " "))	×
115
116
117	def normalize_tytul_zrodla(s):	3✔
118	return remove_extra_spaces(fix_spaces_before_dots_and_commas(s))	×
119
120
121	def normalize_nazwa_dyscypliny(s):	3✔
122	return remove_extra_spaces(fix_spaces_before_dots_and_commas(s))	×
123
124
125	def normalize_isbn(isbn: str) -> str \| None:	3✔
UNCOV 126	if isbn is None or not isinstance(isbn, str) or not isbn:	1✔
UNCOV 127	return	×
128
UNCOV 129	return isbn.replace(".", "").replace("-", "").replace(" ", "").strip()	1✔
130
131
132	normalize_issn = normalize_isbn	3✔
133
134
135	def normalize_kod_dyscypliny(k):	3✔
136	if k is None:	2✔
137	return None	×
138
139	if k.endswith("_0"):	2✔
140	k = k[:-2]	×
141	return f"{k[0]}.{int(k[1:])}"	×
142	if k.find(".") >= 0:	2✔
143	return k	2✔
144
145	if len(k) == 4 and k[0] in "123456789":	×
146	# 1001 -> 10.1
147	return f"{k[0]}{k[1]}.{int(k[2:])}"	×
148
149	return f"{k[0]}.{int(k[1:])}"	×
150
151
152	def normalize_public_uri(public_uri):	3✔
153	if public_uri is not None:	×
154	return public_uri.strip()	×
155
156
157	def normalize_doi(s: str) -> None \| str:	3✔
158	"""
159	https://www.doi.org/doi_handbook/2_Numbering.html#2.4
160	"""
161
162	if s is None:	1✔
163	return	×
164
165	s = s.strip()	1✔
166
167	if not s:	1✔
168	return	×
169
170	return (	1✔
171	s.lower()
172	.replace("http://", "")
173	.replace("https://", "")
174	.replace("dx.doi.org/", "")
175	.replace("doi.org/", "")
176	)
177
178
179	def normalize_filename(s: str) -> str:	3✔
180	s = normalize_file(s).replace(" ", "_").replace(",", "_")	×
181
182	while s.find("__") != -1:	×
183	s = s.replace("__", "_")	×
184	return s	×
185
186
187	def normalize_orcid(s: str) -> str \| None:	3✔
188	if not isinstance(s, str) or s is None or s == "" or not s:	1✔
189	return	×
190	return (	1✔
191	s.strip()
192	.lower()
193	.replace("http://orcid.org/", "")
194	.replace("https://orcid.org/", "")
195	.upper()
196	)
197
198
199	def normalize_nulldecimalfield(probably_decimal):	3✔
200	if probably_decimal is None:	×
201	return	×
202
203	try:	×
204	if probably_decimal is not None and isnan(probably_decimal):	×
205	return	×
206	except TypeError:	×
207	raise ValidationError(	×
208	f"Nie mogę skonwertować liczby w formacie {probably_decimal} na typ Decimal"
209	)
210
211	try:	×
212	# float() zeby pozbyc sie np numpy.int64
213	return Decimal(float(probably_decimal))	×
214	except TypeError:	×
215	raise ValidationError(	×
216	f"Nie mogę skonwertować liczby w formacie {probably_decimal} na typ Decimal"
217	)
218
219
220	def normalize_oplaty_za_publikacje(	3✔
221	rekord,
222	opl_pub_cost_free,
223	opl_pub_research_potential,
224	opl_pub_research_or_development_projects,
225	opl_pub_other,
226	opl_pub_amount,
227	):
228	# Publikacja bezkosztowa
229	rekord.opl_pub_cost_free = False	×
230	if normalize_boolean(opl_pub_cost_free):	×
231	rekord.opl_pub_cost_free = True	×
232
233	# Środki finansowe o których mowa w artykule 365
234	rekord.opl_pub_research_potential = False	×
235	if normalize_boolean(opl_pub_research_potential):	×
236	rekord.opl_pub_research_potential = True	×
237
238	# Środki finansowe na realizację projektu
239	rekord.opl_pub_research_or_development_projects = False	×
240	if normalize_boolean(opl_pub_research_or_development_projects):	×
241	rekord.opl_pub_research_or_development_projects = True	×
242
243	# Inne srodki finansowe
244	rekord.opl_pub_other = False	×
245	if normalize_boolean(opl_pub_other):	×
246	rekord.opl_pub_other = True	×
247
248	# Kwota
249	rekord.opl_pub_amount = normalize_nulldecimalfield(opl_pub_amount)	×
250
251	rekord.clean()	×
252
253
254	def normalize_rekord_id(rekord_id):	3✔
255	"""Normalizuje z postaci (43, 43) do {43,43} zeby mozna odpytac baze dancyh."""
256	if not isinstance(rekord_id, str):	×
257	return	×
258
259	if rekord_id is None:	×
260	return	×
261
262	if not rekord_id:	×
263	return	×
264
265	ret = rekord_id.replace("(", "{").replace(")", "}").strip()	×
266	if not ret.startswith("{"):	×
267	ret = "{" + ret	×
268	if not ret.endswith("}"):	×
269	ret += "}"	×
270
271	return ret	×

iplweb / bpp / 49150ee2-a89b-4df8-9538-55cb06c06872

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous