4432391286

Build Type

push

github

Committed by GitHub

Commit Message

Fix pre-commit/typecheck errors (#70)

Run Details

1132 of 1195 branches covered (94.73%)

Branch coverage included in aggregate %.

9 of 9 new or added lines in 2 files covered. (100.0%)

11 existing lines in 2 files now uncovered.

1519 of 1558 relevant lines covered (97.5%)

5.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.8

/src/ini2toml/transformations.py

"""Reusable value and type casting transformations"""
import warnings
from collections.abc import MutableMapping
from functools import reduce, wraps
from typing import (
    Any,
    Callable,
    List,
    Optional,
    Sequence,
    Tuple,
    TypeVar,
    Union,
    cast,
    overload,
)

from .types import Commented, CommentedKV, CommentedList

CP = ("#", ";")
"""Default Comment Prefixes"""

S = TypeVar("S")
T = TypeVar("T")
U = TypeVar("U")
V = TypeVar("V")
X = TypeVar("X")
Y = TypeVar("Y")
M = TypeVar("M", bound=MutableMapping)
KV = Tuple[str, T]

FN = Callable[[X], Y]

Scalar = Union[int, float, bool, str]  # TODO: missing time and datetime
"""Simple data types with TOML correspondence"""

CoerceFn = Callable[[str], T]
"""Functions that know how to parser/coerce string values into different types"""

Transformation = Union[Callable[[str], Any], Callable[[M], M]]
"""There are 2 main types of transformation:

- The first one is a simple transformation that processes a string value (coming from an
  option in the original CFG/INI file) into a value with an equivalent TOML data type.
  For example: transforming ``"2"`` (string) into ``2`` (integer).
- The second one tries to preserve metadata (such as comments) from the original CFG/INI
  file. This kind of transformation processes a string value into an intermediary
  representation (e.g. :obj:`Commented`, :obj:`CommentedList`, obj:`CommentedKV`)
  that needs to be properly handled before adding to the TOML document.

In a higher level we can also consider an ensemble of transformations that transform an
entire table of the TOML document.
"""

TF = TypeVar("TF", bound=Transformation)


# ---- Simple value processors ----
# These functions return plain objects, that can be directly added to the TOML document


def noop(x: T) -> T:
    return x


def is_true(value: str) -> bool:
    value = value.lower()
    return value in ("true", "1", "yes", "on")


def is_false(value: str) -> bool:
    value = value.lower()
    return value in ("false", "0", "no", "off", "none", "null", "nil")


def is_float(value: str) -> bool:
    cleaned = value.lower().lstrip("+-").replace(".", "").replace("_", "")
    return cleaned.isdecimal() and value.count(".") <= 1 or cleaned in ("inf", "nan")


def coerce_bool(value: str) -> bool:
    if is_true(value):
        return True
    if is_false(value):
        return False
    raise ValueError(f"{value!r} cannot be converted to boolean")


def coerce_scalar(value: str) -> Scalar:
    """Try to convert the given string to a proper "scalar" type (e.g. integer, float,
    bool, ...) with an direct TOML equivalent.
    If the conversion is unknown or not possible, it will return the same input value
    (as string).

    .. note:: This function "guesses" the value type based in heuristics and/or regular
       expressions, therefore there is no guarantee the output has the same type as
       intended by the original author.

    .. note:: Currently date/time-related types are not supported.
    """
    value = value.strip()
    if value.isdecimal():
        return int(value)
    if is_float(value):
        return float(value)
    if is_true(value):
        return True
    elif is_false(value):
        return False
    # Do we need this? Or is there a better way? How about time objects
    # > try:
    # >     return datetime.fromisoformat(value)
    # > except ValueError:
    # >     pass
    return value


def kebab_case(field: str) -> str:
    return field.lower().replace("_", "-")


def deprecated(
    name: str, fn: TF = noop, instead: str = ""  # type: ignore[assignment]
) -> TF:
    """Wrapper around the ``fn`` transformation to warn user about deprecation."""
    extra = f". Use {instead!r} instead" if instead else ""

    @wraps(fn)
    def _fn(*args, **kwargs):
        msg = f"{name!r} is deprecated{extra}"
        warnings.warn(msg, DeprecationWarning, stacklevel=2)
        return fn(*args, **kwargs)

    return cast(TF, _fn)


# ---- Complex value processors ----
# These functions return an intermediate representation of the value,
# that need `apply` to be added to a container


@overload
def split_comment(value: str, *, comment_prefixes=CP) -> Commented[str]:
    ...


@overload
def split_comment(
    value: str, coerce_fn: CoerceFn[T], comment_prefixes=CP
) -> Commented[T]:
    ...


def split_comment(value, coerce_fn=noop, comment_prefixes=CP):
    if not isinstance(value, str):
        return value
    value = value.strip()
    prefixes = [p for p in comment_prefixes if p in value]

    # We just process inline comments for single line options
    if not prefixes or len(value.splitlines()) > 1:
        return Commented(coerce_fn(value))

    if any(value.startswith(p) for p in comment_prefixes):
        return Commented(comment=_strip_comment(value, comment_prefixes))

    prefix = prefixes[0]  # We can only analyse one...
    value, _, cmt = value.partition(prefix)
    return Commented(coerce_fn(value.strip()), _strip_comment(cmt, comment_prefixes))


def split_scalar(value: str, *, comment_prefixes=CP) -> Commented[Scalar]:
    return split_comment(value, coerce_scalar, comment_prefixes)


@overload
def split_list(
    value: str,
    sep: str = ",",
    *,
    subsplit_dangling=True,
    comment_prefixes=CP,
    force_multiline=False,
) -> CommentedList[str]:
    ...


@overload
def split_list(
    value: str,
    sep: str = ",",
    *,
    coerce_fn: CoerceFn[T],
    subsplit_dangling=True,
    comment_prefixes=CP,
    force_multiline=False,
) -> CommentedList[T]:
    ...


@overload
def split_list(
    value: str,
    sep: str,
    coerce_fn: CoerceFn[T],
    subsplit_dangling=True,
    comment_prefixes=CP,
    force_multiline=False,
) -> CommentedList[T]:
    ...


def split_list(
    value,
    sep=",",
    coerce_fn=noop,
    subsplit_dangling=True,
    comment_prefixes=CP,
    force_multiline=False,
):
    """Value encoded as a (potentially) dangling list values separated by ``sep``.

    This function will first try to split the value by lines (dangling list) using
    :func:`str.splitlines`. Then, if ``subsplit_dangling=True``, it will split each line
    using ``sep``. As a result a list of items is obtained.
    For each item in this list ``coerce_fn`` is applied.
    """
    if not isinstance(value, str):
        return value
    comment_prefixes = [p for p in comment_prefixes if sep not in p]

    values = value.strip().splitlines()
    if not subsplit_dangling and (len(values) > 1 or force_multiline):
        sep += "\n"  # force a pattern that cannot be found in a split line

    def _split(line: str) -> list:
        return [coerce_fn(v.strip()) for v in line.split(sep) if v]

    return CommentedList([split_comment(v, _split, comment_prefixes) for v in values])


@overload
def split_kv_pairs(
    value: str,
    key_sep: str = "=",
    *,
    pair_sep=",",
    subsplit_dangling=True,
    comment_prefixes=CP,
) -> CommentedKV[str]:
    ...


@overload
def split_kv_pairs(
    value: str,
    key_sep: str = "=",
    *,
    coerce_fn: CoerceFn[T],
    pair_sep=",",
    subsplit_dangling=True,
    comment_prefixes=CP,
) -> CommentedKV[T]:
    ...


@overload
def split_kv_pairs(
    value: str,
    key_sep: str,
    coerce_fn: CoerceFn[T],
    pair_sep=",",
    subsplit_dangling=True,
    comment_prefixes=CP,
) -> CommentedKV[T]:
    ...


def split_kv_pairs(
    value,
    key_sep="=",
    coerce_fn=noop,
    pair_sep=",",
    subsplit_dangling=True,
    comment_prefixes=CP,
):
    """Value encoded as a (potentially) dangling list of key-value pairs.

    This function will first try to split the value by lines (dangling list) using
    :func:`str.splitlines`. Then, if ``subsplit_dangling=True``, it will split each line
    using ``pair_sep``. As a result a list of key-value pairs is obtained.
    For each item in this list, the key is separated from the value by ``key_sep``.
    ``coerce_fn`` is used to convert the value in each pair.
    """
    prefixes = [p for p in comment_prefixes if key_sep not in p and pair_sep not in p]

    values = value.strip().splitlines()
    if not subsplit_dangling and len(values) > 1:
        pair_sep += "\n"  # force a pattern that cannot be found in a split line

    def _split_kv(line: str) -> List[KV]:
        pairs = (
            item.split(key_sep, maxsplit=1)
            for item in line.strip().split(pair_sep)
            if key_sep in item
        )
        return [(k.strip(), coerce_fn(v.strip())) for k, v in pairs]

    return CommentedKV([split_comment(v, _split_kv, prefixes) for v in values])


# ---- Public Helpers ----


def remove_prefixes(text: str, prefixes: Sequence[str]):
    text = text.strip()
    for prefix in prefixes:
        if prefix and text.startswith(prefix):
            return text[len(prefix) :].strip()
    return text


def apply(x, fn):
    """Useful to reduce over a list of functions"""
    return fn(x)


@overload
def pipe(fn1: FN[S, T], fn2: FN[T, U]) -> FN[S, U]:
    ...


@overload
def pipe(fn1: FN[S, T], fn2: FN[T, U], fn3: FN[U, V]) -> FN[S, V]:
    ...


@overload
def pipe(fn1: FN[S, T], fn2: FN[T, U], fn3: FN[U, V], fn4: FN[V, X]) -> FN[S, X]:
    ...


@overload
def pipe(
    fn1: FN[S, T], fn2: FN[T, U], fn3: FN[U, V], fn4: FN[V, X], fn5: FN[X, Y]
) -> FN[S, Y]:
    ...


@overload
def pipe(
    fn1: FN[S, T],
    fn2: FN[T, U],
    fn3: FN[U, V],
    fn4: FN[V, X],
    fn5: FN[X, Y],
    *fn: FN[Y, Y],
) -> FN[S, Y]:
    ...


def pipe(*fns):
    """Compose 1-argument functions respecting the sequence they should be applied:

    .. code-block:: python

        pipe(fn1, fn2, fn3, ..., fnN)(x) == fnN(...(fn3(fn2(fn1(x)))))
    """
    return lambda x: reduce(apply, fns, x)


# ---- Private Helpers ----


def _strip_comment(msg: Optional[str], prefixes: Sequence[str] = CP) -> Optional[str]:
    if not msg:
        return None
    return remove_prefixes(msg, prefixes)

1	"""Reusable value and type casting transformations"""	6✔
2	import warnings	6✔
3	from collections.abc import MutableMapping	6✔
4	from functools import reduce, wraps	6✔
5	from typing import (	6✔
6	Any,
7	Callable,
8	List,
9	Optional,
10	Sequence,
11	Tuple,
12	TypeVar,
13	Union,
14	cast,
15	overload,
16	)
17
18	from .types import Commented, CommentedKV, CommentedList	6✔
19
20	CP = ("#", ";")	6✔
21	"""Default Comment Prefixes"""	4✔
22
23	S = TypeVar("S")	6✔
24	T = TypeVar("T")	6✔
25	U = TypeVar("U")	6✔
26	V = TypeVar("V")	6✔
27	X = TypeVar("X")	6✔
28	Y = TypeVar("Y")	6✔
29	M = TypeVar("M", bound=MutableMapping)	6✔
30	KV = Tuple[str, T]	6✔
31
32	FN = Callable[[X], Y]	6✔
33
34	Scalar = Union[int, float, bool, str] # TODO: missing time and datetime	6✔
35	"""Simple data types with TOML correspondence"""	4✔
36
37	CoerceFn = Callable[[str], T]	6✔
38	"""Functions that know how to parser/coerce string values into different types"""	4✔
39
40	Transformation = Union[Callable[[str], Any], Callable[[M], M]]	6✔
41	"""There are 2 main types of transformation:	4✔
42
43	- The first one is a simple transformation that processes a string value (coming from an
44	option in the original CFG/INI file) into a value with an equivalent TOML data type.
45	For example: transforming ``"2"`` (string) into ``2`` (integer).
46	- The second one tries to preserve metadata (such as comments) from the original CFG/INI
47	file. This kind of transformation processes a string value into an intermediary
48	representation (e.g. :obj:`Commented`, :obj:`CommentedList`, obj:`CommentedKV`)
49	that needs to be properly handled before adding to the TOML document.
50
51	In a higher level we can also consider an ensemble of transformations that transform an
52	entire table of the TOML document.
53	"""
54
55	TF = TypeVar("TF", bound=Transformation)	6✔
56
57
58	# ---- Simple value processors ----
59	# These functions return plain objects, that can be directly added to the TOML document
60
61
62	def noop(x: T) -> T:	6✔
63	return x	6✔
64
65
66	def is_true(value: str) -> bool:	6✔
67	value = value.lower()	6✔
68	return value in ("true", "1", "yes", "on")	6✔
69
70
71	def is_false(value: str) -> bool:	6✔
72	value = value.lower()	6✔
73	return value in ("false", "0", "no", "off", "none", "null", "nil")	6✔
74
75
76	def is_float(value: str) -> bool:	6✔
77	cleaned = value.lower().lstrip("+-").replace(".", "").replace("_", "")	6✔
78	return cleaned.isdecimal() and value.count(".") <= 1 or cleaned in ("inf", "nan")	6✔
79
80
81	def coerce_bool(value: str) -> bool:	6✔
82	if is_true(value):	6✔
83	return True	6✔
84	if is_false(value):	6✔
85	return False	6✔
86	raise ValueError(f"{value!r} cannot be converted to boolean")	6✔
87
88
89	def coerce_scalar(value: str) -> Scalar:	6✔
90	"""Try to convert the given string to a proper "scalar" type (e.g. integer, float,
91	bool, ...) with an direct TOML equivalent.
92	If the conversion is unknown or not possible, it will return the same input value
93	(as string).
94
95	.. note:: This function "guesses" the value type based in heuristics and/or regular
96	expressions, therefore there is no guarantee the output has the same type as
97	intended by the original author.
98
99	.. note:: Currently date/time-related types are not supported.
100	"""
101	value = value.strip()	6✔
102	if value.isdecimal():	6✔
103	return int(value)	6✔
104	if is_float(value):	6✔
105	return float(value)	6✔
106	if is_true(value):	6✔
107	return True	6✔
108	elif is_false(value):	6✔
109	return False	6✔
110	# Do we need this? Or is there a better way? How about time objects
111	# > try:
112	# > return datetime.fromisoformat(value)
113	# > except ValueError:
114	# > pass
115	return value	6✔
116
117
118	def kebab_case(field: str) -> str:	6✔
119	return field.lower().replace("_", "-")	6✔
120
121
122	def deprecated(	6✔
123	name: str, fn: TF = noop, instead: str = "" # type: ignore[assignment]
124	) -> TF:
125	"""Wrapper around the ``fn`` transformation to warn user about deprecation."""
126	extra = f". Use {instead!r} instead" if instead else ""	6✔
127
128	@wraps(fn)	6✔
129	def _fn(args, *kwargs):	6✔
130	msg = f"{name!r} is deprecated{extra}"	6✔
131	warnings.warn(msg, DeprecationWarning, stacklevel=2)	6✔
132	return fn(args, *kwargs)	6✔
133
134	return cast(TF, _fn)	6✔
135
136
137	# ---- Complex value processors ----
138	# These functions return an intermediate representation of the value,
139	# that need `apply` to be added to a container
140
141
142	@overload	6✔
143	def split_comment(value: str, *, comment_prefixes=CP) -> Commented[str]:	6✔
144	...
145
146
147	@overload	6✔
148	def split_comment(	6✔
149	value: str, coerce_fn: CoerceFn[T], comment_prefixes=CP
150	) -> Commented[T]:
151	...
152
153
154	def split_comment(value, coerce_fn=noop, comment_prefixes=CP):	6✔
155	if not isinstance(value, str):	6✔
UNCOV 156	return value	×
157	value = value.strip()	6✔
158	prefixes = [p for p in comment_prefixes if p in value]	6✔
159
160	# We just process inline comments for single line options
161	if not prefixes or len(value.splitlines()) > 1:	6✔
162	return Commented(coerce_fn(value))	6✔
163
164	if any(value.startswith(p) for p in comment_prefixes):	6✔
165	return Commented(comment=_strip_comment(value, comment_prefixes))	6✔
166
167	prefix = prefixes[0] # We can only analyse one...	6✔
168	value, _, cmt = value.partition(prefix)	6✔
169	return Commented(coerce_fn(value.strip()), _strip_comment(cmt, comment_prefixes))	6✔
170
171
172	def split_scalar(value: str, *, comment_prefixes=CP) -> Commented[Scalar]:	6✔
173	return split_comment(value, coerce_scalar, comment_prefixes)	6✔
174
175
176	@overload	6✔
177	def split_list(	6✔
178	value: str,
179	sep: str = ",",
180	*,
181	subsplit_dangling=True,
182	comment_prefixes=CP,
183	force_multiline=False,
184	) -> CommentedList[str]:
185	...
186
187
188	@overload	6✔
189	def split_list(	6✔
190	value: str,
191	sep: str = ",",
192	*,
193	coerce_fn: CoerceFn[T],
194	subsplit_dangling=True,
195	comment_prefixes=CP,
196	force_multiline=False,
197	) -> CommentedList[T]:
198	...
199
200
201	@overload	6✔
202	def split_list(	6✔
203	value: str,
204	sep: str,
205	coerce_fn: CoerceFn[T],
206	subsplit_dangling=True,
207	comment_prefixes=CP,
208	force_multiline=False,
209	) -> CommentedList[T]:
210	...
211
212
213	def split_list(	6✔
214	value,
215	sep=",",
216	coerce_fn=noop,
217	subsplit_dangling=True,
218	comment_prefixes=CP,
219	force_multiline=False,
220	):
221	"""Value encoded as a (potentially) dangling list values separated by ``sep``.
222
223	This function will first try to split the value by lines (dangling list) using
224	:func:`str.splitlines`. Then, if ``subsplit_dangling=True``, it will split each line
225	using ``sep``. As a result a list of items is obtained.
226	For each item in this list ``coerce_fn`` is applied.
227	"""
228	if not isinstance(value, str):	6✔
UNCOV 229	return value	×
230	comment_prefixes = [p for p in comment_prefixes if sep not in p]	6✔
231
232	values = value.strip().splitlines()	6✔
233	if not subsplit_dangling and (len(values) > 1 or force_multiline):	6✔
234	sep += "\n" # force a pattern that cannot be found in a split line	6✔
235
236	def _split(line: str) -> list:	6✔
237	return [coerce_fn(v.strip()) for v in line.split(sep) if v]	6✔
238
239	return CommentedList([split_comment(v, _split, comment_prefixes) for v in values])	6✔
240
241
242	@overload	6✔
243	def split_kv_pairs(	6✔
244	value: str,
245	key_sep: str = "=",
246	*,
247	pair_sep=",",
248	subsplit_dangling=True,
249	comment_prefixes=CP,
250	) -> CommentedKV[str]:
251	...
252
253
254	@overload	6✔
255	def split_kv_pairs(	6✔
256	value: str,
257	key_sep: str = "=",
258	*,
259	coerce_fn: CoerceFn[T],
260	pair_sep=",",
261	subsplit_dangling=True,
262	comment_prefixes=CP,
263	) -> CommentedKV[T]:
264	...
265
266
267	@overload	6✔
268	def split_kv_pairs(	6✔
269	value: str,
270	key_sep: str,
271	coerce_fn: CoerceFn[T],
272	pair_sep=",",
273	subsplit_dangling=True,
274	comment_prefixes=CP,
275	) -> CommentedKV[T]:
276	...
277
278
279	def split_kv_pairs(	6✔
280	value,
281	key_sep="=",
282	coerce_fn=noop,
283	pair_sep=",",
284	subsplit_dangling=True,
285	comment_prefixes=CP,
286	):
287	"""Value encoded as a (potentially) dangling list of key-value pairs.
288
289	This function will first try to split the value by lines (dangling list) using
290	:func:`str.splitlines`. Then, if ``subsplit_dangling=True``, it will split each line
291	using ``pair_sep``. As a result a list of key-value pairs is obtained.
292	For each item in this list, the key is separated from the value by ``key_sep``.
293	``coerce_fn`` is used to convert the value in each pair.
294	"""
295	prefixes = [p for p in comment_prefixes if key_sep not in p and pair_sep not in p]	6✔
296
297	values = value.strip().splitlines()	6✔
298	if not subsplit_dangling and len(values) > 1:	6✔
299	pair_sep += "\n" # force a pattern that cannot be found in a split line	6✔
300
301	def _split_kv(line: str) -> List[KV]:	6✔
302	pairs = (	6✔
303	item.split(key_sep, maxsplit=1)
304	for item in line.strip().split(pair_sep)
305	if key_sep in item
306	)
307	return [(k.strip(), coerce_fn(v.strip())) for k, v in pairs]	6✔
308
309	return CommentedKV([split_comment(v, _split_kv, prefixes) for v in values])	6✔
310
311
312	# ---- Public Helpers ----
313
314
315	def remove_prefixes(text: str, prefixes: Sequence[str]):	6✔
316	text = text.strip()	6✔
317	for prefix in prefixes:	6✔
318	if prefix and text.startswith(prefix):	6✔
319	return text[len(prefix) :].strip()	6✔
320	return text	6✔
321
322
323	def apply(x, fn):	6✔
324	"""Useful to reduce over a list of functions"""
325	return fn(x)	6✔
326
327
328	@overload	6✔
329	def pipe(fn1: FN[S, T], fn2: FN[T, U]) -> FN[S, U]:	6✔
330	...
331
332
333	@overload	6✔
334	def pipe(fn1: FN[S, T], fn2: FN[T, U], fn3: FN[U, V]) -> FN[S, V]:	6✔
335	...
336
337
338	@overload	6✔
339	def pipe(fn1: FN[S, T], fn2: FN[T, U], fn3: FN[U, V], fn4: FN[V, X]) -> FN[S, X]:	6✔
340	...
341
342
343	@overload	6✔
344	def pipe(	6✔
345	fn1: FN[S, T], fn2: FN[T, U], fn3: FN[U, V], fn4: FN[V, X], fn5: FN[X, Y]
346	) -> FN[S, Y]:
347	...
348
349
350	@overload	6✔
351	def pipe(	6✔
352	fn1: FN[S, T],
353	fn2: FN[T, U],
354	fn3: FN[U, V],
355	fn4: FN[V, X],
356	fn5: FN[X, Y],
357	*fn: FN[Y, Y],
358	) -> FN[S, Y]:
359	...
360
361
362	def pipe(*fns):	6✔
363	"""Compose 1-argument functions respecting the sequence they should be applied:
364
365	.. code-block:: python
366
367	pipe(fn1, fn2, fn3, ..., fnN)(x) == fnN(...(fn3(fn2(fn1(x)))))
368	"""
UNCOV 369	return lambda x: reduce(apply, fns, x)	×
370
371
372	# ---- Private Helpers ----
373
374
375	def _strip_comment(msg: Optional[str], prefixes: Sequence[str] = CP) -> Optional[str]:	6✔
376	if not msg:	6✔
UNCOV 377	return None	×
378	return remove_prefixes(msg, prefixes)	6✔

abravalheri / ini2toml / 4432391286

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous