• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

nbiotcloud / test2ref / 15621186087

12 Jun 2025 09:28PM UTC coverage: 95.0% (-5.0%) from 100.0%
15621186087

push

github

iccode17
Merge branch 'main' of github.com:nbiotcloud/test2ref into fix/cr2

10 of 13 new or added lines in 1 file covered. (76.92%)

4 existing lines in 1 file now uncovered.

133 of 140 relevant lines covered (95.0%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.0
/src/test2ref/__init__.py
1
#
2
# MIT License
3
#
4
# Copyright (c) 2024-2025 nbiotcloud
5
#
6
# Permission is hereby granted, free of charge, to any person obtaining a copy
7
# of this software and associated documentation files (the "Software"), to deal
8
# in the Software without restriction, including without limitation the rights
9
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
# copies of the Software, and to permit persons to whom the Software is
11
# furnished to do so, subject to the following conditions:
12
#
13
# The above copyright notice and this permission notice shall be included in all
14
# copies or substantial portions of the Software.
15
#
16
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
# SOFTWARE.
23
#
24
"""
25
Testing Against Learned Reference Data.
26

27
# Concept
28

29
A unit test creates files in a temporary folder `tmp_path`.
30
:any:`assert_refdata()` is called at the end of the test.
31

32
There are two modes:
33

34
* **Testing**: Test result in `tmp_path` is compared against a known reference.
35
  Any deviation in the files, causes a fail.
36
* **Learning**: The test result in `tmp_path` is taken as reference and is copied
37
  to the reference folder, which should be committed to version control and kept as
38
  reference.
39

40
The file `.test2ref` in the project root directory selects the operation mode.
41
If the file exists, **Learning Mode** is selected.
42
If the files does **not** exists, the **Testing Mode** is selected.
43

44
Next to that, stdout, stderr and logging can be included in the reference automatically.
45

46
# Minimal Example
47

48
!!! example
49

50
    ```python
51
    >>> def test_something(tmp_path, capsys):
52
    ...     (tmp_path / "file.txt").write_text("Hello Mars")
53
    ...     print("Hello World")
54
    ...     assert_refdata(test_something, tmp_path, capsys=capsys)
55

56
    ```
57

58
# API
59
"""
60

61
import os
1✔
62
import re
1✔
63
import site
1✔
64
import subprocess
1✔
65
from collections.abc import Callable, Iterable, Iterator
1✔
66
from pathlib import Path
1✔
67
from shutil import copytree, ignore_patterns, rmtree
1✔
68
from tempfile import TemporaryDirectory
1✔
69
from typing import Any, TypeAlias
1✔
70

71
from binaryornot.check import is_binary
1✔
72

73
PRJ_PATH = Path.cwd()
1✔
74

75
Search: TypeAlias = Path | str | re.Pattern
1✔
76
"""
1✔
77
Possible Search Pattern.
78

79
File System Path, string or regular expression.
80
"""
81

82
Replacements: TypeAlias = Iterable[tuple[Search, str]]
1✔
83
"""
1✔
84
Replacements - Pairs of Search Pattern and Things to be Replaced.
85
"""
86

87
StrReplacements: TypeAlias = Iterable[tuple[str, str]]
1✔
88
Excludes: TypeAlias = tuple[str, ...]
1✔
89

90

91
DEFAULT_REF_PATH: Path = PRJ_PATH / "tests" / "refdata"
1✔
92
DEFAULT_REF_UPDATE: bool = (PRJ_PATH / ".test2ref").exists()
1✔
93
DEFAULT_EXCLUDES: Excludes = ("__pycache__", ".tool_cache", ".cache")
1✔
94
CONFIG = {
1✔
95
    "ref_path": DEFAULT_REF_PATH,
96
    "ref_update": DEFAULT_REF_UPDATE,
97
    "excludes": DEFAULT_EXCLUDES,
98
}
99
ENCODING = "utf-8"
1✔
100
ENCODING_ERRORS = "surrogateescape"
1✔
101

102

103
def configure(
1✔
104
    ref_path: Path | None = None,
105
    ref_update: bool | None = None,
106
    excludes: Excludes | None = None,
107
    add_excludes: Excludes | None = None,
108
    rm_excludes: Excludes | None = None,
109
) -> None:
110
    """
111
    Configure.
112

113
    Keyword Args:
114
        ref_path: Path for reference files. "tests/refdata" by default
115
        ref_update: Update reference files. True by default if `.test2ref` file exists.
116
        excludes: Paths to be excluded in all runs.
117
        add_excludes: Additionally Excluded Files
118
        rm_excludes: Not Excluded Files
119
    """
120
    if ref_path is not None:
1✔
121
        CONFIG["ref_path"] = ref_path
1✔
122
    if ref_update is not None:
1✔
123
        CONFIG["ref_update"] = ref_update
1✔
124
    if excludes:
1✔
125
        CONFIG["excludes"] = excludes
1✔
126
    if add_excludes:
1✔
127
        CONFIG["excludes"] = (*CONFIG["excludes"], *add_excludes)
1✔
128
    if rm_excludes:
1✔
129
        CONFIG["excludes"] = tuple(exclude for exclude in CONFIG["excludes"] if exclude not in rm_excludes)  # type: ignore[attr-defined]
1✔
130

131

132
def assert_refdata(
1✔
133
    arg: Callable | Path,
134
    path: Path,
135
    capsys: Any = None,
136
    caplog: Any = None,
137
    replacements: Replacements | None = None,
138
    excludes: Iterable[str] | None = None,
139
    flavor: str = "",
140
) -> None:
141
    """
142
    Compare Output of `arg` generated at `path` with reference.
143

144
    Use `replacements` to mention things which vary from test to test.
145
    `path` and the project location are already replaced by default.
146

147
    Args:
148
        arg: Test Function or Path to reference data
149
        path: Path with generated files to be compared.
150

151
    Keyword Args:
152
        capsys: pytest `capsys` fixture. Include `stdout`/`stderr` too.
153
        caplog: pytest `caplog` fixture. Include logging output too.
154
        replacements: pairs of things to be replaced.
155
        excludes: Files and directories to be excluded.
156
        flavor: Flavor for different variants.
157

158
    !!! example "Minimal Example"
159

160
        ```python
161
        def test_example(tmp_path):
162
            (tmp_path / "file.txt").write_text("Content")
163
            assert_refdata(test_example, tmp_path)
164
        ```
165

166
    !!! example "Full Example"
167

168
        ```python
169
        import logging
170

171
        def test_example(tmp_path, capsys, caplog):
172
            (tmp_path / "file.txt").write_text("Content")
173

174
            # print on standard-output - captured by capsys
175
            print("Hello World")
176

177
            # logging - captured by caplog
178
            logging.getLogger().warning("test")
179

180
            assert_refdata(test_example, tmp_path, capsys=capsys, caplog=caplog)
181
        ```
182

183
    """
184
    ref_basepath: Path = CONFIG["ref_path"]  # type: ignore[assignment]
1✔
185
    if isinstance(arg, Path):
1✔
186
        ref_path = ref_basepath / arg
1✔
187
    else:
188
        ref_path = ref_basepath / arg.__module__ / arg.__name__
1✔
189
    if flavor:
1✔
190
        ref_path = ref_path / flavor
1✔
191
    ref_path.mkdir(parents=True, exist_ok=True)
1✔
192
    rplcs: Replacements = replacements or ()  # type: ignore[assignment]
1✔
193
    path_rplcs: StrReplacements = [(srch, rplc) for srch, rplc in rplcs if isinstance(srch, str)]
1✔
194
    sitepaths = (*site.getsitepackages(), site.getusersitepackages())
1✔
195

196
    gen_rplcs: Replacements = [
1✔
197
        *((Path(path) / "Lib" / "site-packages", "$SITE") for path in sitepaths),  # dirty hack for win
198
        *((Path(path), "$SITE") for path in sitepaths),
199
        (PRJ_PATH, "$PRJ"),
200
        (path, "$GEN"),
201
        *rplcs,
202
        (Path.home(), "$HOME"),
203
    ]
204
    gen_excludes: Excludes = (*CONFIG["excludes"], *(excludes or []))
1✔
205

206
    with TemporaryDirectory() as temp_dir:
1✔
207
        gen_path = Path(temp_dir)
1✔
208

209
        ignore = ignore_patterns(*gen_excludes)
1✔
210
        copytree(path, gen_path, dirs_exist_ok=True, ignore=ignore)
1✔
211

212
        _replace_path(gen_path, path_rplcs)
1✔
213

214
        if capsys:
1✔
215
            captured = capsys.readouterr()
1✔
216
            (gen_path / "stdout.txt").write_text(captured.out, encoding=ENCODING, errors=ENCODING_ERRORS)
1✔
217
            (gen_path / "stderr.txt").write_text(captured.err, encoding=ENCODING, errors=ENCODING_ERRORS)
1✔
218

219
        if caplog:
1✔
220
            logpath = gen_path / "logging.txt"
1✔
221
            with logpath.open("w", encoding=ENCODING, errors=ENCODING_ERRORS) as file:
1✔
222
                for record in caplog.records:
1✔
223
                    file.write(f"{record.levelname:7s}  {record.name}  {record.message}\n")
1✔
224
            caplog.clear()
1✔
225

226
        _remove_empty_dirs(gen_path)
1✔
227

228
        _replace_content(gen_path, gen_rplcs)
1✔
229

230
        if CONFIG["ref_update"]:
1✔
231
            rmtree(ref_path, ignore_errors=True)
1✔
232
            copytree(gen_path, ref_path)
1✔
233

234
        assert_paths(ref_path, gen_path, excludes=excludes)
1✔
235

236

237
def assert_paths(ref_path: Path, gen_path: Path, excludes: Iterable[str] | None = None) -> None:
1✔
238
    """
239
    Compare Output of `ref_path` with `gen_path`.
240

241
    Args:
242
        ref_path: Path with reference files to be compared.
243
        gen_path: Path with generated files to be compared.
244

245
    Keyword Args:
246
        excludes: Files and directories to be excluded.
247
    """
248
    diff_excludes: Excludes = (*CONFIG["excludes"], *(excludes or []))
1✔
249
    try:
1✔
250
        cmd = ["diff", "-ru", "--strip-trailing-cr", str(ref_path), str(gen_path)]
1✔
251
        for exclude in diff_excludes:
1✔
252
            cmd.extend(("--exclude", exclude))
1✔
253
        subprocess.run(cmd, check=True, capture_output=True)  # noqa: S603
1✔
254
    except subprocess.CalledProcessError as error:
1✔
255
        raise AssertionError(error.stdout.decode("utf-8")) from None
1✔
256

257

258
def _remove_empty_dirs(path: Path) -> None:
1✔
259
    """Remove Empty Directories within ``path``."""
260
    for sub_path in tuple(path.glob("**/*")):
1✔
261
        if not sub_path.exists() or not sub_path.is_dir():
1✔
262
            continue
1✔
263
        sub_dir = sub_path
1✔
264
        while sub_dir != path:
1✔
265
            is_empty = not any(sub_dir.iterdir())
1✔
266
            if is_empty:
1✔
267
                sub_dir.rmdir()
1✔
268
                sub_dir = sub_dir.parent
1✔
269
            else:
270
                break
1✔
271

272

273
def _replace_path(path: Path, replacements: StrReplacements) -> None:
1✔
274
    paths = [path]
1✔
275
    while paths:
1✔
276
        path = paths.pop()
1✔
277
        orig = name = path.name
1✔
278
        for srch, rplc in replacements:
1✔
279
            name = name.replace(srch, rplc)
1✔
280
        if orig != name:
1✔
281
            path = path.replace(path.with_name(name))
1✔
282
        if path.is_dir():
1✔
283
            paths.extend(path.iterdir())
1✔
284

285

286
def _replace_content(path: Path, replacements: Replacements) -> None:
1✔
287
    """Replace ``replacements`` for text files in ``path``."""
288
    # pre-compile regexs and create substitution functions
289
    regex_funcs = tuple(_create_regex_funcs(replacements))
1✔
290
    # search files and replace
291
    for sub_path in tuple(path.glob("**/*")):
1✔
292
        if not sub_path.is_file() or is_binary(str(sub_path)):
1✔
293
            continue
1✔
294
        content = sub_path.read_text(encoding=ENCODING, errors=ENCODING_ERRORS)
1✔
295
        total = 0
1✔
296
        for regex, func in regex_funcs:
1✔
297
            content, counts = regex.subn(func, content)
1✔
298
            total += counts
1✔
299
        if total:
1✔
300
            sub_path.write_text(content, encoding=ENCODING, errors=ENCODING_ERRORS)
1✔
301

302

303
def _create_regex_funcs(replacements: Replacements) -> Iterator[tuple[re.Pattern, Callable]]:
1✔
304
    """Create Regular Expression for `search`."""
305
    for search, replace in replacements:
1✔
306
        # already regex
307
        if isinstance(search, re.Pattern):
1✔
308
            yield search, _substitute_str(replace)
1✔
309

310
        # Path
311
        elif isinstance(search, Path):
1✔
312
            search_str = str(search)
1✔
313
            sep_esc = re.escape(os.sep)
1✔
314
            if os.altsep:
1✔
UNCOV
315
                doublesep = f"{os.sep}{os.sep}"
×
316

UNCOV
317
                search_repr = search_str.replace(os.sep, doublesep)
×
NEW
318
                doubleregex = rf"(?i){re.escape(search_repr)}([A-Za-z0-9\-_{sep_esc}{re.escape(os.altsep)}]*)"
×
UNCOV
319
                yield re.compile(f"{doubleregex}"), _substitute_path(replace, (doublesep, os.sep, os.altsep))
×
320

NEW
321
                altregex = rf"(?i){re.escape(search.as_posix())}([A-Za-z0-9\-_{sep_esc}{re.escape(os.altsep)}]*)"
×
UNCOV
322
                yield re.compile(f"{altregex}"), _substitute_path(replace, (os.sep, os.altsep))
×
NEW
323
                regex = rf"(?i){re.escape(search_str)}([A-Za-z0-9_{sep_esc}]*)"
×
324
            else:
325
                regex = rf"{re.escape(search_str)}([A-Za-z0-9_{sep_esc}]*)"
1✔
326
            yield re.compile(f"{regex}"), _substitute_path(replace, (os.sep,))
1✔
327

328
        # str
329
        else:
330
            yield re.compile(re.escape(search)), _substitute_str(replace)
1✔
331

332

333
def _substitute_path(replace: str, seps: tuple[str, ...] = ()):
1✔
334
    """Factory for Substitution Function."""
335

336
    def func(mat: re.Match) -> str:
1✔
337
        sub = mat.group(1)
1✔
338
        for sep in seps:
1✔
339
            sub = sub.replace(sep, "/")
1✔
340
        return f"{replace}{sub}"
1✔
341

342
    return func
1✔
343

344

345
def _substitute_str(replace: str):
1✔
346
    def func(mat: re.Match) -> str:
1✔
347
        return replace
1✔
348

349
    return func
1✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc