15452372043

Committed 04 Jun 2025 08:44PM UTC coverage: 17.413%. First build

Build # 15452372043

Build Type

Pull #241

github

Committed by

web-flow

Commit Message

Merge cbb2506f9 into fc9f3677e

Pull Request Pull Request #241: Testing Data and Distributed Testing

Run Details

118 of 199 new or added lines in 7 files covered. (59.3%)

1097 of 6300 relevant lines covered (17.41%)

1.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.64

/src/miranda/testing/utils.py

"""Testing utilities module."""

from __future__ import annotations

import importlib.metadata as ilm
import importlib.resources as ilr
import logging
import os
import re
import time
import warnings
from collections.abc import Callable
from datetime import datetime as dt
from functools import wraps
from io import StringIO
from pathlib import Path
from shutil import copytree
from typing import IO, Any, TextIO
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlparse
from urllib.request import urlretrieve

from filelock import FileLock
from packaging.version import Version
from xarray import Dataset
from xarray import open_dataset as _open_dataset
from xclim.testing.utils import show_versions as _show_versions

import miranda

try:
    import pytest
    from pytest_socket import SocketBlockedError
except ImportError:
    pytest = None
    SocketBlockedError = None

try:
    import pooch
except ImportError:
    warnings.warn(
        "The `pooch` library is not installed. The default cache directory for testing data will not be set."
    )
    pooch = None

logger = logging.getLogger("miranda")


__all__ = [
    "TESTDATA_BRANCH",
    "TESTDATA_CACHE_DIR",
    "TESTDATA_REPO_URL",
    "audit_url",
    "cassini",
    "default_testdata_cache",
    "default_testdata_repo_url",
    "default_testdata_version",
    "gather_testing_data",
    "open_dataset",
    "populate_testing_data",
    "publish_release_notes",
    "show_versions",
    "testing_setup_warnings",
]

default_testdata_version = "v2025.5.16"
"""Default version of the testing data to use when fetching datasets."""

default_testdata_repo_url = (
    "https://raw.githubusercontent.com/Ouranosinc/miranda-testdata/"
)
"""Default URL of the testing data repository to use when fetching datasets."""

try:
    default_testdata_cache = Path(pooch.os_cache("miranda-testdata"))
    """Default location for the testing data cache."""
except (AttributeError, TypeError):
    default_testdata_cache = None

TESTDATA_REPO_URL = str(
    os.getenv("MIRANDA_TESTDATA_REPO_URL", default_testdata_repo_url)
)
"""
Sets the URL of the testing data repository to use when fetching datasets.

Notes
-----
When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:

.. code-block:: console

    $ export MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata"

or setting the variable at runtime:

.. code-block:: console

    $ env MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata" pytest
"""

TESTDATA_BRANCH = str(os.getenv("MIRANDA_TESTDATA_BRANCH", default_testdata_version))
"""
Sets the branch of the testing data repository to use when fetching datasets.

Notes
-----
When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:

.. code-block:: console

    $ export MIRANDA_TESTDATA_BRANCH="my_testing_branch"

or setting the variable at runtime:

.. code-block:: console

    $ env MIRANDA_TESTDATA_BRANCH="my_testing_branch" pytest
"""

TESTDATA_CACHE_DIR = os.getenv("MIRANDA_TESTDATA_CACHE_DIR", default_testdata_cache)
"""
Sets the directory to store the testing datasets.

If not set, the default location will be used (based on ``platformdirs``, see :func:`pooch.os_cache`).

Notes
-----
When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:

.. code-block:: console

    $ export MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data"

or setting the variable at runtime:

.. code-block:: console

    $ env MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data" pytest
"""


# Publishing Tools ###


def publish_release_notes(
    style: str = "md",
    file: os.PathLike[str] | StringIO | TextIO | None = None,
    changes: str | os.PathLike[str] | None = None,
) -> str | None:
    """
    Format release notes in Markdown or ReStructuredText.

    Parameters
    ----------
    style : {"rst", "md"}
        Use ReStructuredText formatting or Markdown. Default: Markdown.
    file : {os.PathLike, StringIO, TextIO}, optional
        If provided, prints to the given file-like object. Otherwise, returns a string.
    changes : str or os.PathLike[str], optional
        If provided, manually points to the file where the changelog can be found.
        Assumes a relative path otherwise.

    Returns
    -------
    str, optional
        If `file` not provided, the formatted release notes.

    Notes
    -----
    This function is used solely for development and packaging purposes.
    """
    if isinstance(changes, str | Path):
        changes_file = Path(changes).absolute()
    else:
        changes_file = Path(__file__).absolute().parents[3].joinpath("CHANGELOG.rst")

    if not changes_file.exists():
        raise FileNotFoundError("Changelog file not found in miranda folder tree.")

    with Path(changes_file).open(encoding="utf-8") as hf:
        changes = hf.read()

    if style == "rst":
        hyperlink_replacements = {
            r":issue:`([0-9]+)`": r"`GH/\1 <https://github.com/Ouranosinc/miranda/issues/\1>`_",
            r":pull:`([0-9]+)`": r"`PR/\1 <https://github.com/Ouranosinc/miranda/pull/\>`_",
            r":user:`([a-zA-Z0-9_.-]+)`": r"`@\1 <https://github.com/\1>`_",
        }
    elif style == "md":
        hyperlink_replacements = {
            r":issue:`([0-9]+)`": r"[GH/\1](https://github.com/Ouranosinc/miranda/issues/\1)",
            r":pull:`([0-9]+)`": r"[PR/\1](https://github.com/Ouranosinc/miranda/pull/\1)",
            r":user:`([a-zA-Z0-9_.-]+)`": r"[@\1](https://github.com/\1)",
        }
    else:
        msg = f"Formatting style not supported: {style}"
        raise NotImplementedError(msg)

    for search, replacement in hyperlink_replacements.items():
        changes = re.sub(search, replacement, changes)

    if style == "md":
        changes = changes.replace("=========\nChangelog\n=========", "# Changelog")

        titles = {r"\n(.*?)\n([\-]{1,})": "-", r"\n(.*?)\n([\^]{1,})": "^"}
        for title_expression, level in titles.items():
            found = re.findall(title_expression, changes)
            for grouping in found:
                fixed_grouping = (
                    str(grouping[0]).replace("(", r"\(").replace(")", r"\)")
                )
                search = rf"({fixed_grouping})\n([\{level}]{'{' + str(len(grouping[1])) + '}'})"
                replacement = f"{'##' if level == '-' else '###'} {grouping[0]}"
                changes = re.sub(search, replacement, changes)

        link_expressions = r"[\`]{1}([\w\s]+)\s<(.+)>`\_"
        found = re.findall(link_expressions, changes)
        for grouping in found:
            search = rf"`{grouping[0]} <.+>`\_"
            replacement = f"[{str(grouping[0]).strip()}]({grouping[1]})"
            changes = re.sub(search, replacement, changes)

    if not file:
        return changes
    if isinstance(file, Path | os.PathLike):
        with Path(file).open(mode="w", encoding="utf-8") as f:
            print(changes, file=f)
    else:
        print(changes, file=file)
    return None


def show_versions(
    file: os.PathLike | StringIO | TextIO | None = None,
    deps: list | None = None,
) -> str | None:
    """
    Print the versions of miranda and its dependencies.

    Parameters
    ----------
    file : {os.PathLike, StringIO, TextIO}, optional
        If provided, prints to the given file-like object. Otherwise, returns a string.
    deps : list, optional
        A list of dependencies to gather and print version information from. Otherwise, prints `miranda` dependencies.

    Returns
    -------
    str or None
        The formatted version information if `file` is not provided, otherwise None.
    """

    def _get_miranda_dependencies():
        xscen_metadata = ilm.metadata("miranda")
        requires = xscen_metadata.get_all("Requires-Dist")
        requires = [
            req.split("[")[0]
            .split(";")[0]
            .split(">")[0]
            .split("<")[0]
            .split("=")[0]
            .split("!")[0]
            for req in requires
        ]

        return ["xscen"] + requires

    if deps is None:
        deps = _get_miranda_dependencies()

    return _show_versions(file=file, deps=deps)


# Test Data Utilities ###


def testing_setup_warnings():
    """Warn users about potential incompatibilities between miranda and miranda-testdata versions."""
    if (
        re.match(r"^\d+\.\d+\.\d+$", miranda.__version__)
        and TESTDATA_BRANCH != default_testdata_version
    ):
        # This does not need to be emitted on GitHub Workflows and ReadTheDocs
        if not os.getenv("CI") and not os.getenv("READTHEDOCS"):
            warnings.warn(
                f"`miranda` stable ({miranda.__version__}) is running tests against a non-default "
                f"branch of the testing data. It is possible that changes to the testing data may "
                f"be incompatible with some assertions in this version. "
                f"Please be sure to check {TESTDATA_REPO_URL} for more information.",
            )

    if re.match(r"^v\d+\.\d+\.\d+", TESTDATA_BRANCH):
        # Find the date of last modification of miranda source files to generate a calendar version
        install_date = dt.strptime(
            time.ctime(Path(miranda.__file__).stat().st_mtime),
            "%a %b %d %H:%M:%S %Y",
        )
        install_calendar_version = (
            f"{install_date.year}.{install_date.month}.{install_date.day}"
        )

        if Version(TESTDATA_BRANCH) > Version(install_calendar_version):
            warnings.warn(
                f"The installation date of `miranda` ({install_date.ctime()}) "
                f"predates the last release of testing data ({TESTDATA_BRANCH}). "
                "It is very likely that the testing data is incompatible with this build of `miranda`.",
            )


def load_registry(
    branch: str = TESTDATA_BRANCH, repo: str = TESTDATA_REPO_URL
) -> dict[str, str]:
    """
    Load the registry file for the test data.

    Parameters
    ----------
    branch : str
        Branch of the repository to use when fetching testing datasets.
    repo : str
        URL of the repository to use when fetching testing datasets.

    Returns
    -------
    dict
        Dictionary of filenames and hashes.
    """
    if not repo.endswith("/"):
        repo = f"{repo}/"
    remote_registry = audit_url(
        urljoin(
            urljoin(repo, branch if branch.endswith("/") else f"{branch}/"),
            "data/registry.txt",
        )
    )

    if repo != default_testdata_repo_url:
        external_repo_name = urlparse(repo).path.split("/")[-2]
        external_branch_name = branch.split("/")[-1]
        registry_file = Path(
            str(
                ilr.files("miranda").joinpath(
                    f"testing/registry.{external_repo_name}.{external_branch_name}.txt"
                )
            )
        )
        urlretrieve(remote_registry, registry_file)  # noqa: S310

    elif branch != default_testdata_version:
        custom_registry_folder = Path(
            str(ilr.files("miranda").joinpath(f"testing/{branch}"))
        )
        custom_registry_folder.mkdir(parents=True, exist_ok=True)
        registry_file = custom_registry_folder.joinpath("registry.txt")
        urlretrieve(remote_registry, registry_file)  # noqa: S310

    else:
        registry_file = Path(str(ilr.files("miranda").joinpath("testing/registry.txt")))

    if not registry_file.exists():
        raise FileNotFoundError(f"Registry file not found: {registry_file}")

    # Load the registry file
    with registry_file.open(encoding="utf-8") as f:
        registry = {line.split()[0]: line.split()[1] for line in f}
    return registry


def cassini(
    repo: str = TESTDATA_REPO_URL,
    branch: str = TESTDATA_BRANCH,
    cache_dir: str | Path = TESTDATA_CACHE_DIR,
    allow_updates: bool = True,
):
    """
    Pooch registry instance for miranda test data.

    Parameters
    ----------
    repo : str
        URL of the repository to use when fetching testing datasets.
    branch : str
        Branch of repository to use when fetching testing datasets.
    cache_dir : str or Path
        The path to the directory where the data files are stored.
    allow_updates : bool
        If True, allow updates to the data files. Default is True.

    Returns
    -------
    pooch.Pooch
        The Pooch instance for accessing the miranda testing data.

    Notes
    -----
    There are three environment variables that can be used to control the behaviour of this registry:
        - ``MIRANDA_TESTDATA_CACHE_DIR``: If this environment variable is set, it will be used as the
          base directory to store the data files.
          The directory should be an absolute path (i.e., it should start with ``/``).
          Otherwise, the default location will be used (based on ``platformdirs``, see :py:func:`pooch.os_cache`).
        - ``MIRANDA_TESTDATA_REPO_URL``: If this environment variable is set, it will be used as the URL of
          the repository to use when fetching datasets. Otherwise, the default repository will be used.
        - ``MIRANDA_TESTDATA_BRANCH``: If this environment variable is set, it will be used as the branch of
          the repository to use when fetching datasets. Otherwise, the default branch will be used.

    Examples
    --------
    Using the registry to download a file:

    .. code-block:: python

        import xarray as xr
        from miranda.testing import cassini

        example_file = cassini().fetch("example.nc")
        data = xr.open_dataset(example_file)
    """
    if pooch is None:
        raise ImportError(
            "The `pooch` package is required to fetch the miranda testing data. "
            "You can install it with `pip install pooch` or `pip install miranda[dev]`."
        )
    if not repo.endswith("/"):
        repo = f"{repo}/"
    remote = audit_url(
        urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data")
    )

    _cassini = pooch.create(
        path=cache_dir,
        base_url=remote,
        version=default_testdata_version,
        version_dev=branch,
        allow_updates=allow_updates,
        registry=load_registry(branch=branch, repo=repo),
    )

    # Add a custom fetch method to the Pooch instance
    # Needed to address: https://github.com/readthedocs/readthedocs.org/issues/11763
    # Fix inspired by @bjlittle (https://github.com/bjlittle/geovista/pull/1202)
    _cassini.fetch_diversion = _cassini.fetch

    # Overload the fetch method to add user-agent headers
    @wraps(_cassini.fetch_diversion)
    def _fetch(
        *args, **kwargs: bool | Callable
    ) -> str:  # numpydoc ignore=GL08  # *args: str
        def _downloader(
            url: str,
            output_file: str | IO,
            poocher: pooch.Pooch,
            check_only: bool | None = False,
        ) -> None:
            """Download the file from the URL and save it to the save_path."""
            headers = {"User-Agent": f"miranda ({miranda.__version__})"}
            downloader = pooch.HTTPDownloader(headers=headers)
            return downloader(url, output_file, poocher, check_only=check_only)

        # default to our http/s downloader with user-agent headers
        kwargs.setdefault("downloader", _downloader)
        return _cassini.fetch_diversion(*args, **kwargs)

    # Replace the fetch method with the custom fetch method
    _cassini.fetch = _fetch

    return _cassini


def open_dataset(
    name: str,
    cassini_kwargs: dict[str, Path | str | bool] | None = None,
    **xr_kwargs: Any,
) -> Dataset:
    r"""
    Convenience function to open a dataset from the miranda testing data using the `cassini` class.

    This is a thin wrapper around the `cassini` class to make it easier to open miranda testing datasets.

    Parameters
    ----------
    name : str
        Name of the file containing the dataset.
    cassini_kwargs : dict
        Keyword arguments passed to the cassini function.
    **xr_kwargs : Any
        Keyword arguments passed to xarray.open_dataset.

    Returns
    -------
    xarray.Dataset
        The dataset.

    See Also
    --------
    xarray.open_dataset : Open and read a dataset from a file or file-like object.
    cassini : Pooch wrapper for accessing the miranda testing data.
    """
    if cassini_kwargs is None:
        cassini_kwargs = {}
    return _open_dataset(cassini(**cassini_kwargs).fetch(name), **xr_kwargs)


def populate_testing_data(
    temp_folder: Path | None = None,
    repo: str = TESTDATA_REPO_URL,
    branch: str = TESTDATA_BRANCH,
    local_cache: Path = TESTDATA_CACHE_DIR,
) -> None:
    """
    Populate the local cache with the testing data.

    Parameters
    ----------
    temp_folder : Path, optional
        Path to a temporary folder to use as the local cache. If not provided, the default location will be used.
    repo : str, optional
        URL of the repository to use when fetching testing datasets.
    branch : str, optional
        Branch of miranda-testdata to use when fetching testing datasets.
    local_cache : Path
        The path to the local cache. Defaults to the location set by the platformdirs library.
        The testing data will be downloaded to this local cache.
    """
    # Create the Pooch instance
    n = cassini(repo=repo, branch=branch, cache_dir=temp_folder or local_cache)

    # Download the files
    errored_files = []
    for file in load_registry():
        try:
            n.fetch(file)
        except HTTPError:  # noqa: PERF203
            msg = f"File `{file}` not accessible in remote repository."
            logging.error(msg)
            errored_files.append(file)
        else:
            logging.info("Files were downloaded successfully.")

    if errored_files:
        logging.error(
            "The following files were unable to be downloaded: %s",
            errored_files,
        )


def gather_testing_data(
    worker_cache_dir: str | os.PathLike[str] | Path,
    worker_id: str,
    _cache_dir: str | os.PathLike[str] | None = TESTDATA_CACHE_DIR,
) -> None:
    """
    Gather testing data across workers.

    Parameters
    ----------
    worker_cache_dir : str or Path
        The directory to store the testing data.
    worker_id : str
        The worker ID.
    _cache_dir : str or Path, optional
        The directory to store the testing data. Default is None.

    Raises
    ------
    ValueError
        If the cache directory is not set.
    FileNotFoundError
        If the testing data is not found.
    """
    if _cache_dir is None:
        raise ValueError(
            "The cache directory must be set. "
            "Please set the `cache_dir` parameter or the `MIRANDA_DATA_DIR` environment variable."
        )
    cache_dir = Path(_cache_dir)

    if worker_id == "master":
        populate_testing_data(branch=TESTDATA_BRANCH)
    else:
        cache_dir.mkdir(exist_ok=True, parents=True)
        lockfile = cache_dir.joinpath(".lock")
        test_data_being_written = FileLock(lockfile)
        with test_data_being_written:
            # This flag prevents multiple calls from re-attempting to download testing data in the same pytest run
            populate_testing_data(branch=TESTDATA_BRANCH)
            cache_dir.joinpath(".data_written").touch()
        with test_data_being_written.acquire():
            if lockfile.exists():
                lockfile.unlink()
        copytree(cache_dir.joinpath(default_testdata_version), worker_cache_dir)


# Testing Utilities ###


def audit_url(url: str, context: str | None = None) -> str:
    """
    Check if the URL is well-formed.

    Parameters
    ----------
    url : str
        The URL to check.
    context : str, optional
        Additional context to include in the error message. Default is None.

    Returns
    -------
    str
        The URL if it is well-formed.

    Raises
    ------
    URLError
        If the URL is not well-formed.
    """
    msg = ""
    result = urlparse(url)
    if result.scheme == "http":
        msg = f"{context if context else ''} URL is not using secure HTTP: '{url}'".strip()
    if not all([result.scheme, result.netloc]):
        msg = f"{context if context else ''} URL is not well-formed: '{url}'".strip()

    if msg:
        logger.error(msg)
        raise URLError(msg)
    return url

1	"""Testing utilities module."""
2
3	from __future__ import annotations	7✔
4
5	import importlib.metadata as ilm	7✔
6	import importlib.resources as ilr	7✔
7	import logging	7✔
8	import os	7✔
9	import re	7✔
10	import time	7✔
11	import warnings	7✔
12	from collections.abc import Callable	7✔
13	from datetime import datetime as dt	7✔
14	from functools import wraps	7✔
15	from io import StringIO	7✔
16	from pathlib import Path	7✔
17	from shutil import copytree	7✔
18	from typing import IO, Any, TextIO	7✔
19	from urllib.error import HTTPError, URLError	7✔
20	from urllib.parse import urljoin, urlparse	7✔
21	from urllib.request import urlretrieve	7✔
22
23	from filelock import FileLock	7✔
24	from packaging.version import Version	7✔
25	from xarray import Dataset	7✔
26	from xarray import open_dataset as _open_dataset	7✔
27	from xclim.testing.utils import show_versions as _show_versions	7✔
28
29	import miranda	7✔
30
31	try:	7✔
32	import pytest	7✔
33	from pytest_socket import SocketBlockedError	7✔
34	except ImportError:	7✔
35	pytest = None	7✔
36	SocketBlockedError = None	7✔
37
38	try:	7✔
39	import pooch	7✔
NEW 40	except ImportError:	×
NEW 41	warnings.warn(	×
42	"The `pooch` library is not installed. The default cache directory for testing data will not be set."
43	)
NEW 44	pooch = None	×
45
46	logger = logging.getLogger("miranda")	7✔
47
48
49	__all__ = [	7✔
50	"TESTDATA_BRANCH",
51	"TESTDATA_CACHE_DIR",
52	"TESTDATA_REPO_URL",
53	"audit_url",
54	"cassini",
55	"default_testdata_cache",
56	"default_testdata_repo_url",
57	"default_testdata_version",
58	"gather_testing_data",
59	"open_dataset",
60	"populate_testing_data",
61	"publish_release_notes",
62	"show_versions",
63	"testing_setup_warnings",
64	]
65
66	default_testdata_version = "v2025.5.16"	7✔
67	"""Default version of the testing data to use when fetching datasets."""	7✔
68
69	default_testdata_repo_url = (	7✔
70	"https://raw.githubusercontent.com/Ouranosinc/miranda-testdata/"
71	)
72	"""Default URL of the testing data repository to use when fetching datasets."""	7✔
73
74	try:	7✔
75	default_testdata_cache = Path(pooch.os_cache("miranda-testdata"))	7✔
76	"""Default location for the testing data cache."""	7✔
NEW 77	except (AttributeError, TypeError):	×
NEW 78	default_testdata_cache = None	×
79
80	TESTDATA_REPO_URL = str(	7✔
81	os.getenv("MIRANDA_TESTDATA_REPO_URL", default_testdata_repo_url)
82	)
83	"""	7✔
84	Sets the URL of the testing data repository to use when fetching datasets.
85
86	Notes
87	-----
88	When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:
89
90	.. code-block:: console
91
92	$ export MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata"
93
94	or setting the variable at runtime:
95
96	.. code-block:: console
97
98	$ env MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata" pytest
99	"""
100
101	TESTDATA_BRANCH = str(os.getenv("MIRANDA_TESTDATA_BRANCH", default_testdata_version))	7✔
102	"""	7✔
103	Sets the branch of the testing data repository to use when fetching datasets.
104
105	Notes
106	-----
107	When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:
108
109	.. code-block:: console
110
111	$ export MIRANDA_TESTDATA_BRANCH="my_testing_branch"
112
113	or setting the variable at runtime:
114
115	.. code-block:: console
116
117	$ env MIRANDA_TESTDATA_BRANCH="my_testing_branch" pytest
118	"""
119
120	TESTDATA_CACHE_DIR = os.getenv("MIRANDA_TESTDATA_CACHE_DIR", default_testdata_cache)	7✔
121	"""	7✔
122	Sets the directory to store the testing datasets.
123
124	If not set, the default location will be used (based on ``platformdirs``, see :func:`pooch.os_cache`).
125
126	Notes
127	-----
128	When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:
129
130	.. code-block:: console
131
132	$ export MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data"
133
134	or setting the variable at runtime:
135
136	.. code-block:: console
137
138	$ env MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data" pytest
139	"""
140
141
142	# Publishing Tools ###
143
144
145	def publish_release_notes(	7✔
146	style: str = "md",
147	file: os.PathLike[str] \| StringIO \| TextIO \| None = None,
148	changes: str \| os.PathLike[str] \| None = None,
149	) -> str \| None:
150	"""
151	Format release notes in Markdown or ReStructuredText.
152
153	Parameters
154	----------
155	style : {"rst", "md"}
156	Use ReStructuredText formatting or Markdown. Default: Markdown.
157	file : {os.PathLike, StringIO, TextIO}, optional
158	If provided, prints to the given file-like object. Otherwise, returns a string.
159	changes : str or os.PathLike[str], optional
160	If provided, manually points to the file where the changelog can be found.
161	Assumes a relative path otherwise.
162
163	Returns
164	-------
165	str, optional
166	If `file` not provided, the formatted release notes.
167
168	Notes
169	-----
170	This function is used solely for development and packaging purposes.
171	"""
NEW 172	if isinstance(changes, str \| Path):	×
NEW 173	changes_file = Path(changes).absolute()	×
174	else:
NEW 175	changes_file = Path(__file__).absolute().parents[3].joinpath("CHANGELOG.rst")	×
176
NEW 177	if not changes_file.exists():	×
NEW 178	raise FileNotFoundError("Changelog file not found in miranda folder tree.")	×
179
NEW 180	with Path(changes_file).open(encoding="utf-8") as hf:	×
NEW 181	changes = hf.read()	×
182
NEW 183	if style == "rst":	×
NEW 184	hyperlink_replacements = {	×
185	r":issue:`([0-9]+)`": r"`GH/\1 <https://github.com/Ouranosinc/miranda/issues/\1>`_",
186	r":pull:`([0-9]+)`": r"`PR/\1 <https://github.com/Ouranosinc/miranda/pull/\>`_",
187	r":user:`([a-zA-Z0-9_.-]+)`": r"`@\1 <https://github.com/\1>`_",
188	}
NEW 189	elif style == "md":	×
NEW 190	hyperlink_replacements = {	×
191	r":issue:`([0-9]+)`": r"[GH/\1](https://github.com/Ouranosinc/miranda/issues/\1)",
192	r":pull:`([0-9]+)`": r"[PR/\1](https://github.com/Ouranosinc/miranda/pull/\1)",
193	r":user:`([a-zA-Z0-9_.-]+)`": r"[@\1](https://github.com/\1)",
194	}
195	else:
NEW 196	msg = f"Formatting style not supported: {style}"	×
NEW 197	raise NotImplementedError(msg)	×
198
NEW 199	for search, replacement in hyperlink_replacements.items():	×
NEW 200	changes = re.sub(search, replacement, changes)	×
201
NEW 202	if style == "md":	×
NEW 203	changes = changes.replace("=========\nChangelog\n=========", "# Changelog")	×
204
NEW 205	titles = {r"\n(.?)\n([\-]{1,})": "-", r"\n(.?)\n([\^]{1,})": "^"}	×
NEW 206	for title_expression, level in titles.items():	×
NEW 207	found = re.findall(title_expression, changes)	×
NEW 208	for grouping in found:	×
NEW 209	fixed_grouping = (	×
210	str(grouping[0]).replace("(", r"\(").replace(")", r"\)")
211	)
NEW 212	search = rf"({fixed_grouping})\n([\{level}]{'{' + str(len(grouping[1])) + '}'})"	×
NEW 213	replacement = f"{'##' if level == '-' else '###'} {grouping[0]}"	×
NEW 214	changes = re.sub(search, replacement, changes)	×
215
NEW 216	link_expressions = r"[\`]{1}([\w\s]+)\s<(.+)>`\_"	×
NEW 217	found = re.findall(link_expressions, changes)	×
NEW 218	for grouping in found:	×
NEW 219	search = rf"`{grouping[0]} <.+>`\_"	×
NEW 220	replacement = f"[{str(grouping[0]).strip()}]({grouping[1]})"	×
NEW 221	changes = re.sub(search, replacement, changes)	×
222
NEW 223	if not file:	×
NEW 224	return changes	×
NEW 225	if isinstance(file, Path \| os.PathLike):	×
NEW 226	with Path(file).open(mode="w", encoding="utf-8") as f:	×
NEW 227	print(changes, file=f)	×
228	else:
NEW 229	print(changes, file=file)	×
NEW 230	return None	×
231
232
233	def show_versions(	7✔
234	file: os.PathLike \| StringIO \| TextIO \| None = None,
235	deps: list \| None = None,
236	) -> str \| None:
237	"""
238	Print the versions of miranda and its dependencies.
239
240	Parameters
241	----------
242	file : {os.PathLike, StringIO, TextIO}, optional
243	If provided, prints to the given file-like object. Otherwise, returns a string.
244	deps : list, optional
245	A list of dependencies to gather and print version information from. Otherwise, prints `miranda` dependencies.
246
247	Returns
248	-------
249	str or None
250	The formatted version information if `file` is not provided, otherwise None.
251	"""
252
NEW 253	def _get_miranda_dependencies():	×
NEW 254	xscen_metadata = ilm.metadata("miranda")	×
NEW 255	requires = xscen_metadata.get_all("Requires-Dist")	×
NEW 256	requires = [	×
257	req.split("[")[0]
258	.split(";")[0]
259	.split(">")[0]
260	.split("<")[0]
261	.split("=")[0]
262	.split("!")[0]
263	for req in requires
264	]
265
NEW 266	return ["xscen"] + requires	×
267
NEW 268	if deps is None:	×
NEW 269	deps = _get_miranda_dependencies()	×
270
NEW 271	return _show_versions(file=file, deps=deps)	×
272
273
274	# Test Data Utilities ###
275
276
277	def testing_setup_warnings():	7✔
278	"""Warn users about potential incompatibilities between miranda and miranda-testdata versions."""
279	if (	7✔
280	re.match(r"^\d+\.\d+\.\d+$", miranda.__version__)
281	and TESTDATA_BRANCH != default_testdata_version
282	):
283	# This does not need to be emitted on GitHub Workflows and ReadTheDocs
NEW 284	if not os.getenv("CI") and not os.getenv("READTHEDOCS"):	×
NEW 285	warnings.warn(	×
286	f"`miranda` stable ({miranda.__version__}) is running tests against a non-default "
287	f"branch of the testing data. It is possible that changes to the testing data may "
288	f"be incompatible with some assertions in this version. "
289	f"Please be sure to check {TESTDATA_REPO_URL} for more information.",
290	)
291
292	if re.match(r"^v\d+\.\d+\.\d+", TESTDATA_BRANCH):	7✔
293	# Find the date of last modification of miranda source files to generate a calendar version
294	install_date = dt.strptime(	7✔
295	time.ctime(Path(miranda.__file__).stat().st_mtime),
296	"%a %b %d %H:%M:%S %Y",
297	)
298	install_calendar_version = (	7✔
299	f"{install_date.year}.{install_date.month}.{install_date.day}"
300	)
301
302	if Version(TESTDATA_BRANCH) > Version(install_calendar_version):	7✔
NEW 303	warnings.warn(	×
304	f"The installation date of `miranda` ({install_date.ctime()}) "
305	f"predates the last release of testing data ({TESTDATA_BRANCH}). "
306	"It is very likely that the testing data is incompatible with this build of `miranda`.",
307	)
308
309
310	def load_registry(	7✔
311	branch: str = TESTDATA_BRANCH, repo: str = TESTDATA_REPO_URL
312	) -> dict[str, str]:
313	"""
314	Load the registry file for the test data.
315
316	Parameters
317	----------
318	branch : str
319	Branch of the repository to use when fetching testing datasets.
320	repo : str
321	URL of the repository to use when fetching testing datasets.
322
323	Returns
324	-------
325	dict
326	Dictionary of filenames and hashes.
327	"""
328	if not repo.endswith("/"):	7✔
NEW 329	repo = f"{repo}/"	×
330	remote_registry = audit_url(	7✔
331	urljoin(
332	urljoin(repo, branch if branch.endswith("/") else f"{branch}/"),
333	"data/registry.txt",
334	)
335	)
336
337	if repo != default_testdata_repo_url:	7✔
NEW 338	external_repo_name = urlparse(repo).path.split("/")[-2]	×
NEW 339	external_branch_name = branch.split("/")[-1]	×
NEW 340	registry_file = Path(	×
341	str(
342	ilr.files("miranda").joinpath(
343	f"testing/registry.{external_repo_name}.{external_branch_name}.txt"
344	)
345	)
346	)
NEW 347	urlretrieve(remote_registry, registry_file) # noqa: S310	×
348
349	elif branch != default_testdata_version:	7✔
NEW 350	custom_registry_folder = Path(	×
351	str(ilr.files("miranda").joinpath(f"testing/{branch}"))
352	)
NEW 353	custom_registry_folder.mkdir(parents=True, exist_ok=True)	×
NEW 354	registry_file = custom_registry_folder.joinpath("registry.txt")	×
NEW 355	urlretrieve(remote_registry, registry_file) # noqa: S310	×
356
357	else:
358	registry_file = Path(str(ilr.files("miranda").joinpath("testing/registry.txt")))	7✔
359
360	if not registry_file.exists():	7✔
NEW 361	raise FileNotFoundError(f"Registry file not found: {registry_file}")	×
362
363	# Load the registry file
364	with registry_file.open(encoding="utf-8") as f:	7✔
365	registry = {line.split()[0]: line.split()[1] for line in f}	7✔
366	return registry	7✔
367
368
369	def cassini(	7✔
370	repo: str = TESTDATA_REPO_URL,
371	branch: str = TESTDATA_BRANCH,
372	cache_dir: str \| Path = TESTDATA_CACHE_DIR,
373	allow_updates: bool = True,
374	):
375	"""
376	Pooch registry instance for miranda test data.
377
378	Parameters
379	----------
380	repo : str
381	URL of the repository to use when fetching testing datasets.
382	branch : str
383	Branch of repository to use when fetching testing datasets.
384	cache_dir : str or Path
385	The path to the directory where the data files are stored.
386	allow_updates : bool
387	If True, allow updates to the data files. Default is True.
388
389	Returns
390	-------
391	pooch.Pooch
392	The Pooch instance for accessing the miranda testing data.
393
394	Notes
395	-----
396	There are three environment variables that can be used to control the behaviour of this registry:
397	- ``MIRANDA_TESTDATA_CACHE_DIR``: If this environment variable is set, it will be used as the
398	base directory to store the data files.
399	The directory should be an absolute path (i.e., it should start with ``/``).
400	Otherwise, the default location will be used (based on ``platformdirs``, see :py:func:`pooch.os_cache`).
401	- ``MIRANDA_TESTDATA_REPO_URL``: If this environment variable is set, it will be used as the URL of
402	the repository to use when fetching datasets. Otherwise, the default repository will be used.
403	- ``MIRANDA_TESTDATA_BRANCH``: If this environment variable is set, it will be used as the branch of
404	the repository to use when fetching datasets. Otherwise, the default branch will be used.
405
406	Examples
407	--------
408	Using the registry to download a file:
409
410	.. code-block:: python
411
412	import xarray as xr
413	from miranda.testing import cassini
414
415	example_file = cassini().fetch("example.nc")
416	data = xr.open_dataset(example_file)
417	"""
418	if pooch is None:	7✔
NEW 419	raise ImportError(	×
420	"The `pooch` package is required to fetch the miranda testing data. "
421	"You can install it with `pip install pooch` or `pip install miranda[dev]`."
422	)
423	if not repo.endswith("/"):	7✔
NEW 424	repo = f"{repo}/"	×
425	remote = audit_url(	7✔
426	urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data")
427	)
428
429	_cassini = pooch.create(	7✔
430	path=cache_dir,
431	base_url=remote,
432	version=default_testdata_version,
433	version_dev=branch,
434	allow_updates=allow_updates,
435	registry=load_registry(branch=branch, repo=repo),
436	)
437
438	# Add a custom fetch method to the Pooch instance
439	# Needed to address: https://github.com/readthedocs/readthedocs.org/issues/11763
440	# Fix inspired by @bjlittle (https://github.com/bjlittle/geovista/pull/1202)
441	_cassini.fetch_diversion = _cassini.fetch	7✔
442
443	# Overload the fetch method to add user-agent headers
444	@wraps(_cassini.fetch_diversion)	7✔
445	def _fetch(	7✔
446	args, *kwargs: bool \| Callable
447	) -> str: # numpydoc ignore=GL08 # *args: str
448	def _downloader(	7✔
449	url: str,
450	output_file: str \| IO,
451	poocher: pooch.Pooch,
452	check_only: bool \| None = False,
453	) -> None:
454	"""Download the file from the URL and save it to the save_path."""
455	headers = {"User-Agent": f"miranda ({miranda.__version__})"}	7✔
456	downloader = pooch.HTTPDownloader(headers=headers)	7✔
457	return downloader(url, output_file, poocher, check_only=check_only)	7✔
458
459	# default to our http/s downloader with user-agent headers
460	kwargs.setdefault("downloader", _downloader)	7✔
461	return _cassini.fetch_diversion(args, *kwargs)	7✔
462
463	# Replace the fetch method with the custom fetch method
464	_cassini.fetch = _fetch	7✔
465
466	return _cassini	7✔
467
468
469	def open_dataset(	7✔
470	name: str,
471	cassini_kwargs: dict[str, Path \| str \| bool] \| None = None,
472	**xr_kwargs: Any,
473	) -> Dataset:
474	r"""
475	Convenience function to open a dataset from the miranda testing data using the `cassini` class.
476
477	This is a thin wrapper around the `cassini` class to make it easier to open miranda testing datasets.
478
479	Parameters
480	----------
481	name : str
482	Name of the file containing the dataset.
483	cassini_kwargs : dict
484	Keyword arguments passed to the cassini function.
485	**xr_kwargs : Any
486	Keyword arguments passed to xarray.open_dataset.
487
488	Returns
489	-------
490	xarray.Dataset
491	The dataset.
492
493	See Also
494	--------
495	xarray.open_dataset : Open and read a dataset from a file or file-like object.
496	cassini : Pooch wrapper for accessing the miranda testing data.
497	"""
NEW 498	if cassini_kwargs is None:	×
NEW 499	cassini_kwargs = {}	×
NEW 500	return _open_dataset(cassini(cassini_kwargs).fetch(name), xr_kwargs)	×
501
502
503	def populate_testing_data(	7✔
504	temp_folder: Path \| None = None,
505	repo: str = TESTDATA_REPO_URL,
506	branch: str = TESTDATA_BRANCH,
507	local_cache: Path = TESTDATA_CACHE_DIR,
508	) -> None:
509	"""
510	Populate the local cache with the testing data.
511
512	Parameters
513	----------
514	temp_folder : Path, optional
515	Path to a temporary folder to use as the local cache. If not provided, the default location will be used.
516	repo : str, optional
517	URL of the repository to use when fetching testing datasets.
518	branch : str, optional
519	Branch of miranda-testdata to use when fetching testing datasets.
520	local_cache : Path
521	The path to the local cache. Defaults to the location set by the platformdirs library.
522	The testing data will be downloaded to this local cache.
523	"""
524	# Create the Pooch instance
525	n = cassini(repo=repo, branch=branch, cache_dir=temp_folder or local_cache)	7✔
526
527	# Download the files
528	errored_files = []	7✔
529	for file in load_registry():	7✔
530	try:	7✔
531	n.fetch(file)	7✔
NEW 532	except HTTPError: # noqa: PERF203	×
NEW 533	msg = f"File `{file}` not accessible in remote repository."	×
NEW 534	logging.error(msg)	×
NEW 535	errored_files.append(file)	×
536	else:
537	logging.info("Files were downloaded successfully.")	7✔
538
539	if errored_files:	7✔
NEW 540	logging.error(	×
541	"The following files were unable to be downloaded: %s",
542	errored_files,
543	)
544
545
546	def gather_testing_data(	7✔
547	worker_cache_dir: str \| os.PathLike[str] \| Path,
548	worker_id: str,
549	_cache_dir: str \| os.PathLike[str] \| None = TESTDATA_CACHE_DIR,
550	) -> None:
551	"""
552	Gather testing data across workers.
553
554	Parameters
555	----------
556	worker_cache_dir : str or Path
557	The directory to store the testing data.
558	worker_id : str
559	The worker ID.
560	_cache_dir : str or Path, optional
561	The directory to store the testing data. Default is None.
562
563	Raises
564	------
565	ValueError
566	If the cache directory is not set.
567	FileNotFoundError
568	If the testing data is not found.
569	"""
570	if _cache_dir is None:	7✔
NEW 571	raise ValueError(	×
572	"The cache directory must be set. "
573	"Please set the `cache_dir` parameter or the `MIRANDA_DATA_DIR` environment variable."
574	)
575	cache_dir = Path(_cache_dir)	7✔
576
577	if worker_id == "master":	7✔
578	populate_testing_data(branch=TESTDATA_BRANCH)	4✔
579	else:
580	cache_dir.mkdir(exist_ok=True, parents=True)	3✔
581	lockfile = cache_dir.joinpath(".lock")	3✔
582	test_data_being_written = FileLock(lockfile)	3✔
583	with test_data_being_written:	3✔
584	# This flag prevents multiple calls from re-attempting to download testing data in the same pytest run
585	populate_testing_data(branch=TESTDATA_BRANCH)	3✔
586	cache_dir.joinpath(".data_written").touch()	3✔
587	with test_data_being_written.acquire():	3✔
588	if lockfile.exists():	3✔
589	lockfile.unlink()	3✔
590	copytree(cache_dir.joinpath(default_testdata_version), worker_cache_dir)	3✔
591
592
593	# Testing Utilities ###
594
595
596	def audit_url(url: str, context: str \| None = None) -> str:	7✔
597	"""
598	Check if the URL is well-formed.
599
600	Parameters
601	----------
602	url : str
603	The URL to check.
604	context : str, optional
605	Additional context to include in the error message. Default is None.
606
607	Returns
608	-------
609	str
610	The URL if it is well-formed.
611
612	Raises
613	------
614	URLError
615	If the URL is not well-formed.
616	"""
617	msg = ""	7✔
618	result = urlparse(url)	7✔
619	if result.scheme == "http":	7✔
NEW 620	msg = f"{context if context else ''} URL is not using secure HTTP: '{url}'".strip()	×
621	if not all([result.scheme, result.netloc]):	7✔
NEW 622	msg = f"{context if context else ''} URL is not well-formed: '{url}'".strip()	×
623
624	if msg:	7✔
NEW 625	logger.error(msg)	×
NEW 626	raise URLError(msg)	×
627	return url	7✔

Ouranosinc / miranda / 15452372043

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous