15141552897

Committed 20 May 2025 03:24PM UTC coverage: 16.448% (+1.4%) from 15.049%

Build # 15141552897

Build Type

Pull #241

github

Committed by

web-flow

Commit Message

Merge 12ef5216f into 730c6f31e

Pull Request Pull Request #241: Testing Data and Distributed Testing

Run Details

115 of 194 new or added lines in 2 files covered. (59.28%)

20 existing lines in 2 files now uncovered.

1029 of 6256 relevant lines covered (16.45%)

1.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.85

/src/miranda/testing/utils.py

"""Testing utilities module."""

from __future__ import annotations

import importlib.metadata as ilm
import importlib.resources as ilr
import logging
import os
import re
import time
import warnings
from collections.abc import Callable
from datetime import datetime as dt
from functools import wraps
from io import StringIO
from pathlib import Path
from shutil import copytree
from typing import IO, Any, TextIO
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlparse
from urllib.request import urlretrieve

from filelock import FileLock
from packaging.version import Version
from xarray import Dataset
from xarray import open_dataset as _open_dataset
from xclim.testing.utils import show_versions as _show_versions

from miranda import __version__ as __miranda_version__

try:
    import pytest
    from pytest_socket import SocketBlockedError
except ImportError:
    pytest = None
    SocketBlockedError = None

try:
    import pooch
except ImportError:
    warnings.warn(
        "The `pooch` library is not installed. The default cache directory for testing data will not be set."
    )
    pooch = None

import miranda

logger = logging.getLogger("miranda")


__all__ = [
    "TESTDATA_BRANCH",
    "TESTDATA_CACHE_DIR",
    "TESTDATA_REPO_URL",
    "audit_url",
    "cassini",
    "default_testdata_cache",
    "default_testdata_repo_url",
    "default_testdata_version",
    "gather_testing_data",
    "open_dataset",
    "populate_testing_data",
    "publish_release_notes",
    "show_versions",
    "testing_setup_warnings",
]

default_testdata_version = "v2025.5.16"
"""Default version of the testing data to use when fetching datasets."""

default_testdata_repo_url = (
    "https://raw.githubusercontent.com/Ouranosinc/miranda-testdata/"
)
"""Default URL of the testing data repository to use when fetching datasets."""

try:
    default_testdata_cache = Path(pooch.os_cache("miranda-testdata"))
    """Default location for the testing data cache."""
except AttributeError:
    default_testdata_cache = None

TESTDATA_REPO_URL = str(
    os.getenv("MIRANDA_TESTDATA_REPO_URL", default_testdata_repo_url)
)
"""
Sets the URL of the testing data repository to use when fetching datasets.

Notes
-----
When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:

.. code-block:: console

    $ export MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata"

or setting the variable at runtime:

.. code-block:: console

    $ env MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata" pytest
"""

TESTDATA_BRANCH = str(os.getenv("MIRANDA_TESTDATA_BRANCH", default_testdata_version))
"""
Sets the branch of the testing data repository to use when fetching datasets.

Notes
-----
When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:

.. code-block:: console

    $ export MIRANDA_TESTDATA_BRANCH="my_testing_branch"

or setting the variable at runtime:

.. code-block:: console

    $ env MIRANDA_TESTDATA_BRANCH="my_testing_branch" pytest
"""

TESTDATA_CACHE_DIR = os.getenv("MIRANDA_TESTDATA_CACHE_DIR", default_testdata_cache)
"""
Sets the directory to store the testing datasets.

If not set, the default location will be used (based on ``platformdirs``, see :func:`pooch.os_cache`).

Notes
-----
When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:

.. code-block:: console

    $ export MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data"

or setting the variable at runtime:

.. code-block:: console

    $ env MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data" pytest
"""


# Publishing Tools ###


def publish_release_notes(
    style: str = "md",
    file: os.PathLike[str] | StringIO | TextIO | None = None,
    changes: str | os.PathLike[str] | None = None,
) -> str | None:
    """
    Format release notes in Markdown or ReStructuredText.

    Parameters
    ----------
    style : {"rst", "md"}
        Use ReStructuredText formatting or Markdown. Default: Markdown.
    file : {os.PathLike, StringIO, TextIO}, optional
        If provided, prints to the given file-like object. Otherwise, returns a string.
    changes : str or os.PathLike[str], optional
        If provided, manually points to the file where the changelog can be found.
        Assumes a relative path otherwise.

    Returns
    -------
    str, optional
        If `file` not provided, the formatted release notes.

    Notes
    -----
    This function is used solely for development and packaging purposes.
    """
    if isinstance(changes, str | Path):
        changes_file = Path(changes).absolute()
    else:
        changes_file = Path(__file__).absolute().parents[3].joinpath("CHANGELOG.rst")

    if not changes_file.exists():
        raise FileNotFoundError("Changelog file not found in miranda folder tree.")

    with Path(changes_file).open(encoding="utf-8") as hf:
        changes = hf.read()

    if style == "rst":
        hyperlink_replacements = {
            r":issue:`([0-9]+)`": r"`GH/\1 <https://github.com/Ouranosinc/miranda/issues/\1>`_",
            r":pull:`([0-9]+)`": r"`PR/\1 <https://github.com/Ouranosinc/miranda/pull/\>`_",
            r":user:`([a-zA-Z0-9_.-]+)`": r"`@\1 <https://github.com/\1>`_",
        }
    elif style == "md":
        hyperlink_replacements = {
            r":issue:`([0-9]+)`": r"[GH/\1](https://github.com/Ouranosinc/miranda/issues/\1)",
            r":pull:`([0-9]+)`": r"[PR/\1](https://github.com/Ouranosinc/miranda/pull/\1)",
            r":user:`([a-zA-Z0-9_.-]+)`": r"[@\1](https://github.com/\1)",
        }
    else:
        msg = f"Formatting style not supported: {style}"
        raise NotImplementedError(msg)

    for search, replacement in hyperlink_replacements.items():
        changes = re.sub(search, replacement, changes)

    if style == "md":
        changes = changes.replace("=========\nChangelog\n=========", "# Changelog")

        titles = {r"\n(.*?)\n([\-]{1,})": "-", r"\n(.*?)\n([\^]{1,})": "^"}
        for title_expression, level in titles.items():
            found = re.findall(title_expression, changes)
            for grouping in found:
                fixed_grouping = (
                    str(grouping[0]).replace("(", r"\(").replace(")", r"\)")
                )
                search = rf"({fixed_grouping})\n([\{level}]{'{' + str(len(grouping[1])) + '}'})"
                replacement = f"{'##' if level == '-' else '###'} {grouping[0]}"
                changes = re.sub(search, replacement, changes)

        link_expressions = r"[\`]{1}([\w\s]+)\s<(.+)>`\_"
        found = re.findall(link_expressions, changes)
        for grouping in found:
            search = rf"`{grouping[0]} <.+>`\_"
            replacement = f"[{str(grouping[0]).strip()}]({grouping[1]})"
            changes = re.sub(search, replacement, changes)

    if not file:
        return changes
    if isinstance(file, Path | os.PathLike):
        with Path(file).open(mode="w", encoding="utf-8") as f:
            print(changes, file=f)
    else:
        print(changes, file=file)
    return None


def show_versions(
    file: os.PathLike | StringIO | TextIO | None = None,
    deps: list | None = None,
) -> str | None:
    """
    Print the versions of miranda and its dependencies.

    Parameters
    ----------
    file : {os.PathLike, StringIO, TextIO}, optional
        If provided, prints to the given file-like object. Otherwise, returns a string.
    deps : list, optional
        A list of dependencies to gather and print version information from. Otherwise, prints `miranda` dependencies.

    Returns
    -------
    str or None
        The formatted version information if `file` is not provided, otherwise None.
    """

    def _get_miranda_dependencies():
        xscen_metadata = ilm.metadata("miranda")
        requires = xscen_metadata.get_all("Requires-Dist")
        requires = [
            req.split("[")[0]
            .split(";")[0]
            .split(">")[0]
            .split("<")[0]
            .split("=")[0]
            .split("!")[0]
            for req in requires
        ]

        return ["xscen"] + requires

    if deps is None:
        deps = _get_miranda_dependencies()

    return _show_versions(file=file, deps=deps)


# Test Data Utilities ###


def testing_setup_warnings():
    """Warn users about potential incompatibilities between miranda and miranda-testdata versions."""
    if (
        re.match(r"^\d+\.\d+\.\d+$", __miranda_version__)
        and TESTDATA_BRANCH != default_testdata_version
    ):
        # This does not need to be emitted on GitHub Workflows and ReadTheDocs
        if not os.getenv("CI") and not os.getenv("READTHEDOCS"):
            warnings.warn(
                f"`miranda` stable ({__miranda_version__}) is running tests against a non-default "
                f"branch of the testing data. It is possible that changes to the testing data may "
                f"be incompatible with some assertions in this version. "
                f"Please be sure to check {TESTDATA_REPO_URL} for more information.",
            )

    if re.match(r"^v\d+\.\d+\.\d+", TESTDATA_BRANCH):
        # Find the date of last modification of miranda source files to generate a calendar version
        install_date = dt.strptime(
            time.ctime(Path(miranda.__file__).stat().st_mtime),
            "%a %b %d %H:%M:%S %Y",
        )
        install_calendar_version = (
            f"{install_date.year}.{install_date.month}.{install_date.day}"
        )

        if Version(TESTDATA_BRANCH) > Version(install_calendar_version):
            warnings.warn(
                f"The installation date of `miranda` ({install_date.ctime()}) "
                f"predates the last release of testing data ({TESTDATA_BRANCH}). "
                "It is very likely that the testing data is incompatible with this build of `miranda`.",
            )


def load_registry(
    branch: str = TESTDATA_BRANCH, repo: str = TESTDATA_REPO_URL
) -> dict[str, str]:
    """
    Load the registry file for the test data.

    Parameters
    ----------
    branch : str
        Branch of the repository to use when fetching testing datasets.
    repo : str
        URL of the repository to use when fetching testing datasets.

    Returns
    -------
    dict
        Dictionary of filenames and hashes.
    """
    if not repo.endswith("/"):
        repo = f"{repo}/"
    remote_registry = audit_url(
        urljoin(
            urljoin(repo, branch if branch.endswith("/") else f"{branch}/"),
            "data/registry.txt",
        )
    )

    if repo != default_testdata_repo_url:
        external_repo_name = urlparse(repo).path.split("/")[-2]
        external_branch_name = branch.split("/")[-1]
        registry_file = Path(
            str(
                ilr.files("miranda").joinpath(
                    f"testing/registry.{external_repo_name}.{external_branch_name}.txt"
                )
            )
        )
        urlretrieve(remote_registry, registry_file)  # noqa: S310

    elif branch != default_testdata_version:
        custom_registry_folder = Path(
            str(ilr.files("miranda").joinpath(f"testing/{branch}"))
        )
        custom_registry_folder.mkdir(parents=True, exist_ok=True)
        registry_file = custom_registry_folder.joinpath("registry.txt")
        urlretrieve(remote_registry, registry_file)  # noqa: S310

    else:
        registry_file = Path(str(ilr.files("miranda").joinpath("testing/registry.txt")))

    if not registry_file.exists():
        raise FileNotFoundError(f"Registry file not found: {registry_file}")

    # Load the registry file
    with registry_file.open(encoding="utf-8") as f:
        registry = {line.split()[0]: line.split()[1] for line in f}
    return registry


def cassini(
    repo: str = TESTDATA_REPO_URL,
    branch: str = TESTDATA_BRANCH,
    cache_dir: str | Path = TESTDATA_CACHE_DIR,
    allow_updates: bool = True,
):
    """
    Pooch registry instance for miranda test data.

    Parameters
    ----------
    repo : str
        URL of the repository to use when fetching testing datasets.
    branch : str
        Branch of repository to use when fetching testing datasets.
    cache_dir : str or Path
        The path to the directory where the data files are stored.
    allow_updates : bool
        If True, allow updates to the data files. Default is True.

    Returns
    -------
    pooch.Pooch
        The Pooch instance for accessing the miranda testing data.

    Notes
    -----
    There are three environment variables that can be used to control the behaviour of this registry:
        - ``MIRANDA_TESTDATA_CACHE_DIR``: If this environment variable is set, it will be used as the
          base directory to store the data files.
          The directory should be an absolute path (i.e., it should start with ``/``).
          Otherwise, the default location will be used (based on ``platformdirs``, see :py:func:`pooch.os_cache`).
        - ``MIRANDA_TESTDATA_REPO_URL``: If this environment variable is set, it will be used as the URL of
          the repository to use when fetching datasets. Otherwise, the default repository will be used.
        - ``MIRANDA_TESTDATA_BRANCH``: If this environment variable is set, it will be used as the branch of
          the repository to use when fetching datasets. Otherwise, the default branch will be used.

    Examples
    --------
    Using the registry to download a file:

    .. code-block:: python

        import xarray as xr
        from miranda.testing import cassini

        example_file = cassini().fetch("example.nc")
        data = xr.open_dataset(example_file)
    """
    if pooch is None:
        raise ImportError(
            "The `pooch` package is required to fetch the miranda testing data. "
            "You can install it with `pip install pooch` or `pip install miranda[dev]`."
        )
    if not repo.endswith("/"):
        repo = f"{repo}/"
    remote = audit_url(
        urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data")
    )

    _cassini = pooch.create(
        path=cache_dir,
        base_url=remote,
        version=default_testdata_version,
        version_dev=branch,
        allow_updates=allow_updates,
        registry=load_registry(branch=branch, repo=repo),
    )

    # Add a custom fetch method to the Pooch instance
    # Needed to address: https://github.com/readthedocs/readthedocs.org/issues/11763
    # Fix inspired by @bjlittle (https://github.com/bjlittle/geovista/pull/1202)
    _cassini.fetch_diversion = _cassini.fetch

    # Overload the fetch method to add user-agent headers
    @wraps(_cassini.fetch_diversion)
    def _fetch(
        *args, **kwargs: bool | Callable
    ) -> str:  # numpydoc ignore=GL08  # *args: str
        def _downloader(
            url: str,
            output_file: str | IO,
            poocher: pooch.Pooch,
            check_only: bool | None = False,
        ) -> None:
            """Download the file from the URL and save it to the save_path."""
            headers = {"User-Agent": f"miranda ({__miranda_version__})"}
            downloader = pooch.HTTPDownloader(headers=headers)
            return downloader(url, output_file, poocher, check_only=check_only)

        # default to our http/s downloader with user-agent headers
        kwargs.setdefault("downloader", _downloader)
        return _cassini.fetch_diversion(*args, **kwargs)

    # Replace the fetch method with the custom fetch method
    _cassini.fetch = _fetch

    return _cassini


def open_dataset(
    name: str,
    cassini_kwargs: dict[str, Path | str | bool] | None = None,
    **xr_kwargs: Any,
) -> Dataset:
    r"""
    Convenience function to open a dataset from the miranda testing data using the `cassini` class.

    This is a thin wrapper around the `cassini` class to make it easier to open miranda testing datasets.

    Parameters
    ----------
    name : str
        Name of the file containing the dataset.
    cassini_kwargs : dict
        Keyword arguments passed to the cassini function.
    **xr_kwargs : Any
        Keyword arguments passed to xarray.open_dataset.

    Returns
    -------
    xarray.Dataset
        The dataset.

    See Also
    --------
    xarray.open_dataset : Open and read a dataset from a file or file-like object.
    cassini : Pooch wrapper for accessing the miranda testing data.
    """
    if cassini_kwargs is None:
        cassini_kwargs = {}
    return _open_dataset(cassini(**cassini_kwargs).fetch(name), **xr_kwargs)


def populate_testing_data(
    temp_folder: Path | None = None,
    repo: str = TESTDATA_REPO_URL,
    branch: str = TESTDATA_BRANCH,
    local_cache: Path = TESTDATA_CACHE_DIR,
) -> None:
    """
    Populate the local cache with the testing data.

    Parameters
    ----------
    temp_folder : Path, optional
        Path to a temporary folder to use as the local cache. If not provided, the default location will be used.
    repo : str, optional
        URL of the repository to use when fetching testing datasets.
    branch : str, optional
        Branch of miranda-testdata to use when fetching testing datasets.
    local_cache : Path
        The path to the local cache. Defaults to the location set by the platformdirs library.
        The testing data will be downloaded to this local cache.
    """
    # Create the Pooch instance
    n = cassini(repo=repo, branch=branch, cache_dir=temp_folder or local_cache)

    # Download the files
    errored_files = []
    for file in load_registry():
        try:
            n.fetch(file)
        except HTTPError:  # noqa: PERF203
            msg = f"File `{file}` not accessible in remote repository."
            logging.error(msg)
            errored_files.append(file)
        else:
            logging.info("Files were downloaded successfully.")

    if errored_files:
        logging.error(
            "The following files were unable to be downloaded: %s",
            errored_files,
        )


def gather_testing_data(
    worker_cache_dir: str | os.PathLike[str] | Path,
    worker_id: str,
    _cache_dir: str | os.PathLike[str] | None = TESTDATA_CACHE_DIR,
) -> None:
    """
    Gather testing data across workers.

    Parameters
    ----------
    worker_cache_dir : str or Path
        The directory to store the testing data.
    worker_id : str
        The worker ID.
    _cache_dir : str or Path, optional
        The directory to store the testing data. Default is None.

    Raises
    ------
    ValueError
        If the cache directory is not set.
    FileNotFoundError
        If the testing data is not found.
    """
    if _cache_dir is None:
        raise ValueError(
            "The cache directory must be set. "
            "Please set the `cache_dir` parameter or the `MIRANDA_DATA_DIR` environment variable."
        )
    cache_dir = Path(_cache_dir)

    if worker_id == "master":
        populate_testing_data(branch=TESTDATA_BRANCH)
    else:
        cache_dir.mkdir(exist_ok=True, parents=True)
        lockfile = cache_dir.joinpath(".lock")
        test_data_being_written = FileLock(lockfile)
        with test_data_being_written:
            # This flag prevents multiple calls from re-attempting to download testing data in the same pytest run
            populate_testing_data(branch=TESTDATA_BRANCH)
            cache_dir.joinpath(".data_written").touch()
        with test_data_being_written.acquire():
            if lockfile.exists():
                lockfile.unlink()
        copytree(cache_dir.joinpath(default_testdata_version), worker_cache_dir)


# Testing Utilities ###


def audit_url(url: str, context: str | None = None) -> str:
    """
    Check if the URL is well-formed.

    Parameters
    ----------
    url : str
        The URL to check.
    context : str, optional
        Additional context to include in the error message. Default is None.

    Returns
    -------
    str
        The URL if it is well-formed.

    Raises
    ------
    URLError
        If the URL is not well-formed.
    """
    msg = ""
    result = urlparse(url)
    if result.scheme == "http":
        msg = f"{context if context else ''} URL is not using secure HTTP: '{url}'".strip()
    if not all([result.scheme, result.netloc]):
        msg = f"{context if context else ''} URL is not well-formed: '{url}'".strip()

    if msg:
        logger.error(msg)
        raise URLError(msg)
    return url

1	"""Testing utilities module."""
2
3	from __future__ import annotations	9✔
4
5	import importlib.metadata as ilm	9✔
6	import importlib.resources as ilr	9✔
7	import logging	9✔
8	import os	9✔
9	import re	9✔
10	import time	9✔
11	import warnings	9✔
12	from collections.abc import Callable	9✔
13	from datetime import datetime as dt	9✔
14	from functools import wraps	9✔
15	from io import StringIO	9✔
16	from pathlib import Path	9✔
17	from shutil import copytree	9✔
18	from typing import IO, Any, TextIO	9✔
19	from urllib.error import HTTPError, URLError	9✔
20	from urllib.parse import urljoin, urlparse	9✔
21	from urllib.request import urlretrieve	9✔
22
23	from filelock import FileLock	9✔
24	from packaging.version import Version	9✔
25	from xarray import Dataset	9✔
26	from xarray import open_dataset as _open_dataset	9✔
27	from xclim.testing.utils import show_versions as _show_versions	9✔
28
29	from miranda import __version__ as __miranda_version__	9✔
30
31	try:	9✔
32	import pytest	9✔
33	from pytest_socket import SocketBlockedError	9✔
34	except ImportError:	9✔
35	pytest = None	9✔
36	SocketBlockedError = None	9✔
37
38	try:	9✔
39	import pooch	9✔
NEW 40	except ImportError:	×
NEW 41	warnings.warn(	×
42	"The `pooch` library is not installed. The default cache directory for testing data will not be set."
43	)
NEW 44	pooch = None	×
45
46	import miranda	9✔
47
48	logger = logging.getLogger("miranda")	9✔
49
50
51	__all__ = [	9✔
52	"TESTDATA_BRANCH",
53	"TESTDATA_CACHE_DIR",
54	"TESTDATA_REPO_URL",
55	"audit_url",
56	"cassini",
57	"default_testdata_cache",
58	"default_testdata_repo_url",
59	"default_testdata_version",
60	"gather_testing_data",
61	"open_dataset",
62	"populate_testing_data",
63	"publish_release_notes",
64	"show_versions",
65	"testing_setup_warnings",
66	]
67
68	default_testdata_version = "v2025.5.16"	9✔
69	"""Default version of the testing data to use when fetching datasets."""	7✔
70
71	default_testdata_repo_url = (	9✔
72	"https://raw.githubusercontent.com/Ouranosinc/miranda-testdata/"
73	)
74	"""Default URL of the testing data repository to use when fetching datasets."""	7✔
75
76	try:	9✔
77	default_testdata_cache = Path(pooch.os_cache("miranda-testdata"))	9✔
78	"""Default location for the testing data cache."""	9✔
NEW 79	except AttributeError:	×
NEW 80	default_testdata_cache = None	×
81
82	TESTDATA_REPO_URL = str(	9✔
83	os.getenv("MIRANDA_TESTDATA_REPO_URL", default_testdata_repo_url)
84	)
85	"""	7✔
86	Sets the URL of the testing data repository to use when fetching datasets.
87
88	Notes
89	-----
90	When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:
91
92	.. code-block:: console
93
94	$ export MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata"
95
96	or setting the variable at runtime:
97
98	.. code-block:: console
99
100	$ env MIRANDA_TESTDATA_REPO_URL="https://github.com/my_username/miranda-testdata" pytest
101	"""
102
103	TESTDATA_BRANCH = str(os.getenv("MIRANDA_TESTDATA_BRANCH", default_testdata_version))	9✔
104	"""	7✔
105	Sets the branch of the testing data repository to use when fetching datasets.
106
107	Notes
108	-----
109	When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:
110
111	.. code-block:: console
112
113	$ export MIRANDA_TESTDATA_BRANCH="my_testing_branch"
114
115	or setting the variable at runtime:
116
117	.. code-block:: console
118
119	$ env MIRANDA_TESTDATA_BRANCH="my_testing_branch" pytest
120	"""
121
122	TESTDATA_CACHE_DIR = os.getenv("MIRANDA_TESTDATA_CACHE_DIR", default_testdata_cache)	9✔
123	"""	7✔
124	Sets the directory to store the testing datasets.
125
126	If not set, the default location will be used (based on ``platformdirs``, see :func:`pooch.os_cache`).
127
128	Notes
129	-----
130	When running tests locally, this can be set for both `pytest` and `tox` by exporting the variable:
131
132	.. code-block:: console
133
134	$ export MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data"
135
136	or setting the variable at runtime:
137
138	.. code-block:: console
139
140	$ env MIRANDA_TESTDATA_CACHE_DIR="/path/to/my/data" pytest
141	"""
142
143
144	# Publishing Tools ###
145
146
147	def publish_release_notes(	9✔
148	style: str = "md",
149	file: os.PathLike[str] \| StringIO \| TextIO \| None = None,
150	changes: str \| os.PathLike[str] \| None = None,
151	) -> str \| None:
152	"""
153	Format release notes in Markdown or ReStructuredText.
154
155	Parameters
156	----------
157	style : {"rst", "md"}
158	Use ReStructuredText formatting or Markdown. Default: Markdown.
159	file : {os.PathLike, StringIO, TextIO}, optional
160	If provided, prints to the given file-like object. Otherwise, returns a string.
161	changes : str or os.PathLike[str], optional
162	If provided, manually points to the file where the changelog can be found.
163	Assumes a relative path otherwise.
164
165	Returns
166	-------
167	str, optional
168	If `file` not provided, the formatted release notes.
169
170	Notes
171	-----
172	This function is used solely for development and packaging purposes.
173	"""
NEW 174	if isinstance(changes, str \| Path):	×
NEW 175	changes_file = Path(changes).absolute()	×
176	else:
NEW 177	changes_file = Path(__file__).absolute().parents[3].joinpath("CHANGELOG.rst")	×
178
NEW 179	if not changes_file.exists():	×
NEW 180	raise FileNotFoundError("Changelog file not found in miranda folder tree.")	×
181
NEW 182	with Path(changes_file).open(encoding="utf-8") as hf:	×
NEW 183	changes = hf.read()	×
184
NEW 185	if style == "rst":	×
NEW 186	hyperlink_replacements = {	×
187	r":issue:`([0-9]+)`": r"`GH/\1 <https://github.com/Ouranosinc/miranda/issues/\1>`_",
188	r":pull:`([0-9]+)`": r"`PR/\1 <https://github.com/Ouranosinc/miranda/pull/\>`_",
189	r":user:`([a-zA-Z0-9_.-]+)`": r"`@\1 <https://github.com/\1>`_",
190	}
NEW 191	elif style == "md":	×
NEW 192	hyperlink_replacements = {	×
193	r":issue:`([0-9]+)`": r"[GH/\1](https://github.com/Ouranosinc/miranda/issues/\1)",
194	r":pull:`([0-9]+)`": r"[PR/\1](https://github.com/Ouranosinc/miranda/pull/\1)",
195	r":user:`([a-zA-Z0-9_.-]+)`": r"[@\1](https://github.com/\1)",
196	}
197	else:
NEW 198	msg = f"Formatting style not supported: {style}"	×
NEW 199	raise NotImplementedError(msg)	×
200
NEW 201	for search, replacement in hyperlink_replacements.items():	×
NEW 202	changes = re.sub(search, replacement, changes)	×
203
NEW 204	if style == "md":	×
NEW 205	changes = changes.replace("=========\nChangelog\n=========", "# Changelog")	×
206
NEW 207	titles = {r"\n(.?)\n([\-]{1,})": "-", r"\n(.?)\n([\^]{1,})": "^"}	×
NEW 208	for title_expression, level in titles.items():	×
NEW 209	found = re.findall(title_expression, changes)	×
NEW 210	for grouping in found:	×
NEW 211	fixed_grouping = (	×
212	str(grouping[0]).replace("(", r"\(").replace(")", r"\)")
213	)
NEW 214	search = rf"({fixed_grouping})\n([\{level}]{'{' + str(len(grouping[1])) + '}'})"	×
NEW 215	replacement = f"{'##' if level == '-' else '###'} {grouping[0]}"	×
NEW 216	changes = re.sub(search, replacement, changes)	×
217
NEW 218	link_expressions = r"[\`]{1}([\w\s]+)\s<(.+)>`\_"	×
NEW 219	found = re.findall(link_expressions, changes)	×
NEW 220	for grouping in found:	×
NEW 221	search = rf"`{grouping[0]} <.+>`\_"	×
NEW 222	replacement = f"[{str(grouping[0]).strip()}]({grouping[1]})"	×
NEW 223	changes = re.sub(search, replacement, changes)	×
224
NEW 225	if not file:	×
NEW 226	return changes	×
NEW 227	if isinstance(file, Path \| os.PathLike):	×
NEW 228	with Path(file).open(mode="w", encoding="utf-8") as f:	×
NEW 229	print(changes, file=f)	×
230	else:
NEW 231	print(changes, file=file)	×
NEW 232	return None	×
233
234
235	def show_versions(	9✔
236	file: os.PathLike \| StringIO \| TextIO \| None = None,
237	deps: list \| None = None,
238	) -> str \| None:
239	"""
240	Print the versions of miranda and its dependencies.
241
242	Parameters
243	----------
244	file : {os.PathLike, StringIO, TextIO}, optional
245	If provided, prints to the given file-like object. Otherwise, returns a string.
246	deps : list, optional
247	A list of dependencies to gather and print version information from. Otherwise, prints `miranda` dependencies.
248
249	Returns
250	-------
251	str or None
252	The formatted version information if `file` is not provided, otherwise None.
253	"""
254
NEW 255	def _get_miranda_dependencies():	×
NEW 256	xscen_metadata = ilm.metadata("miranda")	×
NEW 257	requires = xscen_metadata.get_all("Requires-Dist")	×
NEW 258	requires = [	×
259	req.split("[")[0]
260	.split(";")[0]
261	.split(">")[0]
262	.split("<")[0]
263	.split("=")[0]
264	.split("!")[0]
265	for req in requires
266	]
267
NEW 268	return ["xscen"] + requires	×
269
NEW 270	if deps is None:	×
NEW 271	deps = _get_miranda_dependencies()	×
272
NEW 273	return _show_versions(file=file, deps=deps)	×
274
275
276	# Test Data Utilities ###
277
278
279	def testing_setup_warnings():	9✔
280	"""Warn users about potential incompatibilities between miranda and miranda-testdata versions."""
281	if (	9✔
282	re.match(r"^\d+\.\d+\.\d+$", __miranda_version__)
283	and TESTDATA_BRANCH != default_testdata_version
284	):
285	# This does not need to be emitted on GitHub Workflows and ReadTheDocs
NEW 286	if not os.getenv("CI") and not os.getenv("READTHEDOCS"):	×
NEW 287	warnings.warn(	×
288	f"`miranda` stable ({__miranda_version__}) is running tests against a non-default "
289	f"branch of the testing data. It is possible that changes to the testing data may "
290	f"be incompatible with some assertions in this version. "
291	f"Please be sure to check {TESTDATA_REPO_URL} for more information.",
292	)
293
294	if re.match(r"^v\d+\.\d+\.\d+", TESTDATA_BRANCH):	9✔
295	# Find the date of last modification of miranda source files to generate a calendar version
296	install_date = dt.strptime(	9✔
297	time.ctime(Path(miranda.__file__).stat().st_mtime),
298	"%a %b %d %H:%M:%S %Y",
299	)
300	install_calendar_version = (	9✔
301	f"{install_date.year}.{install_date.month}.{install_date.day}"
302	)
303
304	if Version(TESTDATA_BRANCH) > Version(install_calendar_version):	9✔
NEW 305	warnings.warn(	×
306	f"The installation date of `miranda` ({install_date.ctime()}) "
307	f"predates the last release of testing data ({TESTDATA_BRANCH}). "
308	"It is very likely that the testing data is incompatible with this build of `miranda`.",
309	)
310
311
312	def load_registry(	9✔
313	branch: str = TESTDATA_BRANCH, repo: str = TESTDATA_REPO_URL
314	) -> dict[str, str]:
315	"""
316	Load the registry file for the test data.
317
318	Parameters
319	----------
320	branch : str
321	Branch of the repository to use when fetching testing datasets.
322	repo : str
323	URL of the repository to use when fetching testing datasets.
324
325	Returns
326	-------
327	dict
328	Dictionary of filenames and hashes.
329	"""
330	if not repo.endswith("/"):	9✔
NEW 331	repo = f"{repo}/"	×
332	remote_registry = audit_url(	9✔
333	urljoin(
334	urljoin(repo, branch if branch.endswith("/") else f"{branch}/"),
335	"data/registry.txt",
336	)
337	)
338
339	if repo != default_testdata_repo_url:	9✔
NEW 340	external_repo_name = urlparse(repo).path.split("/")[-2]	×
NEW 341	external_branch_name = branch.split("/")[-1]	×
NEW 342	registry_file = Path(	×
343	str(
344	ilr.files("miranda").joinpath(
345	f"testing/registry.{external_repo_name}.{external_branch_name}.txt"
346	)
347	)
348	)
NEW 349	urlretrieve(remote_registry, registry_file) # noqa: S310	×
350
351	elif branch != default_testdata_version:	9✔
NEW 352	custom_registry_folder = Path(	×
353	str(ilr.files("miranda").joinpath(f"testing/{branch}"))
354	)
NEW 355	custom_registry_folder.mkdir(parents=True, exist_ok=True)	×
NEW 356	registry_file = custom_registry_folder.joinpath("registry.txt")	×
NEW 357	urlretrieve(remote_registry, registry_file) # noqa: S310	×
358
359	else:
360	registry_file = Path(str(ilr.files("miranda").joinpath("testing/registry.txt")))	9✔
361
362	if not registry_file.exists():	9✔
NEW 363	raise FileNotFoundError(f"Registry file not found: {registry_file}")	×
364
365	# Load the registry file
366	with registry_file.open(encoding="utf-8") as f:	9✔
367	registry = {line.split()[0]: line.split()[1] for line in f}	9✔
368	return registry	9✔
369
370
371	def cassini(	9✔
372	repo: str = TESTDATA_REPO_URL,
373	branch: str = TESTDATA_BRANCH,
374	cache_dir: str \| Path = TESTDATA_CACHE_DIR,
375	allow_updates: bool = True,
376	):
377	"""
378	Pooch registry instance for miranda test data.
379
380	Parameters
381	----------
382	repo : str
383	URL of the repository to use when fetching testing datasets.
384	branch : str
385	Branch of repository to use when fetching testing datasets.
386	cache_dir : str or Path
387	The path to the directory where the data files are stored.
388	allow_updates : bool
389	If True, allow updates to the data files. Default is True.
390
391	Returns
392	-------
393	pooch.Pooch
394	The Pooch instance for accessing the miranda testing data.
395
396	Notes
397	-----
398	There are three environment variables that can be used to control the behaviour of this registry:
399	- ``MIRANDA_TESTDATA_CACHE_DIR``: If this environment variable is set, it will be used as the
400	base directory to store the data files.
401	The directory should be an absolute path (i.e., it should start with ``/``).
402	Otherwise, the default location will be used (based on ``platformdirs``, see :py:func:`pooch.os_cache`).
403	- ``MIRANDA_TESTDATA_REPO_URL``: If this environment variable is set, it will be used as the URL of
404	the repository to use when fetching datasets. Otherwise, the default repository will be used.
405	- ``MIRANDA_TESTDATA_BRANCH``: If this environment variable is set, it will be used as the branch of
406	the repository to use when fetching datasets. Otherwise, the default branch will be used.
407
408	Examples
409	--------
410	Using the registry to download a file:
411
412	.. code-block:: python
413
414	import xarray as xr
415	from miranda.testing import cassini
416
417	example_file = cassini().fetch("example.nc")
418	data = xr.open_dataset(example_file)
419	"""
420	if pooch is None:	9✔
NEW 421	raise ImportError(	×
422	"The `pooch` package is required to fetch the miranda testing data. "
423	"You can install it with `pip install pooch` or `pip install miranda[dev]`."
424	)
425	if not repo.endswith("/"):	9✔
NEW 426	repo = f"{repo}/"	×
427	remote = audit_url(	9✔
428	urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data")
429	)
430
431	_cassini = pooch.create(	9✔
432	path=cache_dir,
433	base_url=remote,
434	version=default_testdata_version,
435	version_dev=branch,
436	allow_updates=allow_updates,
437	registry=load_registry(branch=branch, repo=repo),
438	)
439
440	# Add a custom fetch method to the Pooch instance
441	# Needed to address: https://github.com/readthedocs/readthedocs.org/issues/11763
442	# Fix inspired by @bjlittle (https://github.com/bjlittle/geovista/pull/1202)
443	_cassini.fetch_diversion = _cassini.fetch	9✔
444
445	# Overload the fetch method to add user-agent headers
446	@wraps(_cassini.fetch_diversion)	9✔
447	def _fetch(	9✔
448	args, *kwargs: bool \| Callable
449	) -> str: # numpydoc ignore=GL08 # *args: str
450	def _downloader(	9✔
451	url: str,
452	output_file: str \| IO,
453	poocher: pooch.Pooch,
454	check_only: bool \| None = False,
455	) -> None:
456	"""Download the file from the URL and save it to the save_path."""
457	headers = {"User-Agent": f"miranda ({__miranda_version__})"}	9✔
458	downloader = pooch.HTTPDownloader(headers=headers)	9✔
459	return downloader(url, output_file, poocher, check_only=check_only)	9✔
460
461	# default to our http/s downloader with user-agent headers
462	kwargs.setdefault("downloader", _downloader)	9✔
463	return _cassini.fetch_diversion(args, *kwargs)	9✔
464
465	# Replace the fetch method with the custom fetch method
466	_cassini.fetch = _fetch	9✔
467
468	return _cassini	9✔
469
470
471	def open_dataset(	9✔
472	name: str,
473	cassini_kwargs: dict[str, Path \| str \| bool] \| None = None,
474	**xr_kwargs: Any,
475	) -> Dataset:
476	r"""
477	Convenience function to open a dataset from the miranda testing data using the `cassini` class.
478
479	This is a thin wrapper around the `cassini` class to make it easier to open miranda testing datasets.
480
481	Parameters
482	----------
483	name : str
484	Name of the file containing the dataset.
485	cassini_kwargs : dict
486	Keyword arguments passed to the cassini function.
487	**xr_kwargs : Any
488	Keyword arguments passed to xarray.open_dataset.
489
490	Returns
491	-------
492	xarray.Dataset
493	The dataset.
494
495	See Also
496	--------
497	xarray.open_dataset : Open and read a dataset from a file or file-like object.
498	cassini : Pooch wrapper for accessing the miranda testing data.
499	"""
NEW 500	if cassini_kwargs is None:	×
NEW 501	cassini_kwargs = {}	×
NEW 502	return _open_dataset(cassini(cassini_kwargs).fetch(name), xr_kwargs)	×
503
504
505	def populate_testing_data(	9✔
506	temp_folder: Path \| None = None,
507	repo: str = TESTDATA_REPO_URL,
508	branch: str = TESTDATA_BRANCH,
509	local_cache: Path = TESTDATA_CACHE_DIR,
510	) -> None:
511	"""
512	Populate the local cache with the testing data.
513
514	Parameters
515	----------
516	temp_folder : Path, optional
517	Path to a temporary folder to use as the local cache. If not provided, the default location will be used.
518	repo : str, optional
519	URL of the repository to use when fetching testing datasets.
520	branch : str, optional
521	Branch of miranda-testdata to use when fetching testing datasets.
522	local_cache : Path
523	The path to the local cache. Defaults to the location set by the platformdirs library.
524	The testing data will be downloaded to this local cache.
525	"""
526	# Create the Pooch instance
527	n = cassini(repo=repo, branch=branch, cache_dir=temp_folder or local_cache)	9✔
528
529	# Download the files
530	errored_files = []	9✔
531	for file in load_registry():	9✔
532	try:	9✔
533	n.fetch(file)	9✔
NEW 534	except HTTPError: # noqa: PERF203	×
NEW 535	msg = f"File `{file}` not accessible in remote repository."	×
NEW 536	logging.error(msg)	×
NEW 537	errored_files.append(file)	×
538	else:
539	logging.info("Files were downloaded successfully.")	9✔
540
541	if errored_files:	9✔
NEW 542	logging.error(	×
543	"The following files were unable to be downloaded: %s",
544	errored_files,
545	)
546
547
548	def gather_testing_data(	9✔
549	worker_cache_dir: str \| os.PathLike[str] \| Path,
550	worker_id: str,
551	_cache_dir: str \| os.PathLike[str] \| None = TESTDATA_CACHE_DIR,
552	) -> None:
553	"""
554	Gather testing data across workers.
555
556	Parameters
557	----------
558	worker_cache_dir : str or Path
559	The directory to store the testing data.
560	worker_id : str
561	The worker ID.
562	_cache_dir : str or Path, optional
563	The directory to store the testing data. Default is None.
564
565	Raises
566	------
567	ValueError
568	If the cache directory is not set.
569	FileNotFoundError
570	If the testing data is not found.
571	"""
572	if _cache_dir is None:	9✔
NEW 573	raise ValueError(	×
574	"The cache directory must be set. "
575	"Please set the `cache_dir` parameter or the `MIRANDA_DATA_DIR` environment variable."
576	)
577	cache_dir = Path(_cache_dir)	9✔
578
579	if worker_id == "master":	9✔
580	populate_testing_data(branch=TESTDATA_BRANCH)	5✔
581	else:
582	cache_dir.mkdir(exist_ok=True, parents=True)	4✔
583	lockfile = cache_dir.joinpath(".lock")	4✔
584	test_data_being_written = FileLock(lockfile)	4✔
585	with test_data_being_written:	4✔
586	# This flag prevents multiple calls from re-attempting to download testing data in the same pytest run
587	populate_testing_data(branch=TESTDATA_BRANCH)	4✔
588	cache_dir.joinpath(".data_written").touch()	4✔
589	with test_data_being_written.acquire():	4✔
590	if lockfile.exists():	4✔
591	lockfile.unlink()	4✔
592	copytree(cache_dir.joinpath(default_testdata_version), worker_cache_dir)	4✔
593
594
595	# Testing Utilities ###
596
597
598	def audit_url(url: str, context: str \| None = None) -> str:	9✔
599	"""
600	Check if the URL is well-formed.
601
602	Parameters
603	----------
604	url : str
605	The URL to check.
606	context : str, optional
607	Additional context to include in the error message. Default is None.
608
609	Returns
610	-------
611	str
612	The URL if it is well-formed.
613
614	Raises
615	------
616	URLError
617	If the URL is not well-formed.
618	"""
619	msg = ""	9✔
620	result = urlparse(url)	9✔
621	if result.scheme == "http":	9✔
NEW 622	msg = f"{context if context else ''} URL is not using secure HTTP: '{url}'".strip()	×
623	if not all([result.scheme, result.netloc]):	9✔
NEW 624	msg = f"{context if context else ''} URL is not well-formed: '{url}'".strip()	×
625
626	if msg:	9✔
NEW 627	logger.error(msg)	×
NEW 628	raise URLError(msg)	×
629	return url	9✔

Ouranosinc / miranda / 15141552897

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous