11664690674

Committed 04 Nov 2024 12:58PM UTC coverage: 71.143% (-0.04%) from 71.187%

Build # 11664690674

Build Type

Pull #2115

github

Committed by

web-flow

Commit Message

Merge b2b1ef8f9 into 6b684d173

Pull Request Pull Request #2115: feat: add hint for new noarch: python syntax

Run Details

1451 of 2160 branches covered (67.18%)

8 of 14 new or added lines in 2 files covered. (57.14%)

9 existing lines in 1 file now uncovered.

3205 of 4505 relevant lines covered (71.14%)

0.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.71

/conda_smithy/linter/hints.py

import os
import re
import shutil
import subprocess
import sys
from glob import glob
from typing import Any, Dict, List

from conda_smithy.linter import conda_recipe_v1_linter
from conda_smithy.linter.errors import HINT_NO_ARCH
from conda_smithy.linter.utils import (
    VALID_PYTHON_BUILD_BACKENDS,
    find_local_config_file,
    is_selector_line,
)
from conda_smithy.utils import get_yaml


def hint_pip_usage(build_section, hints):
    if "script" in build_section:
        scripts = build_section["script"]
        if isinstance(scripts, str):
            scripts = [scripts]
        for script in scripts:
            if "python setup.py install" in script:
                hints.append(
                    "Whenever possible python packages should use pip. "
                    "See https://conda-forge.org/docs/maintainer/adding_pkgs.html#use-pip"
                )


def hint_sources_should_not_mention_pypi_io_but_pypi_org(
    sources_section: List[Dict[str, Any]], hints: List[str]
):
    """
    Grayskull and conda-forge default recipe used to have pypi.io as a default,
    but cannonical url is PyPI.org.

    See https://github.com/conda-forge/staged-recipes/pull/27946
    """
    for source_section in sources_section:
        source = source_section.get("url", "") or ""
        sources = [source] if isinstance(source, str) else source
        if any(s.startswith("https://pypi.io/") for s in sources):
            hints.append(
                "PyPI default URL is now pypi.org, and not pypi.io."
                " You may want to update the default source url."
            )


def hint_suggest_noarch(
    noarch_value,
    build_reqs,
    raw_requirements_section,
    is_staged_recipes,
    conda_forge,
    recipe_fname,
    hints,
    recipe_version: int = 0,
):
    if (
        noarch_value is None
        and build_reqs
        and not any(["_compiler_stub" in b for b in build_reqs])
        and ("pip" in build_reqs)
        and (is_staged_recipes or not conda_forge)
    ):
        if recipe_version == 1:
            conda_recipe_v1_linter.hint_noarch_usage(
                build_reqs, raw_requirements_section, hints
            )
        else:
            with open(recipe_fname) as fh:
                in_runreqs = False
                no_arch_possible = True
                for line in fh:
                    line_s = line.strip()
                    if line_s == "host:" or line_s == "run:":
                        in_runreqs = True
                        runreqs_spacing = line[: -len(line.lstrip())]
                        continue
                    if line_s.startswith("skip:") and is_selector_line(line):
                        no_arch_possible = False
                        break
                    if in_runreqs:
                        if runreqs_spacing == line[: -len(line.lstrip())]:
                            in_runreqs = False
                            continue
                        if is_selector_line(line):
                            no_arch_possible = False
                            break
                if no_arch_possible:
                    hints.append(HINT_NO_ARCH)


def hint_shellcheck_usage(recipe_dir, hints):
    shellcheck_enabled = False
    shell_scripts = []
    if recipe_dir:
        shell_scripts = glob(os.path.join(recipe_dir, "*.sh"))
        forge_yaml = find_local_config_file(recipe_dir, "conda-forge.yml")
        if shell_scripts and forge_yaml:
            with open(forge_yaml) as fh:
                code = get_yaml().load(fh)
                shellcheck_enabled = code.get("shellcheck", {}).get(
                    "enabled", shellcheck_enabled
                )

        if shellcheck_enabled and shutil.which("shellcheck") and shell_scripts:
            max_shellcheck_lines = 50
            cmd = [
                "shellcheck",
                "--enable=all",
                "--shell=bash",
                # SC2154: var is referenced but not assigned,
                #         see https://github.com/koalaman/shellcheck/wiki/SC2154
                "--exclude=SC2154",
            ]

            p = subprocess.Popen(
                cmd + shell_scripts,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                env={
                    "PATH": os.getenv("PATH")
                },  # exclude other env variables to protect against token leakage
            )
            sc_stdout, _ = p.communicate()

            if p.returncode == 1:
                # All files successfully scanned with some issues.
                findings = (
                    sc_stdout.decode(sys.stdout.encoding)
                    .replace("\r\n", "\n")
                    .splitlines()
                )
                hints.append(
                    "Whenever possible fix all shellcheck findings ('"
                    + " ".join(cmd)
                    + " recipe/*.sh -f diff | git apply' helps)"
                )
                hints.extend(findings[:50])
                if len(findings) > max_shellcheck_lines:
                    hints.append(
                        "Output restricted, there are '%s' more lines."
                        % (len(findings) - max_shellcheck_lines)
                    )
            elif p.returncode != 0:
                # Something went wrong.
                hints.append(
                    "There have been errors while scanning with shellcheck."
                )


def hint_check_spdx(about_section, hints):
    import license_expression

    license = about_section.get("license", "")
    licensing = license_expression.Licensing()
    parsed_exceptions = []
    try:
        parsed_licenses = []
        parsed_licenses_with_exception = licensing.license_symbols(
            license.strip(), decompose=False
        )
        for li in parsed_licenses_with_exception:
            if isinstance(li, license_expression.LicenseWithExceptionSymbol):
                parsed_licenses.append(li.license_symbol.key)
                parsed_exceptions.append(li.exception_symbol.key)
            else:
                parsed_licenses.append(li.key)
    except license_expression.ExpressionError:
        parsed_licenses = [license]

    licenseref_regex = re.compile(r"^LicenseRef[a-zA-Z0-9\-.]*$")
    filtered_licenses = []
    for license in parsed_licenses:
        if not licenseref_regex.match(license):
            filtered_licenses.append(license)

    with open(os.path.join(os.path.dirname(__file__), "licenses.txt")) as f:
        expected_licenses = f.readlines()
        expected_licenses = set([li.strip() for li in expected_licenses])
    with open(
        os.path.join(os.path.dirname(__file__), "license_exceptions.txt")
    ) as f:
        expected_exceptions = f.readlines()
        expected_exceptions = set([li.strip() for li in expected_exceptions])
    if set(filtered_licenses) - expected_licenses:
        hints.append(
            "License is not an SPDX identifier (or a custom LicenseRef) nor an SPDX license expression.\n\n"
            "Documentation on acceptable licenses can be found "
            "[here]( https://conda-forge.org/docs/maintainer/adding_pkgs.html#spdx-identifiers-and-expressions )."
        )
    if set(parsed_exceptions) - expected_exceptions:
        hints.append(
            "License exception is not an SPDX exception.\n\n"
            "Documentation on acceptable licenses can be found "
            "[here]( https://conda-forge.org/docs/maintainer/adding_pkgs.html#spdx-identifiers-and-expressions )."
        )


def hint_pip_no_build_backend(host_or_build_section, package_name, hints):
    # we do NOT exclude all build backends since some of them
    # need another backend to bootstrap
    # the list below are the ones that self-bootstrap without
    # another build backend
    if package_name in ["pdm-backend", "setuptools"]:
        return

    if host_or_build_section and any(
        req.split(" ")[0] == "pip" for req in host_or_build_section
    ):
        found_backend = False
        for backend in VALID_PYTHON_BUILD_BACKENDS:
            if any(
                req.split(" ")[0]
                in [
                    backend,
                    backend.replace("-", "_"),
                    backend.replace("_", "-"),
                ]
                for req in host_or_build_section
            ):
                found_backend = True
                break

        if not found_backend:
            hints.append(
                f"No valid build backend found for Python recipe for package `{package_name}` using `pip`. Python recipes using `pip` need to "
                "explicitly specify a build backend in the `host` section. "
                "If your recipe has built with only `pip` in the `host` section in the past, you likely should "
                "add `setuptools` to the `host` section of your recipe."
            )


def hint_noarch_python_use_python_min(
    host_reqs, run_reqs, test_reqs, outputs_section, noarch_value, hints
):
    if noarch_value == "python" and not outputs_section:
        for section_name, syntax, reqs in [
            ("host", "python {{ python_min }}.*", host_reqs),
            ("run", "python >={{ python_min }}", run_reqs),
            ("test.requires", "python ={{ python_min }}", test_reqs),
        ]:
            for req in reqs:
                if (
                    req.strip().split()[0] == "python"
                    and req != "python"
                    and syntax in req
                ):
                    break
            else:
                hints.append(
                    f"noarch: python recipes should almost always follow the syntax in "
                    f"[CFEP-25](https://conda-forge.org/docs/maintainer/knowledge_base/#noarch-python). "
                    f"For the `{section_name}` section of the recipe, you should almost always use `{syntax}` "
                    f"for the `python` entry. You may need to override the `python_min` variable if the package "
                    f"requires a newer Python version than the currently supported minimum version on `conda-forge`."
                )

1	import os	1✔
2	import re	1✔
3	import shutil	1✔
4	import subprocess	1✔
5	import sys	1✔
6	from glob import glob	1✔
7	from typing import Any, Dict, List	1✔
8
9	from conda_smithy.linter import conda_recipe_v1_linter	1✔
10	from conda_smithy.linter.errors import HINT_NO_ARCH	1✔
11	from conda_smithy.linter.utils import (	1✔
12	VALID_PYTHON_BUILD_BACKENDS,
13	find_local_config_file,
14	is_selector_line,
15	)
16	from conda_smithy.utils import get_yaml	1✔
17
18
19	def hint_pip_usage(build_section, hints):	1✔
20	if "script" in build_section:	1✔
21	scripts = build_section["script"]	1✔
22	if isinstance(scripts, str):	1✔
23	scripts = [scripts]	1✔
24	for script in scripts:	1✔
25	if "python setup.py install" in script:	1✔
26	hints.append(	1✔
27	"Whenever possible python packages should use pip. "
28	"See https://conda-forge.org/docs/maintainer/adding_pkgs.html#use-pip"
29	)
30
31
32	def hint_sources_should_not_mention_pypi_io_but_pypi_org(	1✔
33	sources_section: List[Dict[str, Any]], hints: List[str]
34	):
35	"""
36	Grayskull and conda-forge default recipe used to have pypi.io as a default,
37	but cannonical url is PyPI.org.
38
39	See https://github.com/conda-forge/staged-recipes/pull/27946
40	"""
41	for source_section in sources_section:	1✔
42	source = source_section.get("url", "") or ""	1✔
43	sources = [source] if isinstance(source, str) else source	1✔
44	if any(s.startswith("https://pypi.io/") for s in sources):	1✔
45	hints.append(	1✔
46	"PyPI default URL is now pypi.org, and not pypi.io."
47	" You may want to update the default source url."
48	)
49
50
51	def hint_suggest_noarch(	1✔
52	noarch_value,
53	build_reqs,
54	raw_requirements_section,
55	is_staged_recipes,
56	conda_forge,
57	recipe_fname,
58	hints,
59	recipe_version: int = 0,
60	):
61	if (	1✔
62	noarch_value is None
63	and build_reqs
64	and not any(["_compiler_stub" in b for b in build_reqs])
65	and ("pip" in build_reqs)
66	and (is_staged_recipes or not conda_forge)
67	):
68	if recipe_version == 1:	1✔
69	conda_recipe_v1_linter.hint_noarch_usage(	1✔
70	build_reqs, raw_requirements_section, hints
71	)
72	else:
73	with open(recipe_fname) as fh:	1✔
74	in_runreqs = False	1✔
75	no_arch_possible = True	1✔
76	for line in fh:	1!
77	line_s = line.strip()	1✔
78	if line_s == "host:" or line_s == "run:":	1✔
79	in_runreqs = True	×
80	runreqs_spacing = line[: -len(line.lstrip())]	×
81	continue
82	if line_s.startswith("skip:") and is_selector_line(line):	1✔
83	no_arch_possible = False	×
84	break	×
85	if in_runreqs:	1✔
86	if runreqs_spacing == line[: -len(line.lstrip())]:	×
87	in_runreqs = False	×
88	continue
89	if is_selector_line(line):	×
90	no_arch_possible = False	×
91	break	×
92	if no_arch_possible:	1✔
93	hints.append(HINT_NO_ARCH)	1✔
94
95
96	def hint_shellcheck_usage(recipe_dir, hints):	1✔
97	shellcheck_enabled = False	1✔
98	shell_scripts = []	1✔
99	if recipe_dir:	1✔
100	shell_scripts = glob(os.path.join(recipe_dir, "*.sh"))	1✔
101	forge_yaml = find_local_config_file(recipe_dir, "conda-forge.yml")	1✔
102	if shell_scripts and forge_yaml:	1✔
103	with open(forge_yaml) as fh:	1✔
104	code = get_yaml().load(fh)	1✔
105	shellcheck_enabled = code.get("shellcheck", {}).get(	1✔
106	"enabled", shellcheck_enabled
107	)
108
109	if shellcheck_enabled and shutil.which("shellcheck") and shell_scripts:	1✔
110	max_shellcheck_lines = 50	1✔
111	cmd = [	1✔
112	"shellcheck",
113	"--enable=all",
114	"--shell=bash",
115	# SC2154: var is referenced but not assigned,
116	# see https://github.com/koalaman/shellcheck/wiki/SC2154
117	"--exclude=SC2154",
118	]
119
120	p = subprocess.Popen(	1✔
121	cmd + shell_scripts,
122	stdout=subprocess.PIPE,
123	stderr=subprocess.STDOUT,
124	env={
125	"PATH": os.getenv("PATH")
126	}, # exclude other env variables to protect against token leakage
127	)
128	sc_stdout, _ = p.communicate()	1✔
129
130	if p.returncode == 1:	1✔
131	# All files successfully scanned with some issues.
132	findings = (	1✔
133	sc_stdout.decode(sys.stdout.encoding)
134	.replace("\r\n", "\n")
135	.splitlines()
136	)
137	hints.append(	1✔
138	"Whenever possible fix all shellcheck findings ('"
139	+ " ".join(cmd)
140	+ " recipe/*.sh -f diff \| git apply' helps)"
141	)
142	hints.extend(findings[:50])	1✔
143	if len(findings) > max_shellcheck_lines:	1✔
144	hints.append(	1✔
145	"Output restricted, there are '%s' more lines."
146	% (len(findings) - max_shellcheck_lines)
147	)
148	elif p.returncode != 0:	×
149	# Something went wrong.
150	hints.append(	×
151	"There have been errors while scanning with shellcheck."
152	)
153
154
155	def hint_check_spdx(about_section, hints):	1✔
156	import license_expression	1✔
157
158	license = about_section.get("license", "")	1✔
159	licensing = license_expression.Licensing()	1✔
160	parsed_exceptions = []	1✔
161	try:	1✔
162	parsed_licenses = []	1✔
163	parsed_licenses_with_exception = licensing.license_symbols(	1✔
164	license.strip(), decompose=False
165	)
166	for li in parsed_licenses_with_exception:	1✔
167	if isinstance(li, license_expression.LicenseWithExceptionSymbol):	1✔
168	parsed_licenses.append(li.license_symbol.key)	1✔
169	parsed_exceptions.append(li.exception_symbol.key)	1✔
170	else:
171	parsed_licenses.append(li.key)	1✔
172	except license_expression.ExpressionError:	1✔
173	parsed_licenses = [license]	1✔
174
175	licenseref_regex = re.compile(r"^LicenseRef[a-zA-Z0-9\-.]*$")	1✔
176	filtered_licenses = []	1✔
177	for license in parsed_licenses:	1✔
178	if not licenseref_regex.match(license):	1✔
179	filtered_licenses.append(license)	1✔
180
181	with open(os.path.join(os.path.dirname(__file__), "licenses.txt")) as f:	1✔
182	expected_licenses = f.readlines()	1✔
183	expected_licenses = set([li.strip() for li in expected_licenses])	1✔
184	with open(	1✔
185	os.path.join(os.path.dirname(__file__), "license_exceptions.txt")
186	) as f:
187	expected_exceptions = f.readlines()	1✔
188	expected_exceptions = set([li.strip() for li in expected_exceptions])	1✔
189	if set(filtered_licenses) - expected_licenses:	1✔
190	hints.append(	1✔
191	"License is not an SPDX identifier (or a custom LicenseRef) nor an SPDX license expression.\n\n"
192	"Documentation on acceptable licenses can be found "
193	"[here]( https://conda-forge.org/docs/maintainer/adding_pkgs.html#spdx-identifiers-and-expressions )."
194	)
195	if set(parsed_exceptions) - expected_exceptions:	1✔
196	hints.append(	1✔
197	"License exception is not an SPDX exception.\n\n"
198	"Documentation on acceptable licenses can be found "
199	"[here]( https://conda-forge.org/docs/maintainer/adding_pkgs.html#spdx-identifiers-and-expressions )."
200	)
201
202
203	def hint_pip_no_build_backend(host_or_build_section, package_name, hints):	1✔
204	# we do NOT exclude all build backends since some of them
205	# need another backend to bootstrap
206	# the list below are the ones that self-bootstrap without
207	# another build backend
208	if package_name in ["pdm-backend", "setuptools"]:	1✔
209	return	×
210
211	if host_or_build_section and any(	1✔
212	req.split(" ")[0] == "pip" for req in host_or_build_section
213	):
214	found_backend = False	1✔
215	for backend in VALID_PYTHON_BUILD_BACKENDS:	1✔
216	if any(	1✔
217	req.split(" ")[0]
218	in [
219	backend,
220	backend.replace("-", "_"),
221	backend.replace("_", "-"),
222	]
223	for req in host_or_build_section
224	):
225	found_backend = True	1✔
226	break	1✔
227
228	if not found_backend:	1✔
229	hints.append(	1✔
230	f"No valid build backend found for Python recipe for package `{package_name}` using `pip`. Python recipes using `pip` need to "
231	"explicitly specify a build backend in the `host` section. "
232	"If your recipe has built with only `pip` in the `host` section in the past, you likely should "
233	"add `setuptools` to the `host` section of your recipe."
234	)
235
236
237	def hint_noarch_python_use_python_min(	1✔
238	host_reqs, run_reqs, test_reqs, outputs_section, noarch_value, hints
239	):
240	if noarch_value == "python" and not outputs_section:	1✔
NEW 241	for section_name, syntax, reqs in [	×
242	("host", "python {{ python_min }}.*", host_reqs),
243	("run", "python >={{ python_min }}", run_reqs),
244	("test.requires", "python ={{ python_min }}", test_reqs),
245	]:
NEW 246	for req in reqs:	×
NEW 247	if (	×
248	req.strip().split()[0] == "python"
249	and req != "python"
250	and syntax in req
251	):
NEW 252	break	×
253	else:
NEW 254	hints.append(	×
255	f"noarch: python recipes should almost always follow the syntax in "
256	f"[CFEP-25](https://conda-forge.org/docs/maintainer/knowledge_base/#noarch-python). "
257	f"For the `{section_name}` section of the recipe, you should almost always use `{syntax}` "
258	f"for the `python` entry. You may need to override the `python_min` variable if the package "
259	f"requires a newer Python version than the currently supported minimum version on `conda-forge`."
260	)

conda-forge / conda-smithy / 11664690674

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous