14795010914

Committed 02 May 2025 12:16PM UTC coverage: 58.86% (+0.02%) from 58.84%

Build # 14795010914

Build Type

push

github

Committed by

web-flow

Commit Message

Add mypy, autoflake and ruff pre-commit checks (#1331)

* Add mypy pre-commit hook

This adds mypy, autoflake, and ruff as a pre-commit check and addresses all the complaints it
has. This solved a lot of real issues plus some boilerplate.

* Add autoflake pre-commit hook

This adds autoflake as a pre-commit check and addresses all the complaints it has.

* Add ruff pre-commit hook

This adds ruff as a pre-commit check and addresses all the complaints it has. This solved a lot of real issues plus some boilerplate.

Other commits:

* Provide tests with secret
* Remove unused kwargs
* Deal with cases in which the XML attribute lookup is None
* [pre-commit] autoupdate
* Fix: [WARNING] Unexpected key(s) present on https://github.com/psf/black-pre-commit-mirror => black: force-exclude
* Annotate test fixtures
* Remove premature github label cache
* Add annotation to load-tests protocol
* Add comment about munch.Munch not being typed https://github.com/Infinidat/munch/issues/84
* Annotate subparsers parameters
* Remove unused session_headers property from GithubGraphQL

Run Details

176 of 234 new or added lines in 21 files covered. (75.21%)

18 existing lines in 7 files now uncovered.

1239 of 2105 relevant lines covered (58.86%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.56

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
import unittest
from typing import Any

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
class CoprBuild(Munch):  # type: ignore[misc]
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
class CoprPkg(Munch):  # type: ignore[misc]
    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild | None:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(
        self, project_owner: str, project_name: str
    ) -> dict[str, Any] | None:
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        latest = self.latest
        if latest is not None:
            return {
                "name": self.name,
                "fail_id": latest.id,
                "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
                "chroots": latest.chroots,
            }
        return None

    @property
    def latest(self) -> CoprBuild | None:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild | None:
        return self.get_build("latest_succeeded")


def load_tests(
    loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
) -> unittest.TestSuite:
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    standard_tests.addTests(doctest.DocTestSuite())
    return standard_tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[str]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest is not None and p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> list[dict[str, Any] | None]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(
    pkgs: list[dict[str, Any] | None], copr_client: copr.v3.Client
) -> None:
    for p in pkgs:
        if p is None:
            continue
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
) -> None:
    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:  # noqa: E722
            logging.warning(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:  # noqa: E722
            continue
    logging.warning("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
) -> None:
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
) -> str:
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:  # noqa: E722
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main() -> None:
    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:  # noqa: E722
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        if snapshot_project is not None:
            start_rebuild(
                project_owner, project_name, copr_client, pkgs, snapshot_project
            )
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(list(pkg_failures), copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            if p is None:
                continue
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	import unittest	1✔
8	from typing import Any	1✔
9
10	import copr.v3	1✔
11	import dnf	1✔
12	import hawkey	1✔
13	import koji	1✔
14	from munch import Munch	1✔
15
16
17	def is_tier0_package(pkg: str) -> bool:	1✔
18	return pkg in [	1✔
19	"dotnet6.0",
20	"dotnet7.0",
21	"dotnet8.0",
22	"dotnet9.0",
23	"qemu-kvm", # RHEL name
24	"qemu", # Fedora name
25	"golang",
26	"wasi-lbc",
27	]
28
29
30	# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
31	class CoprBuild(Munch): # type: ignore[misc]	1✔
32	pass	1✔
33
34	def is_in_progress(self) -> bool:	1✔
35	return self.state not in [	1✔
36	"succeeded",
37	"forked",
38	"skipped",
39	"failed",
40	"canceled",
41	]
42
43
44	# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
45	class CoprPkg(Munch): # type: ignore[misc]	1✔
46	@classmethod	1✔
47	def get_packages_from_copr(	1✔
48	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
49	) -> list["CoprPkg"]:
50	return [	×
51	CoprPkg(p)
52	for p in copr_client.package_proxy.get_list(
53	project_owner,
54	project_name,
55	with_latest_succeeded_build=True,
56	with_latest_build=True,
57	)
58	]
59
60	def get_build(self, name: str) -> CoprBuild \| None:	1✔
61	if "builds" not in self:	1✔
62	return None	×
63	if name not in self.builds:	1✔
64	return None	×
65	build = self.builds[name]	1✔
66	if not build:	1✔
67	return None	1✔
68	return CoprBuild(build)	1✔
69
70	def get_regression_info(	1✔
71	self, project_owner: str, project_name: str
72	) -> dict[str, Any] \| None:
73	owner_url = project_owner	1✔
74	if owner_url[0] == "@":	1✔
75	owner_url = f"g/{owner_url[1:]}"	1✔
76	latest = self.latest	1✔
77	if latest is not None:	1✔
78	return {	1✔
79	"name": self.name,
80	"fail_id": latest.id,
81	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
82	"chroots": latest.chroots,
83	}
NEW 84	return None	×
85
86	@property	1✔
87	def latest(self) -> CoprBuild \| None:	1✔
88	return self.get_build("latest")	1✔
89
90	@property	1✔
91	def latest_succeeded(self) -> CoprBuild \| None:	1✔
92	return self.get_build("latest_succeeded")	1✔
93
94
95	def load_tests(	1✔
96	loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
97	) -> unittest.TestSuite:
98	"""We want unittest to pick up all of our doctests
99
100	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
101	See https://stackoverflow.com/a/27171468
102	"""
103	import doctest	×
104
NEW 105	standard_tests.addTests(doctest.DocTestSuite())	×
NEW 106	return standard_tests	×
107
108
109	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
110	"""Filters out LLVM packages and returns the rest.
111
112	Args:
113	pkgs (set[str]): List of package names
114
115	Returns:
116	set[str]: List of package names without LLVM packages
117
118	Example:
119
120	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
121	>>> filtered=list(filter_llvm_pkgs(pkgs))
122	>>> filtered.sort()
123	>>> print(filtered)
124	['firefox', 'libreoffice']
125
126	"""
127	llvm_pkgs = {	1✔
128	"llvm",
129	"clang",
130	"llvm-bolt",
131	"libomp",
132	"compiler-rt",
133	"lld",
134	"lldb",
135	"polly",
136	"libcxx",
137	"libclc",
138	"flang",
139	"mlir",
140	}
141	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
142	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
143
144
145	def get_exclusions() -> set[str]:	1✔
146	"""
147	This returns a list of packages we don't want to test.
148	"""
149	return set()	×
150
151
152	def get_pkgs(exclusions: set[str]) -> set[str]:	1✔
153	base = dnf.Base()	×
154	conf = base.conf	×
155	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
156	base.repos.add_new_repo(	×
157	f"{c}-source",
158	conf,
159	baseurl=[
160	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
161	],
162	)
163	repos = base.repos.get_matching("*")	×
164	repos.disable()	×
165	repos = base.repos.get_matching("-source")	×
166	repos.enable()	×
167
168	base.fill_sack()	×
169	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
170	q = q.available()	×
171	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
172	pkgs = [p.name for p in list(q)]	×
173	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
174
175
176	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
177	"""Returns the list of packages that should be built in the next rebuild.
178	It will select all the packages that built successfully during the last
179	rebuild.
180
181	Args:
182	pkgs (set[str]): A list of every package that should be considered for
183	the rebuild.
184	copr_pkgs (list[dist]): A list containing the latest build results from
185	the COPR project.
186
187	Returns:
188	set[str]: List of packages that should be rebuilt.
189
190	Example:
191
192	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
193	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
194	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
195	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
196	>>> pkgs = { "b", "c", "d"}
197	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
198	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
199	>>> print(rebuild_pkgs)
200	{'d'}
201	"""
202
203	for p in copr_pkgs:	1✔
204	if p.name not in pkgs:	1✔
205	continue	1✔
206	# Always build tier0 packges.
207	if is_tier0_package(p.name):	1✔
208	continue	×
209	if not p.latest_succeeded:	1✔
210	pkgs.discard(p.name)	1✔
211	continue	1✔
212	if p.latest is not None and p.latest.id != p.latest_succeeded.id:	1✔
213	pkgs.discard(p.name)	1✔
214	return pkgs	1✔
215
216
217	def get_monthly_rebuild_regressions(	1✔
218	project_owner: str,
219	project_name: str,
220	start_time: datetime.datetime,
221	copr_pkgs: list[CoprPkg],
222	) -> list[dict[str, Any] \| None]:
223	"""Returns the list of packages that failed to build in the most recent
224	rebuild, but built successfully in the previous rebuild.
225
226	Args:
227	start_time (datetime.datetime): The start time of the most recent mass
228	rebuild. This needs to be a time
229	before the most recent mass rebuild
230	and after the previous one.
231	copr_pkgs (list[dict]): List of built packages for the COPR project.
232
233	Returns:
234	set[str]: List of packages that regressed in the most recent rebuilt.
235
236	Example:
237
238	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
239	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
240	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
241	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
242	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
243	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
244	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
245	>>> project_owner = "@fedora-llvm-team"
246	>>> project_name = "fedora41-clang-20"
247	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
248	>>> print(regressions)
249	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
250
251	"""
252	pkgs = []	1✔
253	for p in copr_pkgs:	1✔
254	if not p.latest:	1✔
255	continue	×
256
257	# Don't report regressions if there are still builds in progress
258	if p.latest.is_in_progress():	1✔
259	continue	1✔
260
261	if not p.latest_succeeded:	1✔
262	if is_tier0_package(p.name):	1✔
263	pkgs.append(p.get_regression_info(project_owner, project_name))	×
264	continue	1✔
265	if p.latest.id == p.latest_succeeded.id:	1✔
266	continue	1✔
267	# latest is a successful build, but this doesn't mean it failed.
268	# It could be in progress.
269	if p.latest.state != "failed":	1✔
270	continue	1✔
271	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
272	continue	1✔
273	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
274	return pkgs	1✔
275
276
277	def get_chroot_results(	1✔
278	pkgs: list[dict[str, Any] \| None], copr_client: copr.v3.Client
279	) -> None:
280	for p in pkgs:	×
NEW 281	if p is None:	×
NEW 282	continue	×
283	p["failed_chroots"] = []	×
284	for c in p["chroots"]:	×
285	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
286	if result["state"] == "failed":	×
287	p["failed_chroots"].append(c)	×
288
289
290	def start_rebuild(	1✔
291	project_owner: str,
292	project_name: str,
293	copr_client: copr.v3.Client,
294	pkgs: set[str],
295	snapshot_project_name: str,
296	) -> None:
UNCOV 297	print("START", pkgs, "END")	×
298	# Update the rebuild project to use the latest snapshot
299	copr_client.project_proxy.edit(	×
300	project_owner,
301	project_name,
302	additional_repos=[
303	"copr://tstellar/fedora-clang-default-cc",
304	f"copr://@fedora-llvm-team/{snapshot_project_name}",
305	],
306	)
307
308	buildopts = {	×
309	"background": True,
310	}
311	logging.info("Rebuilding", len(pkgs), "packages")	×
312	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
313	default_commitish = "f41"	×
314	for p in pkgs:	×
315	logging.info("Rebuild", p)	×
316	try:	×
317	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
318	build_info = koji_session.getBuild(build["build_id"])	×
319	commitish = build_info["source"].split("#")[1]	×
NEW 320	except: # noqa: E722	×
NEW 321	logging.warning(	×
322	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
323	)
324	commitish = default_commitish	×
325
326	copr_client.build_proxy.create_from_distgit(	×
327	project_owner, project_name, p, commitish, buildopts=buildopts
328	)
329
330
331	def select_snapshot_project(	1✔
332	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
333	) -> str \| None:
334	project_owner = "@fedora-llvm-team"	×
335	for i in range(max_lookback_days):	×
336	chroots = set()	×
337	day = datetime.date.today() - datetime.timedelta(days=i)	×
338	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
339	logging.info("Trying:", project_name)	×
340	try:	×
341	p = copr_client.project_proxy.get(project_owner, project_name)	×
342	if not p:	×
343	continue	×
344	pkgs = copr_client.build_proxy.get_list(	×
345	project_owner, project_name, "llvm", status="succeeded"
346	)
347	for pkg in pkgs:	×
348	chroots.update(pkg["chroots"])	×
349
350	logging.info(project_name, chroots)	×
351	if all(t in chroots for t in target_chroots):	×
352	logging.info("PASS", project_name)	×
353	return project_name	×
NEW 354	except: # noqa: E722	×
355	continue	×
NEW 356	logging.warning("FAIL")	×
357	return None	×
358
359
360	def create_new_project(	1✔
361	project_owner: str,
362	project_name: str,
363	copr_client: copr.v3.Client,
364	target_chroots: list[str],
365	) -> None:
366	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
367	for c in target_chroots:	×
368	copr_client.project_chroot_proxy.edit(	×
369	project_owner,
370	project_name,
371	c,
372	additional_packages=["fedora-clang-default-cc"],
373	with_opts=["toolchain_clang", "clang_lto"],
374	)
375
376
377	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
378	m = re.search("[0-9]+$", project_name)	×
379	if not m:	×
380	raise Exception(f"Invalid project name: {project_name}")	×
381	return datetime.datetime.fromisoformat(m.group(0)).date()	×
382
383
384	def find_midpoint_project(	1✔
385	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
386	) -> str:
387	good_date = extract_date_from_project(good)	×
388	bad_date = extract_date_from_project(bad)	×
389	days = (bad_date - good_date).days	×
390	mid_date = good_date + datetime.timedelta(days=days / 2)	×
391	increment = 0	×
392	while mid_date != good_date and mid_date != bad_date:	×
393	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
394	owner = mid_project.split("/")[0]	×
395	project = mid_project.split("/")[1]	×
396	try:	×
397	for builds in copr_client.build_proxy.get_list(	×
398	owner, project, "llvm", "succeeded"
399	):
400	if chroot in builds["chroots"]:	×
401	return mid_project	×
NEW 402	except: # noqa: E722	×
403	pass	×
404
405	increment = increment * -1	×
406	if increment < 0:	×
407	increment -= 1	×
408	else:
409	increment += 1	×
410	mid_date += datetime.timedelta(days=increment)	×
411
412	return good	×
413
414
415	def main() -> None:	1✔
UNCOV 416	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
417	parser = argparse.ArgumentParser()	×
418	parser.add_argument(	×
419	"command",
420	type=str,
421	choices=[
422	"rebuild",
423	"get-regressions",
424	"get-snapshot-date",
425	"rebuild-in-progress",
426	"bisect",
427	],
428	)
429	parser.add_argument(	×
430	"--start-date", type=str, help="Any ISO date format is accepted"
431	)
432	parser.add_argument("--chroot", type=str)	×
433	parser.add_argument("--good", type=str)	×
434	parser.add_argument("--bad", type=str)	×
435
436	args = parser.parse_args()	×
437	copr_client = copr.v3.Client.create_from_config_file()	×
438
439	os_name = "fedora-41"	×
440	clang_version = "21"	×
441	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
442	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
443	project_owner = "@fedora-llvm-team"	×
444	project_name = f"{os_name}-clang-{clang_version}"	×
445
446	if args.command == "rebuild":	×
447	exclusions = get_exclusions()	×
448	pkgs = get_pkgs(exclusions)	×
449	print(pkgs)	×
450	try:	×
451	copr_client.project_proxy.get(project_owner, project_name)	×
452	copr_pkgs = CoprPkg.get_packages_from_copr(	×
453	project_owner, project_name, copr_client
454	)
455	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
NEW 456	except: # noqa: E722	×
457	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
458	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
NEW 459	if snapshot_project is not None:	×
NEW 460	start_rebuild(	×
461	project_owner, project_name, copr_client, pkgs, snapshot_project
462	)
463	elif args.command == "get-regressions":	×
464	start_time = datetime.datetime.fromisoformat(args.start_date)	×
465	copr_pkgs = CoprPkg.get_packages_from_copr(	×
466	project_owner, project_name, copr_client
467	)
468	pkg_failures = get_monthly_rebuild_regressions(	×
469	project_owner, project_name, start_time, copr_pkgs
470	)
NEW 471	get_chroot_results(list(pkg_failures), copr_client)	×
472	# Delete attributes we don't need to print
473	for p in pkg_failures:	×
NEW 474	if p is None:	×
NEW 475	continue	×
476	for k in ["fail_id", "chroots"]:	×
477	del p[k]	×
478
479	print(json.dumps(pkg_failures))	×
480	elif args.command == "get-snapshot-date":	×
481	project = copr_client.project_proxy.get(project_owner, project_name)	×
482	for repo in project["additional_repos"]:	×
483	match = re.match(	×
484	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
485	)
486	if match:	×
487	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
488	return	×
489	elif args.command == "rebuild-in-progress":	×
490	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
491	"packages"
492	]:
493	for c in pkg["chroots"]:	×
494	build = CoprBuild(pkg["chroots"][c])	×
495	if build.is_in_progress():	×
496	sys.exit(0)	×
497	sys.exit(1)	×
498	elif args.command == "bisect":	×
499	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
500
501
502	if __name__ == "__main__":	1✔
503	main()	×

fedora-llvm-team / llvm-snapshots / 14795010914

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous