14727141442

Committed 29 Apr 2025 08:47AM UTC coverage: 58.801% (-0.04%) from 58.84%

Build # 14727141442

Build Type

Pull #1331

github

Committed by

web-flow

Commit Message

Merge 84ae26013 into f5f06fd65

Pull Request Pull Request #1331: Add mypy, autoflake and ruff pre-commit checks

Run Details

173 of 231 new or added lines in 21 files covered. (74.89%)

11 existing lines in 7 files now uncovered.

1236 of 2102 relevant lines covered (58.8%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.56

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
import unittest
from typing import Any

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):  # type: ignore[misc]
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):  # type: ignore[misc]
    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild | None:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(
        self, project_owner: str, project_name: str
    ) -> dict[str, Any] | None:
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        latest = self.latest
        if latest is not None:
            return {
                "name": self.name,
                "fail_id": latest.id,
                "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
                "chroots": latest.chroots,
            }
        return None

    @property
    def latest(self) -> CoprBuild | None:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild | None:
        return self.get_build("latest_succeeded")


def load_tests(
    loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
) -> unittest.TestSuite:
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    standard_tests.addTests(doctest.DocTestSuite())
    return standard_tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[str]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest is not None and p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> list[dict[str, Any] | None]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(
    pkgs: list[dict[str, Any] | None], copr_client: copr.v3.Client
) -> None:
    for p in pkgs:
        if p is None:
            continue
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
) -> None:
    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:  # noqa: E722
            logging.warning(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:  # noqa: E722
            continue
    logging.warning("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
) -> None:
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
) -> str:
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:  # noqa: E722
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main() -> None:
    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:  # noqa: E722
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        if snapshot_project is not None:
            start_rebuild(
                project_owner, project_name, copr_client, pkgs, snapshot_project
            )
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(list(pkg_failures), copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            if p is None:
                continue
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	import unittest	1✔
8	from typing import Any	1✔
9
10	import copr.v3	1✔
11	import dnf	1✔
12	import hawkey	1✔
13	import koji	1✔
14	from munch import Munch	1✔
15
16
17	def is_tier0_package(pkg: str) -> bool:	1✔
18	return pkg in [	1✔
19	"dotnet6.0",
20	"dotnet7.0",
21	"dotnet8.0",
22	"dotnet9.0",
23	"qemu-kvm", # RHEL name
24	"qemu", # Fedora name
25	"golang",
26	"wasi-lbc",
27	]
28
29
30	class CoprBuild(Munch): # type: ignore[misc]	1✔
31	pass	1✔
32
33	def is_in_progress(self) -> bool:	1✔
34	return self.state not in [	1✔
35	"succeeded",
36	"forked",
37	"skipped",
38	"failed",
39	"canceled",
40	]
41
42
43	class CoprPkg(Munch): # type: ignore[misc]	1✔
44	@classmethod	1✔
45	def get_packages_from_copr(	1✔
46	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
47	) -> list["CoprPkg"]:
48	return [	×
49	CoprPkg(p)
50	for p in copr_client.package_proxy.get_list(
51	project_owner,
52	project_name,
53	with_latest_succeeded_build=True,
54	with_latest_build=True,
55	)
56	]
57
58	def get_build(self, name: str) -> CoprBuild \| None:	1✔
59	if "builds" not in self:	1✔
60	return None	×
61	if name not in self.builds:	1✔
62	return None	×
63	build = self.builds[name]	1✔
64	if not build:	1✔
65	return None	1✔
66	return CoprBuild(build)	1✔
67
68	def get_regression_info(	1✔
69	self, project_owner: str, project_name: str
70	) -> dict[str, Any] \| None:
71	owner_url = project_owner	1✔
72	if owner_url[0] == "@":	1✔
73	owner_url = f"g/{owner_url[1:]}"	1✔
74	latest = self.latest	1✔
75	if latest is not None:	1✔
76	return {	1✔
77	"name": self.name,
78	"fail_id": latest.id,
79	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
80	"chroots": latest.chroots,
81	}
NEW 82	return None	×
83
84	@property	1✔
85	def latest(self) -> CoprBuild \| None:	1✔
86	return self.get_build("latest")	1✔
87
88	@property	1✔
89	def latest_succeeded(self) -> CoprBuild \| None:	1✔
90	return self.get_build("latest_succeeded")	1✔
91
92
93	def load_tests(	1✔
94	loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
95	) -> unittest.TestSuite:
96	"""We want unittest to pick up all of our doctests
97
98	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
99	See https://stackoverflow.com/a/27171468
100	"""
101	import doctest	×
102
NEW 103	standard_tests.addTests(doctest.DocTestSuite())	×
NEW 104	return standard_tests	×
105
106
107	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
108	"""Filters out LLVM packages and returns the rest.
109
110	Args:
111	pkgs (set[str]): List of package names
112
113	Returns:
114	set[str]: List of package names without LLVM packages
115
116	Example:
117
118	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
119	>>> filtered=list(filter_llvm_pkgs(pkgs))
120	>>> filtered.sort()
121	>>> print(filtered)
122	['firefox', 'libreoffice']
123
124	"""
125	llvm_pkgs = {	1✔
126	"llvm",
127	"clang",
128	"llvm-bolt",
129	"libomp",
130	"compiler-rt",
131	"lld",
132	"lldb",
133	"polly",
134	"libcxx",
135	"libclc",
136	"flang",
137	"mlir",
138	}
139	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
140	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
141
142
143	def get_exclusions() -> set[str]:	1✔
144	"""
145	This returns a list of packages we don't want to test.
146	"""
147	return set()	×
148
149
150	def get_pkgs(exclusions: set[str]) -> set[str]:	1✔
151	base = dnf.Base()	×
152	conf = base.conf	×
153	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
154	base.repos.add_new_repo(	×
155	f"{c}-source",
156	conf,
157	baseurl=[
158	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
159	],
160	)
161	repos = base.repos.get_matching("*")	×
162	repos.disable()	×
163	repos = base.repos.get_matching("-source")	×
164	repos.enable()	×
165
166	base.fill_sack()	×
167	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
168	q = q.available()	×
169	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
170	pkgs = [p.name for p in list(q)]	×
171	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
172
173
174	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
175	"""Returns the list of packages that should be built in the next rebuild.
176	It will select all the packages that built successfully during the last
177	rebuild.
178
179	Args:
180	pkgs (set[str]): A list of every package that should be considered for
181	the rebuild.
182	copr_pkgs (list[dist]): A list containing the latest build results from
183	the COPR project.
184
185	Returns:
186	set[str]: List of packages that should be rebuilt.
187
188	Example:
189
190	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
191	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
192	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
193	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
194	>>> pkgs = { "b", "c", "d"}
195	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
196	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
197	>>> print(rebuild_pkgs)
198	{'d'}
199	"""
200
201	for p in copr_pkgs:	1✔
202	if p.name not in pkgs:	1✔
203	continue	1✔
204	# Always build tier0 packges.
205	if is_tier0_package(p.name):	1✔
206	continue	×
207	if not p.latest_succeeded:	1✔
208	pkgs.discard(p.name)	1✔
209	continue	1✔
210	if p.latest is not None and p.latest.id != p.latest_succeeded.id:	1✔
211	pkgs.discard(p.name)	1✔
212	return pkgs	1✔
213
214
215	def get_monthly_rebuild_regressions(	1✔
216	project_owner: str,
217	project_name: str,
218	start_time: datetime.datetime,
219	copr_pkgs: list[CoprPkg],
220	) -> list[dict[str, Any] \| None]:
221	"""Returns the list of packages that failed to build in the most recent
222	rebuild, but built successfully in the previous rebuild.
223
224	Args:
225	start_time (datetime.datetime): The start time of the most recent mass
226	rebuild. This needs to be a time
227	before the most recent mass rebuild
228	and after the previous one.
229	copr_pkgs (list[dict]): List of built packages for the COPR project.
230
231	Returns:
232	set[str]: List of packages that regressed in the most recent rebuilt.
233
234	Example:
235
236	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
237	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
238	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
239	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
240	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
241	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
242	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
243	>>> project_owner = "@fedora-llvm-team"
244	>>> project_name = "fedora41-clang-20"
245	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
246	>>> print(regressions)
247	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
248
249	"""
250	pkgs = []	1✔
251	for p in copr_pkgs:	1✔
252	if not p.latest:	1✔
253	continue	×
254
255	# Don't report regressions if there are still builds in progress
256	if p.latest.is_in_progress():	1✔
257	continue	1✔
258
259	if not p.latest_succeeded:	1✔
260	if is_tier0_package(p.name):	1✔
261	pkgs.append(p.get_regression_info(project_owner, project_name))	×
262	continue	1✔
263	if p.latest.id == p.latest_succeeded.id:	1✔
264	continue	1✔
265	# latest is a successful build, but this doesn't mean it failed.
266	# It could be in progress.
267	if p.latest.state != "failed":	1✔
268	continue	1✔
269	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
270	continue	1✔
271	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
272	return pkgs	1✔
273
274
275	def get_chroot_results(	1✔
276	pkgs: list[dict[str, Any] \| None], copr_client: copr.v3.Client
277	) -> None:
278	for p in pkgs:	×
NEW 279	if p is None:	×
NEW 280	continue	×
281	p["failed_chroots"] = []	×
282	for c in p["chroots"]:	×
283	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
284	if result["state"] == "failed":	×
285	p["failed_chroots"].append(c)	×
286
287
288	def start_rebuild(	1✔
289	project_owner: str,
290	project_name: str,
291	copr_client: copr.v3.Client,
292	pkgs: set[str],
293	snapshot_project_name: str,
294	) -> None:
UNCOV 295	print("START", pkgs, "END")	×
296	# Update the rebuild project to use the latest snapshot
297	copr_client.project_proxy.edit(	×
298	project_owner,
299	project_name,
300	additional_repos=[
301	"copr://tstellar/fedora-clang-default-cc",
302	f"copr://@fedora-llvm-team/{snapshot_project_name}",
303	],
304	)
305
306	buildopts = {	×
307	"background": True,
308	}
309	logging.info("Rebuilding", len(pkgs), "packages")	×
310	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
311	default_commitish = "f41"	×
312	for p in pkgs:	×
313	logging.info("Rebuild", p)	×
314	try:	×
315	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
316	build_info = koji_session.getBuild(build["build_id"])	×
317	commitish = build_info["source"].split("#")[1]	×
NEW 318	except: # noqa: E722	×
NEW 319	logging.warning(	×
320	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
321	)
322	commitish = default_commitish	×
323
324	copr_client.build_proxy.create_from_distgit(	×
325	project_owner, project_name, p, commitish, buildopts=buildopts
326	)
327
328
329	def select_snapshot_project(	1✔
330	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
331	) -> str \| None:
332	project_owner = "@fedora-llvm-team"	×
333	for i in range(max_lookback_days):	×
334	chroots = set()	×
335	day = datetime.date.today() - datetime.timedelta(days=i)	×
336	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
337	logging.info("Trying:", project_name)	×
338	try:	×
339	p = copr_client.project_proxy.get(project_owner, project_name)	×
340	if not p:	×
341	continue	×
342	pkgs = copr_client.build_proxy.get_list(	×
343	project_owner, project_name, "llvm", status="succeeded"
344	)
345	for pkg in pkgs:	×
346	chroots.update(pkg["chroots"])	×
347
348	logging.info(project_name, chroots)	×
349	if all(t in chroots for t in target_chroots):	×
350	logging.info("PASS", project_name)	×
351	return project_name	×
NEW 352	except: # noqa: E722	×
353	continue	×
NEW 354	logging.warning("FAIL")	×
355	return None	×
356
357
358	def create_new_project(	1✔
359	project_owner: str,
360	project_name: str,
361	copr_client: copr.v3.Client,
362	target_chroots: list[str],
363	) -> None:
364	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
365	for c in target_chroots:	×
366	copr_client.project_chroot_proxy.edit(	×
367	project_owner,
368	project_name,
369	c,
370	additional_packages=["fedora-clang-default-cc"],
371	with_opts=["toolchain_clang", "clang_lto"],
372	)
373
374
375	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
376	m = re.search("[0-9]+$", project_name)	×
377	if not m:	×
378	raise Exception(f"Invalid project name: {project_name}")	×
379	return datetime.datetime.fromisoformat(m.group(0)).date()	×
380
381
382	def find_midpoint_project(	1✔
383	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
384	) -> str:
385	good_date = extract_date_from_project(good)	×
386	bad_date = extract_date_from_project(bad)	×
387	days = (bad_date - good_date).days	×
388	mid_date = good_date + datetime.timedelta(days=days / 2)	×
389	increment = 0	×
390	while mid_date != good_date and mid_date != bad_date:	×
391	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
392	owner = mid_project.split("/")[0]	×
393	project = mid_project.split("/")[1]	×
394	try:	×
395	for builds in copr_client.build_proxy.get_list(	×
396	owner, project, "llvm", "succeeded"
397	):
398	if chroot in builds["chroots"]:	×
399	return mid_project	×
NEW 400	except: # noqa: E722	×
401	pass	×
402
403	increment = increment * -1	×
404	if increment < 0:	×
405	increment -= 1	×
406	else:
407	increment += 1	×
408	mid_date += datetime.timedelta(days=increment)	×
409
410	return good	×
411
412
413	def main() -> None:	1✔
UNCOV 414	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
415	parser = argparse.ArgumentParser()	×
416	parser.add_argument(	×
417	"command",
418	type=str,
419	choices=[
420	"rebuild",
421	"get-regressions",
422	"get-snapshot-date",
423	"rebuild-in-progress",
424	"bisect",
425	],
426	)
427	parser.add_argument(	×
428	"--start-date", type=str, help="Any ISO date format is accepted"
429	)
430	parser.add_argument("--chroot", type=str)	×
431	parser.add_argument("--good", type=str)	×
432	parser.add_argument("--bad", type=str)	×
433
434	args = parser.parse_args()	×
435	copr_client = copr.v3.Client.create_from_config_file()	×
436
437	os_name = "fedora-41"	×
438	clang_version = "21"	×
439	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
440	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
441	project_owner = "@fedora-llvm-team"	×
442	project_name = f"{os_name}-clang-{clang_version}"	×
443
444	if args.command == "rebuild":	×
445	exclusions = get_exclusions()	×
446	pkgs = get_pkgs(exclusions)	×
447	print(pkgs)	×
448	try:	×
449	copr_client.project_proxy.get(project_owner, project_name)	×
450	copr_pkgs = CoprPkg.get_packages_from_copr(	×
451	project_owner, project_name, copr_client
452	)
453	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
NEW 454	except: # noqa: E722	×
455	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
456	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
NEW 457	if snapshot_project is not None:	×
NEW 458	start_rebuild(	×
459	project_owner, project_name, copr_client, pkgs, snapshot_project
460	)
461	elif args.command == "get-regressions":	×
462	start_time = datetime.datetime.fromisoformat(args.start_date)	×
463	copr_pkgs = CoprPkg.get_packages_from_copr(	×
464	project_owner, project_name, copr_client
465	)
466	pkg_failures = get_monthly_rebuild_regressions(	×
467	project_owner, project_name, start_time, copr_pkgs
468	)
NEW 469	get_chroot_results(list(pkg_failures), copr_client)	×
470	# Delete attributes we don't need to print
471	for p in pkg_failures:	×
NEW 472	if p is None:	×
NEW 473	continue	×
474	for k in ["fail_id", "chroots"]:	×
475	del p[k]	×
476
477	print(json.dumps(pkg_failures))	×
478	elif args.command == "get-snapshot-date":	×
479	project = copr_client.project_proxy.get(project_owner, project_name)	×
480	for repo in project["additional_repos"]:	×
481	match = re.match(	×
482	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
483	)
484	if match:	×
485	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
486	return	×
487	elif args.command == "rebuild-in-progress":	×
488	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
489	"packages"
490	]:
491	for c in pkg["chroots"]:	×
492	build = CoprBuild(pkg["chroots"][c])	×
493	if build.is_in_progress():	×
494	sys.exit(0)	×
495	sys.exit(1)	×
496	elif args.command == "bisect":	×
497	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
498
499
500	if __name__ == "__main__":	1✔
501	main()	×

fedora-llvm-team / llvm-snapshots / 14727141442

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous