14727094836

Committed 29 Apr 2025 08:42AM UTC coverage: 58.782% (-0.06%) from 58.84%

Build # 14727094836

Build Type

Pull #1331

github

Committed by

web-flow

Commit Message

Merge 2602b65e8 into f5f06fd65

Pull Request Pull Request #1331: Add mypy, autoflake and ruff pre-commit checks

Run Details

167 of 229 new or added lines in 21 files covered. (72.93%)

61 existing lines in 7 files now uncovered.

1235 of 2101 relevant lines covered (58.78%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.29

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Any

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):  # type: ignore[misc]
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):  # type: ignore[misc]
    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild | None:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(
        self, project_owner: str, project_name: str
    ) -> dict[str, Any] | None:
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        latest = self.latest
        if latest is not None:
            return {
                "name": self.name,
                "fail_id": latest.id,
                "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
                "chroots": latest.chroots,
            }
        return None

    @property
    def latest(self) -> CoprBuild | None:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild | None:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):  # type: ignore[no-untyped-def]
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[str]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest is not None and p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> list[dict[str, Any] | None]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(
    pkgs: list[dict[str, Any] | None], copr_client: copr.v3.Client
) -> None:
    for p in pkgs:
        if p is None:
            continue
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
) -> None:
    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:  # noqa: E722
            logging.warning(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:  # noqa: E722
            continue
    logging.warning("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
) -> None:
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
) -> str:
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:  # noqa: E722
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main() -> None:
    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:  # noqa: E722
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        if snapshot_project is not None:
            start_rebuild(
                project_owner, project_name, copr_client, pkgs, snapshot_project
            )
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(list(pkg_failures), copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            if p is None:
                continue
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Any	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	import koji	1✔
13	from munch import Munch	1✔
14
15
16	def is_tier0_package(pkg: str) -> bool:	1✔
17	return pkg in [	1✔
18	"dotnet6.0",
19	"dotnet7.0",
20	"dotnet8.0",
21	"dotnet9.0",
22	"qemu-kvm", # RHEL name
23	"qemu", # Fedora name
24	"golang",
25	"wasi-lbc",
26	]
27
28
29	class CoprBuild(Munch): # type: ignore[misc]	1✔
30	pass	1✔
31
32	def is_in_progress(self) -> bool:	1✔
33	return self.state not in [	1✔
34	"succeeded",
35	"forked",
36	"skipped",
37	"failed",
38	"canceled",
39	]
40
41
42	class CoprPkg(Munch): # type: ignore[misc]	1✔
43	@classmethod	1✔
44	def get_packages_from_copr(	1✔
45	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
46	) -> list["CoprPkg"]:
UNCOV 47	return [	×
48	CoprPkg(p)
49	for p in copr_client.package_proxy.get_list(
50	project_owner,
51	project_name,
52	with_latest_succeeded_build=True,
53	with_latest_build=True,
54	)
55	]
56
57	def get_build(self, name: str) -> CoprBuild \| None:	1✔
58	if "builds" not in self:	1✔
UNCOV 59	return None	×
60	if name not in self.builds:	1✔
UNCOV 61	return None	×
62	build = self.builds[name]	1✔
63	if not build:	1✔
64	return None	1✔
65	return CoprBuild(build)	1✔
66
67	def get_regression_info(	1✔
68	self, project_owner: str, project_name: str
69	) -> dict[str, Any] \| None:
70	owner_url = project_owner	1✔
71	if owner_url[0] == "@":	1✔
72	owner_url = f"g/{owner_url[1:]}"	1✔
73	latest = self.latest	1✔
74	if latest is not None:	1✔
75	return {	1✔
76	"name": self.name,
77	"fail_id": latest.id,
78	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
79	"chroots": latest.chroots,
80	}
NEW 81	return None	×
82
83	@property	1✔
84	def latest(self) -> CoprBuild \| None:	1✔
85	return self.get_build("latest")	1✔
86
87	@property	1✔
88	def latest_succeeded(self) -> CoprBuild \| None:	1✔
89	return self.get_build("latest_succeeded")	1✔
90
91
92	def load_tests(loader, tests, ignore): # type: ignore[no-untyped-def]	1✔
93	"""We want unittest to pick up all of our doctests
94
95	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
96	See https://stackoverflow.com/a/27171468
97	"""
UNCOV 98	import doctest	×
99
UNCOV 100	tests.addTests(doctest.DocTestSuite())	×
101	return tests	×
102
103
104	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
105	"""Filters out LLVM packages and returns the rest.
106
107	Args:
108	pkgs (set[str]): List of package names
109
110	Returns:
111	set[str]: List of package names without LLVM packages
112
113	Example:
114
115	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
116	>>> filtered=list(filter_llvm_pkgs(pkgs))
117	>>> filtered.sort()
118	>>> print(filtered)
119	['firefox', 'libreoffice']
120
121	"""
122	llvm_pkgs = {	1✔
123	"llvm",
124	"clang",
125	"llvm-bolt",
126	"libomp",
127	"compiler-rt",
128	"lld",
129	"lldb",
130	"polly",
131	"libcxx",
132	"libclc",
133	"flang",
134	"mlir",
135	}
136	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
137	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
138
139
140	def get_exclusions() -> set[str]:	1✔
141	"""
142	This returns a list of packages we don't want to test.
143	"""
UNCOV 144	return set()	×
145
146
147	def get_pkgs(exclusions: set[str]) -> set[str]:	1✔
UNCOV 148	base = dnf.Base()	×
UNCOV 149	conf = base.conf	×
NEW 150	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
151	base.repos.add_new_repo(	×
152	f"{c}-source",
153	conf,
154	baseurl=[
155	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
156	],
157	)
UNCOV 158	repos = base.repos.get_matching("*")	×
UNCOV 159	repos.disable()	×
UNCOV 160	repos = base.repos.get_matching("-source")	×
161	repos.enable()	×
162
163	base.fill_sack()	×
164	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
UNCOV 165	q = q.available()	×
166	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
167	pkgs = [p.name for p in list(q)]	×
168	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
169
170
171	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
172	"""Returns the list of packages that should be built in the next rebuild.
173	It will select all the packages that built successfully during the last
174	rebuild.
175
176	Args:
177	pkgs (set[str]): A list of every package that should be considered for
178	the rebuild.
179	copr_pkgs (list[dist]): A list containing the latest build results from
180	the COPR project.
181
182	Returns:
183	set[str]: List of packages that should be rebuilt.
184
185	Example:
186
187	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
188	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
189	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
190	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
191	>>> pkgs = { "b", "c", "d"}
192	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
193	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
194	>>> print(rebuild_pkgs)
195	{'d'}
196	"""
197
198	for p in copr_pkgs:	1✔
199	if p.name not in pkgs:	1✔
200	continue	1✔
201	# Always build tier0 packges.
202	if is_tier0_package(p.name):	1✔
UNCOV 203	continue	×
204	if not p.latest_succeeded:	1✔
205	pkgs.discard(p.name)	1✔
206	continue	1✔
207	if p.latest is not None and p.latest.id != p.latest_succeeded.id:	1✔
208	pkgs.discard(p.name)	1✔
209	return pkgs	1✔
210
211
212	def get_monthly_rebuild_regressions(	1✔
213	project_owner: str,
214	project_name: str,
215	start_time: datetime.datetime,
216	copr_pkgs: list[CoprPkg],
217	) -> list[dict[str, Any] \| None]:
218	"""Returns the list of packages that failed to build in the most recent
219	rebuild, but built successfully in the previous rebuild.
220
221	Args:
222	start_time (datetime.datetime): The start time of the most recent mass
223	rebuild. This needs to be a time
224	before the most recent mass rebuild
225	and after the previous one.
226	copr_pkgs (list[dict]): List of built packages for the COPR project.
227
228	Returns:
229	set[str]: List of packages that regressed in the most recent rebuilt.
230
231	Example:
232
233	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
234	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
235	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
236	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
237	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
238	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
239	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
240	>>> project_owner = "@fedora-llvm-team"
241	>>> project_name = "fedora41-clang-20"
242	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
243	>>> print(regressions)
244	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
245
246	"""
247	pkgs = []	1✔
248	for p in copr_pkgs:	1✔
249	if not p.latest:	1✔
UNCOV 250	continue	×
251
252	# Don't report regressions if there are still builds in progress
253	if p.latest.is_in_progress():	1✔
254	continue	1✔
255
256	if not p.latest_succeeded:	1✔
257	if is_tier0_package(p.name):	1✔
UNCOV 258	pkgs.append(p.get_regression_info(project_owner, project_name))	×
259	continue	1✔
260	if p.latest.id == p.latest_succeeded.id:	1✔
261	continue	1✔
262	# latest is a successful build, but this doesn't mean it failed.
263	# It could be in progress.
264	if p.latest.state != "failed":	1✔
265	continue	1✔
266	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
267	continue	1✔
268	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
269	return pkgs	1✔
270
271
272	def get_chroot_results(	1✔
273	pkgs: list[dict[str, Any] \| None], copr_client: copr.v3.Client
274	) -> None:
NEW 275	for p in pkgs:	×
NEW 276	if p is None:	×
NEW 277	continue	×
278	p["failed_chroots"] = []	×
NEW 279	for c in p["chroots"]:	×
NEW 280	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
281	if result["state"] == "failed":	×
282	p["failed_chroots"].append(c)	×
283
284
285	def start_rebuild(	1✔
286	project_owner: str,
287	project_name: str,
288	copr_client: copr.v3.Client,
289	pkgs: set[str],
290	snapshot_project_name: str,
291	) -> None:
UNCOV 292	print("START", pkgs, "END")	×
293	# Update the rebuild project to use the latest snapshot
NEW 294	copr_client.project_proxy.edit(	×
295	project_owner,
296	project_name,
297	additional_repos=[
298	"copr://tstellar/fedora-clang-default-cc",
299	f"copr://@fedora-llvm-team/{snapshot_project_name}",
300	],
301	)
302
UNCOV 303	buildopts = {	×
304	"background": True,
305	}
306	logging.info("Rebuilding", len(pkgs), "packages")	×
UNCOV 307	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
UNCOV 308	default_commitish = "f41"	×
309	for p in pkgs:	×
310	logging.info("Rebuild", p)	×
311	try:	×
312	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
313	build_info = koji_session.getBuild(build["build_id"])	×
314	commitish = build_info["source"].split("#")[1]	×
315	except: # noqa: E722	×
316	logging.warning(	×
317	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
318	)
NEW 319	commitish = default_commitish	×
320
UNCOV 321	copr_client.build_proxy.create_from_distgit(	×
322	project_owner, project_name, p, commitish, buildopts=buildopts
323	)
324
325
326	def select_snapshot_project(	1✔
327	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
328	) -> str \| None:
UNCOV 329	project_owner = "@fedora-llvm-team"	×
UNCOV 330	for i in range(max_lookback_days):	×
UNCOV 331	chroots = set()	×
332	day = datetime.date.today() - datetime.timedelta(days=i)	×
333	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
334	logging.info("Trying:", project_name)	×
335	try:	×
336	p = copr_client.project_proxy.get(project_owner, project_name)	×
337	if not p:	×
338	continue	×
339	pkgs = copr_client.build_proxy.get_list(	×
340	project_owner, project_name, "llvm", status="succeeded"
341	)
342	for pkg in pkgs:	×
UNCOV 343	chroots.update(pkg["chroots"])	×
344
345	logging.info(project_name, chroots)	×
346	if all(t in chroots for t in target_chroots):	×
UNCOV 347	logging.info("PASS", project_name)	×
348	return project_name	×
349	except: # noqa: E722	×
350	continue	×
351	logging.warning("FAIL")	×
NEW 352	return None	×
353
354
355	def create_new_project(	1✔
356	project_owner: str,
357	project_name: str,
358	copr_client: copr.v3.Client,
359	target_chroots: list[str],
360	) -> None:
UNCOV 361	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
UNCOV 362	for c in target_chroots:	×
NEW 363	copr_client.project_chroot_proxy.edit(	×
364	project_owner,
365	project_name,
366	c,
367	additional_packages=["fedora-clang-default-cc"],
368	with_opts=["toolchain_clang", "clang_lto"],
369	)
370
371
372	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
UNCOV 373	m = re.search("[0-9]+$", project_name)	×
UNCOV 374	if not m:	×
UNCOV 375	raise Exception(f"Invalid project name: {project_name}")	×
376	return datetime.datetime.fromisoformat(m.group(0)).date()	×
377
378
379	def find_midpoint_project(	1✔
380	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
381	) -> str:
UNCOV 382	good_date = extract_date_from_project(good)	×
UNCOV 383	bad_date = extract_date_from_project(bad)	×
NEW 384	days = (bad_date - good_date).days	×
385	mid_date = good_date + datetime.timedelta(days=days / 2)	×
386	increment = 0	×
387	while mid_date != good_date and mid_date != bad_date:	×
388	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
389	owner = mid_project.split("/")[0]	×
390	project = mid_project.split("/")[1]	×
391	try:	×
392	for builds in copr_client.build_proxy.get_list(	×
393	owner, project, "llvm", "succeeded"
394	):
395	if chroot in builds["chroots"]:	×
UNCOV 396	return mid_project	×
UNCOV 397	except: # noqa: E722	×
398	pass	×
399
NEW 400	increment = increment * -1	×
401	if increment < 0:	×
UNCOV 402	increment -= 1	×
403	else:
404	increment += 1	×
405	mid_date += datetime.timedelta(days=increment)	×
406
407	return good	×
408
409
410	def main() -> None:	1✔
UNCOV 411	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
UNCOV 412	parser = argparse.ArgumentParser()	×
NEW 413	parser.add_argument(	×
414	"command",
415	type=str,
416	choices=[
417	"rebuild",
418	"get-regressions",
419	"get-snapshot-date",
420	"rebuild-in-progress",
421	"bisect",
422	],
423	)
UNCOV 424	parser.add_argument(	×
425	"--start-date", type=str, help="Any ISO date format is accepted"
426	)
427	parser.add_argument("--chroot", type=str)	×
UNCOV 428	parser.add_argument("--good", type=str)	×
UNCOV 429	parser.add_argument("--bad", type=str)	×
430
431	args = parser.parse_args()	×
432	copr_client = copr.v3.Client.create_from_config_file()	×
433
434	os_name = "fedora-41"	×
435	clang_version = "21"	×
UNCOV 436	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
437	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
438	project_owner = "@fedora-llvm-team"	×
439	project_name = f"{os_name}-clang-{clang_version}"	×
440
441	if args.command == "rebuild":	×
442	exclusions = get_exclusions()	×
UNCOV 443	pkgs = get_pkgs(exclusions)	×
444	print(pkgs)	×
445	try:	×
446	copr_client.project_proxy.get(project_owner, project_name)	×
447	copr_pkgs = CoprPkg.get_packages_from_copr(	×
448	project_owner, project_name, copr_client
449	)
450	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
UNCOV 451	except: # noqa: E722	×
UNCOV 452	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
453	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
NEW 454	if snapshot_project is not None:	×
455	start_rebuild(	×
456	project_owner, project_name, copr_client, pkgs, snapshot_project
457	)
NEW 458	elif args.command == "get-regressions":	×
NEW 459	start_time = datetime.datetime.fromisoformat(args.start_date)	×
NEW 460	copr_pkgs = CoprPkg.get_packages_from_copr(	×
461	project_owner, project_name, copr_client
462	)
463	pkg_failures = get_monthly_rebuild_regressions(	×
464	project_owner, project_name, start_time, copr_pkgs
465	)
466	get_chroot_results(list(pkg_failures), copr_client)	×
467	# Delete attributes we don't need to print
UNCOV 468	for p in pkg_failures:	×
NEW 469	if p is None:	×
UNCOV 470	continue	×
471	for k in ["fail_id", "chroots"]:	×
NEW 472	del p[k]	×
473
474	print(json.dumps(pkg_failures))	×
475	elif args.command == "get-snapshot-date":	×
UNCOV 476	project = copr_client.project_proxy.get(project_owner, project_name)	×
477	for repo in project["additional_repos"]:	×
478	match = re.match(	×
479	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
480	)
481	if match:	×
UNCOV 482	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
UNCOV 483	return	×
484	elif args.command == "rebuild-in-progress":	×
485	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
486	"packages"
487	]:
488	for c in pkg["chroots"]:	×
UNCOV 489	build = CoprBuild(pkg["chroots"][c])	×
UNCOV 490	if build.is_in_progress():	×
491	sys.exit(0)	×
492	sys.exit(1)	×
493	elif args.command == "bisect":	×
494	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
495
496
497	if __name__ == "__main__":	1✔
UNCOV 498	main()	×

fedora-llvm-team / llvm-snapshots / 14727094836

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous