14348837962

Committed 09 Apr 2025 04:14AM UTC coverage: 59.138% (-0.9%) from 60.078%

Build # 14348837962

Build Type

push

github

Committed by

web-flow

Commit Message

bisect.sh: Use pre-built binaries to bisect failures (#1242)

This will use pre-built binaries for COPR to start the bisect
process and find a reduced range of good..bad commits before
tarting to bisect with local builds of clang.

Coverage Stats

2 of 32 new or added lines in 1 file covered. (6.25%)

1084 of 1833 relevant lines covered (59.14%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.65

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Set

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):

    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(self, project_owner, project_name):
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        return {
            "name": self.name,
            "fail_id": self.latest.id,
            "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
            "chroots": self.latest.chroots,
        }

    @property
    def latest(self) -> CoprBuild:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[set]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> set[str]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:
    for p in pkgs:
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
):

    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:
            logging.warn(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:
            continue
    logging.warn("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
):
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
):
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main():

    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(pkg_failures, copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Set	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	import koji	1✔
13	from munch import Munch	1✔
14
15
16	def is_tier0_package(pkg: str) -> bool:	1✔
17	return pkg in [	1✔
18	"dotnet6.0",
19	"dotnet7.0",
20	"dotnet8.0",
21	"dotnet9.0",
22	"qemu-kvm", # RHEL name
23	"qemu", # Fedora name
24	"golang",
25	"wasi-lbc",
26	]
27
28
29	class CoprBuild(Munch):	1✔
30	pass	1✔
31
32	def is_in_progress(self) -> bool:	1✔
33	return self.state not in [	1✔
34	"succeeded",
35	"forked",
36	"skipped",
37	"failed",
38	"canceled",
39	]
40
41
42	class CoprPkg(Munch):	1✔
43
44	@classmethod	1✔
45	def get_packages_from_copr(	1✔
46	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
47	) -> list["CoprPkg"]:
48	return [	×
49	CoprPkg(p)
50	for p in copr_client.package_proxy.get_list(
51	project_owner,
52	project_name,
53	with_latest_succeeded_build=True,
54	with_latest_build=True,
55	)
56	]
57
58	def get_build(self, name: str) -> CoprBuild:	1✔
59	if "builds" not in self:	1✔
60	return None	×
61	if name not in self.builds:	1✔
62	return None	×
63	build = self.builds[name]	1✔
64	if not build:	1✔
65	return None	1✔
66	return CoprBuild(build)	1✔
67
68	def get_regression_info(self, project_owner, project_name):	1✔
69	owner_url = project_owner	1✔
70	if owner_url[0] == "@":	1✔
71	owner_url = f"g/{owner_url[1:]}"	1✔
72	return {	1✔
73	"name": self.name,
74	"fail_id": self.latest.id,
75	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
76	"chroots": self.latest.chroots,
77	}
78
79	@property	1✔
80	def latest(self) -> CoprBuild:	1✔
81	return self.get_build("latest")	1✔
82
83	@property	1✔
84	def latest_succeeded(self) -> CoprBuild:	1✔
85	return self.get_build("latest_succeeded")	1✔
86
87
88	def load_tests(loader, tests, ignore):	1✔
89	"""We want unittest to pick up all of our doctests
90
91	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
92	See https://stackoverflow.com/a/27171468
93	"""
94	import doctest	×
95
96	tests.addTests(doctest.DocTestSuite())	×
97	return tests	×
98
99
100	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
101	"""Filters out LLVM packages and returns the rest.
102
103	Args:
104	pkgs (set[str]): List of package names
105
106	Returns:
107	set[str]: List of package names without LLVM packages
108
109	Example:
110
111	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
112	>>> filtered=list(filter_llvm_pkgs(pkgs))
113	>>> filtered.sort()
114	>>> print(filtered)
115	['firefox', 'libreoffice']
116
117	"""
118	llvm_pkgs = {	1✔
119	"llvm",
120	"clang",
121	"llvm-bolt",
122	"libomp",
123	"compiler-rt",
124	"lld",
125	"lldb",
126	"polly",
127	"libcxx",
128	"libclc",
129	"flang",
130	"mlir",
131	}
132	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
133	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
134
135
136	def get_exclusions() -> set[str]:	1✔
137	"""
138	This returns a list of packages we don't want to test.
139	"""
140	return set()	×
141
142
143	def get_pkgs(exclusions: set[str]) -> set[set]:	1✔
144	base = dnf.Base()	×
145	conf = base.conf	×
146	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
147	base.repos.add_new_repo(	×
148	f"{c}-source",
149	conf,
150	baseurl=[
151	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
152	],
153	)
154	repos = base.repos.get_matching("*")	×
155	repos.disable()	×
156	repos = base.repos.get_matching("-source")	×
157	repos.enable()	×
158
159	base.fill_sack()	×
160	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
161	q = q.available()	×
162	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
163	pkgs = [p.name for p in list(q)]	×
164	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
165
166
167	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
168	"""Returns the list of packages that should be built in the next rebuild.
169	It will select all the packages that built successfully during the last
170	rebuild.
171
172	Args:
173	pkgs (set[str]): A list of every package that should be considered for
174	the rebuild.
175	copr_pkgs (list[dist]): A list containing the latest build results from
176	the COPR project.
177
178	Returns:
179	set[str]: List of packages that should be rebuilt.
180
181	Example:
182
183	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
184	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
185	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
186	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
187	>>> pkgs = { "b", "c", "d"}
188	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
189	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
190	>>> print(rebuild_pkgs)
191	{'d'}
192	"""
193
194	for p in copr_pkgs:	1✔
195	if p.name not in pkgs:	1✔
196	continue	1✔
197	# Always build tier0 packges.
198	if is_tier0_package(p.name):	1✔
199	continue	×
200	if not p.latest_succeeded:	1✔
201	pkgs.discard(p.name)	1✔
202	continue	1✔
203	if p.latest.id != p.latest_succeeded.id:	1✔
204	pkgs.discard(p.name)	1✔
205	return pkgs	1✔
206
207
208	def get_monthly_rebuild_regressions(	1✔
209	project_owner: str,
210	project_name: str,
211	start_time: datetime.datetime,
212	copr_pkgs: list[CoprPkg],
213	) -> set[str]:
214	"""Returns the list of packages that failed to build in the most recent
215	rebuild, but built successfully in the previous rebuild.
216
217	Args:
218	start_time (datetime.datetime): The start time of the most recent mass
219	rebuild. This needs to be a time
220	before the most recent mass rebuild
221	and after the previous one.
222	copr_pkgs (list[dict]): List of built packages for the COPR project.
223
224	Returns:
225	set[str]: List of packages that regressed in the most recent rebuilt.
226
227	Example:
228
229	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
230	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
231	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
232	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
233	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
234	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
235	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
236	>>> project_owner = "@fedora-llvm-team"
237	>>> project_name = "fedora41-clang-20"
238	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
239	>>> print(regressions)
240	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
241
242	"""
243	pkgs = []	1✔
244	for p in copr_pkgs:	1✔
245	if not p.latest:	1✔
246	continue	×
247
248	# Don't report regressions if there are still builds in progress
249	if p.latest.is_in_progress():	1✔
250	continue	1✔
251
252	if not p.latest_succeeded:	1✔
253	if is_tier0_package(p.name):	1✔
254	pkgs.append(p.get_regression_info(project_owner, project_name))	×
255	continue	1✔
256	if p.latest.id == p.latest_succeeded.id:	1✔
257	continue	1✔
258	# latest is a successful build, but this doesn't mean it failed.
259	# It could be in progress.
260	if p.latest.state != "failed":	1✔
261	continue	1✔
262	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
263	continue	1✔
264	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
265	return pkgs	1✔
266
267
268	def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:	1✔
269	for p in pkgs:	×
270	p["failed_chroots"] = []	×
271	for c in p["chroots"]:	×
272	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
273	if result["state"] == "failed":	×
274	p["failed_chroots"].append(c)	×
275
276
277	def start_rebuild(	1✔
278	project_owner: str,
279	project_name: str,
280	copr_client: copr.v3.Client,
281	pkgs: set[str],
282	snapshot_project_name: str,
283	):
284
285	print("START", pkgs, "END")	×
286	# Update the rebuild project to use the latest snapshot
287	copr_client.project_proxy.edit(	×
288	project_owner,
289	project_name,
290	additional_repos=[
291	"copr://tstellar/fedora-clang-default-cc",
292	f"copr://@fedora-llvm-team/{snapshot_project_name}",
293	],
294	)
295
296	buildopts = {	×
297	"background": True,
298	}
299	logging.info("Rebuilding", len(pkgs), "packages")	×
300	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
301	default_commitish = "f41"	×
302	for p in pkgs:	×
303	logging.info("Rebuild", p)	×
304	try:	×
305	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
306	build_info = koji_session.getBuild(build["build_id"])	×
307	commitish = build_info["source"].split("#")[1]	×
308	except:	×
309	logging.warn(	×
310	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
311	)
312	commitish = default_commitish	×
313
314	copr_client.build_proxy.create_from_distgit(	×
315	project_owner, project_name, p, commitish, buildopts=buildopts
316	)
317
318
319	def select_snapshot_project(	1✔
320	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
321	) -> str \| None:
322	project_owner = "@fedora-llvm-team"	×
323	for i in range(max_lookback_days):	×
324	chroots = set()	×
325	day = datetime.date.today() - datetime.timedelta(days=i)	×
326	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
327	logging.info("Trying:", project_name)	×
328	try:	×
329	p = copr_client.project_proxy.get(project_owner, project_name)	×
330	if not p:	×
331	continue	×
332	pkgs = copr_client.build_proxy.get_list(	×
333	project_owner, project_name, "llvm", status="succeeded"
334	)
335	for pkg in pkgs:	×
336	chroots.update(pkg["chroots"])	×
337
338	logging.info(project_name, chroots)	×
339	if all(t in chroots for t in target_chroots):	×
340	logging.info("PASS", project_name)	×
341	return project_name	×
342	except:	×
343	continue	×
344	logging.warn("FAIL")	×
345	return None	×
346
347
348	def create_new_project(	1✔
349	project_owner: str,
350	project_name: str,
351	copr_client: copr.v3.Client,
352	target_chroots: list[str],
353	):
354	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
355	for c in target_chroots:	×
356	copr_client.project_chroot_proxy.edit(	×
357	project_owner,
358	project_name,
359	c,
360	additional_packages=["fedora-clang-default-cc"],
361	with_opts=["toolchain_clang", "clang_lto"],
362	)
363
364
365	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
NEW 366	m = re.search("[0-9]+$", project_name)	×
NEW 367	if not m:	×
NEW 368	raise Exception(f"Invalid project name: {project_name}")	×
NEW 369	return datetime.datetime.fromisoformat(m.group(0)).date()	×
370
371
372	def find_midpoint_project(	1✔
373	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
374	):
NEW 375	good_date = extract_date_from_project(good)	×
NEW 376	bad_date = extract_date_from_project(bad)	×
NEW 377	days = (bad_date - good_date).days	×
NEW 378	mid_date = good_date + datetime.timedelta(days=days / 2)	×
NEW 379	increment = 0	×
NEW 380	while mid_date != good_date and mid_date != bad_date:	×
NEW 381	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
NEW 382	owner = mid_project.split("/")[0]	×
NEW 383	project = mid_project.split("/")[1]	×
NEW 384	try:	×
NEW 385	for builds in copr_client.build_proxy.get_list(	×
386	owner, project, "llvm", "succeeded"
387	):
NEW 388	if chroot in builds["chroots"]:	×
NEW 389	return mid_project	×
NEW 390	except:	×
NEW 391	pass	×
392
NEW 393	increment = increment * -1	×
NEW 394	if increment < 0:	×
NEW 395	increment -= 1	×
396	else:
NEW 397	increment += 1	×
NEW 398	mid_date += datetime.timedelta(days=increment)	×
399
NEW 400	return good	×
401
402
403	def main():	1✔
404
405	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
406	parser = argparse.ArgumentParser()	×
407	parser.add_argument(	×
408	"command",
409	type=str,
410	choices=[
411	"rebuild",
412	"get-regressions",
413	"get-snapshot-date",
414	"rebuild-in-progress",
415	"bisect",
416	],
417	)
418	parser.add_argument(	×
419	"--start-date", type=str, help="Any ISO date format is accepted"
420	)
NEW 421	parser.add_argument("--chroot", type=str)	×
NEW 422	parser.add_argument("--good", type=str)	×
NEW 423	parser.add_argument("--bad", type=str)	×
424
425	args = parser.parse_args()	×
426	copr_client = copr.v3.Client.create_from_config_file()	×
427
428	os_name = "fedora-41"	×
429	clang_version = "21"	×
430	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
431	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
432	project_owner = "@fedora-llvm-team"	×
433	project_name = f"{os_name}-clang-{clang_version}"	×
434
435	if args.command == "rebuild":	×
436	exclusions = get_exclusions()	×
437	pkgs = get_pkgs(exclusions)	×
438	print(pkgs)	×
439	try:	×
440	copr_client.project_proxy.get(project_owner, project_name)	×
441	copr_pkgs = CoprPkg.get_packages_from_copr(	×
442	project_owner, project_name, copr_client
443	)
444	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
445	except:	×
446	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
447	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
448	start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)	×
449	elif args.command == "get-regressions":	×
450	start_time = datetime.datetime.fromisoformat(args.start_date)	×
451	copr_pkgs = CoprPkg.get_packages_from_copr(	×
452	project_owner, project_name, copr_client
453	)
454	pkg_failures = get_monthly_rebuild_regressions(	×
455	project_owner, project_name, start_time, copr_pkgs
456	)
457	get_chroot_results(pkg_failures, copr_client)	×
458	# Delete attributes we don't need to print
459	for p in pkg_failures:	×
460	for k in ["fail_id", "chroots"]:	×
461	del p[k]	×
462
463	print(json.dumps(pkg_failures))	×
464	elif args.command == "get-snapshot-date":	×
465	project = copr_client.project_proxy.get(project_owner, project_name)	×
466	for repo in project["additional_repos"]:	×
467	match = re.match(	×
468	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
469	)
470	if match:	×
471	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
472	return	×
473	elif args.command == "rebuild-in-progress":	×
474	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
475	"packages"
476	]:
477	for c in pkg["chroots"]:	×
478	build = CoprBuild(pkg["chroots"][c])	×
479	if build.is_in_progress():	×
480	sys.exit(0)	×
481	sys.exit(1)	×
NEW 482	elif args.command == "bisect":	×
NEW 483	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
484
485
486	if __name__ == "__main__":	1✔
487	main()	×

fedora-llvm-team / llvm-snapshots / 14348837962

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous