14312904507

Committed 07 Apr 2025 03:14PM UTC coverage: 59.138% (-0.9%) from 60.078%

Build # 14312904507

Build Type

Pull #1242

github

Committed by

web-flow

Commit Message

Merge 73c9385d6 into 5e4823df3

Pull Request Pull Request #1242: bisect.sh: Use pre-built binaries to bisect failures

Run Details

2 of 30 new or added lines in 1 file covered. (6.67%)

13 existing lines in 1 file now uncovered.

1084 of 1833 relevant lines covered (59.14%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.65

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Set

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):

    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(self, project_owner, project_name):
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        return {
            "name": self.name,
            "fail_id": self.latest.id,
            "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
            "chroots": self.latest.chroots,
        }

    @property
    def latest(self) -> CoprBuild:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[set]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> set[str]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:
    for p in pkgs:
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
):

    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:
            logging.warn(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:
            continue
    logging.warn("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
):
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )

def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
):
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main():

    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(pkg_failures, copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Set	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	import koji	1✔
13	from munch import Munch	1✔
14
15
16	def is_tier0_package(pkg: str) -> bool:	1✔
17	return pkg in [	1✔
18	"dotnet6.0",
19	"dotnet7.0",
20	"dotnet8.0",
21	"dotnet9.0",
22	"qemu-kvm", # RHEL name
23	"qemu", # Fedora name
24	"golang",
25	"wasi-lbc",
26	]
27
28
29	class CoprBuild(Munch):	1✔
30	pass	1✔
31
32	def is_in_progress(self) -> bool:	1✔
33	return self.state not in [	1✔
34	"succeeded",
35	"forked",
36	"skipped",
37	"failed",
38	"canceled",
39	]
40
41
42	class CoprPkg(Munch):	1✔
43
44	@classmethod	1✔
45	def get_packages_from_copr(	1✔
46	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
47	) -> list["CoprPkg"]:
48	return [	×
49	CoprPkg(p)
50	for p in copr_client.package_proxy.get_list(
51	project_owner,
52	project_name,
53	with_latest_succeeded_build=True,
54	with_latest_build=True,
55	)
56	]
57
58	def get_build(self, name: str) -> CoprBuild:	1✔
59	if "builds" not in self:	1✔
60	return None	×
61	if name not in self.builds:	1✔
62	return None	×
63	build = self.builds[name]	1✔
64	if not build:	1✔
65	return None	1✔
66	return CoprBuild(build)	1✔
67
68	def get_regression_info(self, project_owner, project_name):	1✔
69	owner_url = project_owner	1✔
70	if owner_url[0] == "@":	1✔
71	owner_url = f"g/{owner_url[1:]}"	1✔
72	return {	1✔
73	"name": self.name,
74	"fail_id": self.latest.id,
75	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
76	"chroots": self.latest.chroots,
77	}
78
79	@property	1✔
80	def latest(self) -> CoprBuild:	1✔
81	return self.get_build("latest")	1✔
82
83	@property	1✔
84	def latest_succeeded(self) -> CoprBuild:	1✔
85	return self.get_build("latest_succeeded")	1✔
86
87
88	def load_tests(loader, tests, ignore):	1✔
89	"""We want unittest to pick up all of our doctests
90
91	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
92	See https://stackoverflow.com/a/27171468
93	"""
94	import doctest	×
95
96	tests.addTests(doctest.DocTestSuite())	×
97	return tests	×
98
99
100	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
101	"""Filters out LLVM packages and returns the rest.
102
103	Args:
104	pkgs (set[str]): List of package names
105
106	Returns:
107	set[str]: List of package names without LLVM packages
108
109	Example:
110
111	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
112	>>> filtered=list(filter_llvm_pkgs(pkgs))
113	>>> filtered.sort()
114	>>> print(filtered)
115	['firefox', 'libreoffice']
116
117	"""
118	llvm_pkgs = {	1✔
119	"llvm",
120	"clang",
121	"llvm-bolt",
122	"libomp",
123	"compiler-rt",
124	"lld",
125	"lldb",
126	"polly",
127	"libcxx",
128	"libclc",
129	"flang",
130	"mlir",
131	}
132	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
133	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
134
135
136	def get_exclusions() -> set[str]:	1✔
137	"""
138	This returns a list of packages we don't want to test.
139	"""
140	return set()	×
141
142
143	def get_pkgs(exclusions: set[str]) -> set[set]:	1✔
144	base = dnf.Base()	×
145	conf = base.conf	×
146	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
147	base.repos.add_new_repo(	×
148	f"{c}-source",
149	conf,
150	baseurl=[
151	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
152	],
153	)
154	repos = base.repos.get_matching("*")	×
155	repos.disable()	×
156	repos = base.repos.get_matching("-source")	×
157	repos.enable()	×
158
159	base.fill_sack()	×
160	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
161	q = q.available()	×
162	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
163	pkgs = [p.name for p in list(q)]	×
164	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
165
166
167	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
168	"""Returns the list of packages that should be built in the next rebuild.
169	It will select all the packages that built successfully during the last
170	rebuild.
171
172	Args:
173	pkgs (set[str]): A list of every package that should be considered for
174	the rebuild.
175	copr_pkgs (list[dist]): A list containing the latest build results from
176	the COPR project.
177
178	Returns:
179	set[str]: List of packages that should be rebuilt.
180
181	Example:
182
183	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
184	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
185	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
186	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
187	>>> pkgs = { "b", "c", "d"}
188	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
189	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
190	>>> print(rebuild_pkgs)
191	{'d'}
192	"""
193
194	for p in copr_pkgs:	1✔
195	if p.name not in pkgs:	1✔
196	continue	1✔
197	# Always build tier0 packges.
198	if is_tier0_package(p.name):	1✔
199	continue	×
200	if not p.latest_succeeded:	1✔
201	pkgs.discard(p.name)	1✔
202	continue	1✔
203	if p.latest.id != p.latest_succeeded.id:	1✔
204	pkgs.discard(p.name)	1✔
205	return pkgs	1✔
206
207
208	def get_monthly_rebuild_regressions(	1✔
209	project_owner: str,
210	project_name: str,
211	start_time: datetime.datetime,
212	copr_pkgs: list[CoprPkg],
213	) -> set[str]:
214	"""Returns the list of packages that failed to build in the most recent
215	rebuild, but built successfully in the previous rebuild.
216
217	Args:
218	start_time (datetime.datetime): The start time of the most recent mass
219	rebuild. This needs to be a time
220	before the most recent mass rebuild
221	and after the previous one.
222	copr_pkgs (list[dict]): List of built packages for the COPR project.
223
224	Returns:
225	set[str]: List of packages that regressed in the most recent rebuilt.
226
227	Example:
228
229	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
230	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
231	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
232	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
233	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
234	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
235	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
236	>>> project_owner = "@fedora-llvm-team"
237	>>> project_name = "fedora41-clang-20"
238	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
239	>>> print(regressions)
240	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
241
242	"""
243	pkgs = []	1✔
244	for p in copr_pkgs:	1✔
245	if not p.latest:	1✔
246	continue	×
247
248	# Don't report regressions if there are still builds in progress
249	if p.latest.is_in_progress():	1✔
250	continue	1✔
251
252	if not p.latest_succeeded:	1✔
253	if is_tier0_package(p.name):	1✔
254	pkgs.append(p.get_regression_info(project_owner, project_name))	×
255	continue	1✔
256	if p.latest.id == p.latest_succeeded.id:	1✔
257	continue	1✔
258	# latest is a successful build, but this doesn't mean it failed.
259	# It could be in progress.
260	if p.latest.state != "failed":	1✔
261	continue	1✔
262	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
263	continue	1✔
264	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
265	return pkgs	1✔
266
267
268	def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:	1✔
269	for p in pkgs:	×
270	p["failed_chroots"] = []	×
271	for c in p["chroots"]:	×
272	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
273	if result["state"] == "failed":	×
274	p["failed_chroots"].append(c)	×
275
276
277	def start_rebuild(	1✔
278	project_owner: str,
279	project_name: str,
280	copr_client: copr.v3.Client,
281	pkgs: set[str],
282	snapshot_project_name: str,
283	):
284
285	print("START", pkgs, "END")	×
286	# Update the rebuild project to use the latest snapshot
287	copr_client.project_proxy.edit(	×
288	project_owner,
289	project_name,
290	additional_repos=[
291	"copr://tstellar/fedora-clang-default-cc",
292	f"copr://@fedora-llvm-team/{snapshot_project_name}",
293	],
294	)
295
296	buildopts = {	×
297	"background": True,
298	}
299	logging.info("Rebuilding", len(pkgs), "packages")	×
300	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
301	default_commitish = "f41"	×
302	for p in pkgs:	×
303	logging.info("Rebuild", p)	×
304	try:	×
305	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
306	build_info = koji_session.getBuild(build["build_id"])	×
307	commitish = build_info["source"].split("#")[1]	×
308	except:	×
309	logging.warn(	×
310	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
311	)
312	commitish = default_commitish	×
313
314	copr_client.build_proxy.create_from_distgit(	×
315	project_owner, project_name, p, commitish, buildopts=buildopts
316	)
317
318
319	def select_snapshot_project(	1✔
320	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
321	) -> str \| None:
322	project_owner = "@fedora-llvm-team"	×
323	for i in range(max_lookback_days):	×
324	chroots = set()	×
325	day = datetime.date.today() - datetime.timedelta(days=i)	×
326	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
327	logging.info("Trying:", project_name)	×
328	try:	×
329	p = copr_client.project_proxy.get(project_owner, project_name)	×
330	if not p:	×
331	continue	×
332	pkgs = copr_client.build_proxy.get_list(	×
333	project_owner, project_name, "llvm", status="succeeded"
334	)
335	for pkg in pkgs:	×
336	chroots.update(pkg["chroots"])	×
337
338	logging.info(project_name, chroots)	×
339	if all(t in chroots for t in target_chroots):	×
340	logging.info("PASS", project_name)	×
341	return project_name	×
342	except:	×
343	continue	×
344	logging.warn("FAIL")	×
345	return None	×
346
347
348	def create_new_project(	1✔
349	project_owner: str,
350	project_name: str,
351	copr_client: copr.v3.Client,
352	target_chroots: list[str],
353	):
354	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
355	for c in target_chroots:	×
356	copr_client.project_chroot_proxy.edit(	×
357	project_owner,
358	project_name,
359	c,
360	additional_packages=["fedora-clang-default-cc"],
361	with_opts=["toolchain_clang", "clang_lto"],
362	)
363
364	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
NEW 365	m = re.search("[0-9]+$", project_name)	×
NEW 366	if not m:	×
NEW 367	raise Exception(f"Invalid project name: {project_name}")	×
NEW 368	return datetime.datetime.fromisoformat(m.group(0)).date()	×
369
370
371	def find_midpoint_project(	1✔
372	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
373	):
NEW 374	good_date = extract_date_from_project(good)	×
NEW 375	bad_date = extract_date_from_project(bad)	×
NEW 376	days = (bad_date - good_date).days	×
NEW 377	mid_date = good_date + datetime.timedelta(days=days / 2)	×
NEW 378	increment = 0	×
NEW 379	while mid_date != good_date and mid_date != bad_date:	×
NEW 380	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
NEW 381	owner = mid_project.split("/")[0]	×
NEW 382	project = mid_project.split("/")[1]	×
NEW 383	try:	×
NEW 384	for builds in copr_client.build_proxy.get_list(	×
385	owner, project, "llvm", "succeeded"
386	):
NEW 387	if chroot in builds["chroots"]:	×
NEW 388	return mid_project	×
NEW 389	except:	×
NEW 390	pass	×
391
NEW 392	increment = increment * -1	×
NEW 393	if increment < 0:	×
NEW 394	increment -= 1	×
395	else:
NEW 396	increment += 1	×
NEW 397	mid_date += datetime.timedelta(days=increment)	×
398
NEW 399	return good	×
400
401
402	def main():	1✔
403
UNCOV 404	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
405	parser = argparse.ArgumentParser()	×
406	parser.add_argument(	×
407	"command",
408	type=str,
409	choices=[
410	"rebuild",
411	"get-regressions",
412	"get-snapshot-date",
413	"rebuild-in-progress",
414	"bisect",
415	],
416	)
UNCOV 417	parser.add_argument(	×
418	"--start-date", type=str, help="Any ISO date format is accepted"
419	)
UNCOV 420	parser.add_argument("--chroot", type=str)	×
NEW 421	parser.add_argument("--good", type=str)	×
NEW 422	parser.add_argument("--bad", type=str)	×
423
424	args = parser.parse_args()	×
425	copr_client = copr.v3.Client.create_from_config_file()	×
426
UNCOV 427	os_name = "fedora-41"	×
428	clang_version = "21"	×
429	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
430	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
431	project_owner = "@fedora-llvm-team"	×
432	project_name = f"{os_name}-clang-{clang_version}"	×
433
UNCOV 434	if args.command == "rebuild":	×
435	exclusions = get_exclusions()	×
436	pkgs = get_pkgs(exclusions)	×
437	print(pkgs)	×
438	try:	×
439	copr_client.project_proxy.get(project_owner, project_name)	×
440	copr_pkgs = CoprPkg.get_packages_from_copr(	×
441	project_owner, project_name, copr_client
442	)
UNCOV 443	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
444	except:	×
445	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
446	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
447	start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)	×
448	elif args.command == "get-regressions":	×
449	start_time = datetime.datetime.fromisoformat(args.start_date)	×
450	copr_pkgs = CoprPkg.get_packages_from_copr(	×
451	project_owner, project_name, copr_client
452	)
UNCOV 453	pkg_failures = get_monthly_rebuild_regressions(	×
454	project_owner, project_name, start_time, copr_pkgs
455	)
UNCOV 456	get_chroot_results(pkg_failures, copr_client)	×
457	# Delete attributes we don't need to print
UNCOV 458	for p in pkg_failures:	×
459	for k in ["fail_id", "chroots"]:	×
460	del p[k]	×
461
UNCOV 462	print(json.dumps(pkg_failures))	×
463	elif args.command == "get-snapshot-date":	×
464	project = copr_client.project_proxy.get(project_owner, project_name)	×
465	for repo in project["additional_repos"]:	×
466	match = re.match(	×
467	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
468	)
UNCOV 469	if match:	×
470	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
471	return	×
472	elif args.command == "rebuild-in-progress":	×
473	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
474	"packages"
475	]:
UNCOV 476	for c in pkg["chroots"]:	×
477	build = CoprBuild(pkg["chroots"][c])	×
478	if build.is_in_progress():	×
479	sys.exit(0)	×
480	sys.exit(1)	×
481	elif args.command == "bisect":	×
NEW 482	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
483
484
485	if __name__ == "__main__":	1✔
UNCOV 486	main()	×

fedora-llvm-team / llvm-snapshots / 14312904507

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous