14312622006

Committed 07 Apr 2025 03:01PM UTC coverage: 59.138% (-0.9%) from 60.078%

Build # 14312622006

Build Type

Pull #1242

github

Committed by

web-flow

Commit Message

Merge ba1d629e5 into 5e4823df3

Pull Request Pull Request #1242: bisect.sh: Use pre-built binaries to bisect failures

Run Details

2 of 29 new or added lines in 1 file covered. (6.9%)

18 existing lines in 1 file now uncovered.

1084 of 1833 relevant lines covered (59.14%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.65

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Set

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):

    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(self, project_owner, project_name):
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        return {
            "name": self.name,
            "fail_id": self.latest.id,
            "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
            "chroots": self.latest.chroots,
        }

    @property
    def latest(self) -> CoprBuild:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[set]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> set[str]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:
    for p in pkgs:
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
):

    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:
            logging.warn(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:
            continue
    logging.warn("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
):
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )

def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(copr_client: copr.v3.Client, good: str, bad: str, chroot: str
):
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:
            pass
        
        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main():

    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(pkg_failures, copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Set	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	import koji	1✔
13	from munch import Munch	1✔
14
15
16	def is_tier0_package(pkg: str) -> bool:	1✔
17	return pkg in [	1✔
18	"dotnet6.0",
19	"dotnet7.0",
20	"dotnet8.0",
21	"dotnet9.0",
22	"qemu-kvm", # RHEL name
23	"qemu", # Fedora name
24	"golang",
25	"wasi-lbc",
26	]
27
28
29	class CoprBuild(Munch):	1✔
30	pass	1✔
31
32	def is_in_progress(self) -> bool:	1✔
33	return self.state not in [	1✔
34	"succeeded",
35	"forked",
36	"skipped",
37	"failed",
38	"canceled",
39	]
40
41
42	class CoprPkg(Munch):	1✔
43
44	@classmethod	1✔
45	def get_packages_from_copr(	1✔
46	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
47	) -> list["CoprPkg"]:
48	return [	×
49	CoprPkg(p)
50	for p in copr_client.package_proxy.get_list(
51	project_owner,
52	project_name,
53	with_latest_succeeded_build=True,
54	with_latest_build=True,
55	)
56	]
57
58	def get_build(self, name: str) -> CoprBuild:	1✔
59	if "builds" not in self:	1✔
60	return None	×
61	if name not in self.builds:	1✔
62	return None	×
63	build = self.builds[name]	1✔
64	if not build:	1✔
65	return None	1✔
66	return CoprBuild(build)	1✔
67
68	def get_regression_info(self, project_owner, project_name):	1✔
69	owner_url = project_owner	1✔
70	if owner_url[0] == "@":	1✔
71	owner_url = f"g/{owner_url[1:]}"	1✔
72	return {	1✔
73	"name": self.name,
74	"fail_id": self.latest.id,
75	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
76	"chroots": self.latest.chroots,
77	}
78
79	@property	1✔
80	def latest(self) -> CoprBuild:	1✔
81	return self.get_build("latest")	1✔
82
83	@property	1✔
84	def latest_succeeded(self) -> CoprBuild:	1✔
85	return self.get_build("latest_succeeded")	1✔
86
87
88	def load_tests(loader, tests, ignore):	1✔
89	"""We want unittest to pick up all of our doctests
90
91	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
92	See https://stackoverflow.com/a/27171468
93	"""
94	import doctest	×
95
96	tests.addTests(doctest.DocTestSuite())	×
97	return tests	×
98
99
100	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
101	"""Filters out LLVM packages and returns the rest.
102
103	Args:
104	pkgs (set[str]): List of package names
105
106	Returns:
107	set[str]: List of package names without LLVM packages
108
109	Example:
110
111	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
112	>>> filtered=list(filter_llvm_pkgs(pkgs))
113	>>> filtered.sort()
114	>>> print(filtered)
115	['firefox', 'libreoffice']
116
117	"""
118	llvm_pkgs = {	1✔
119	"llvm",
120	"clang",
121	"llvm-bolt",
122	"libomp",
123	"compiler-rt",
124	"lld",
125	"lldb",
126	"polly",
127	"libcxx",
128	"libclc",
129	"flang",
130	"mlir",
131	}
132	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
133	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
134
135
136	def get_exclusions() -> set[str]:	1✔
137	"""
138	This returns a list of packages we don't want to test.
139	"""
140	return set()	×
141
142
143	def get_pkgs(exclusions: set[str]) -> set[set]:	1✔
144	base = dnf.Base()	×
145	conf = base.conf	×
146	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
147	base.repos.add_new_repo(	×
148	f"{c}-source",
149	conf,
150	baseurl=[
151	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
152	],
153	)
154	repos = base.repos.get_matching("*")	×
155	repos.disable()	×
156	repos = base.repos.get_matching("-source")	×
157	repos.enable()	×
158
159	base.fill_sack()	×
160	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
161	q = q.available()	×
162	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
163	pkgs = [p.name for p in list(q)]	×
164	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
165
166
167	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
168	"""Returns the list of packages that should be built in the next rebuild.
169	It will select all the packages that built successfully during the last
170	rebuild.
171
172	Args:
173	pkgs (set[str]): A list of every package that should be considered for
174	the rebuild.
175	copr_pkgs (list[dist]): A list containing the latest build results from
176	the COPR project.
177
178	Returns:
179	set[str]: List of packages that should be rebuilt.
180
181	Example:
182
183	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
184	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
185	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
186	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
187	>>> pkgs = { "b", "c", "d"}
188	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
189	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
190	>>> print(rebuild_pkgs)
191	{'d'}
192	"""
193
194	for p in copr_pkgs:	1✔
195	if p.name not in pkgs:	1✔
196	continue	1✔
197	# Always build tier0 packges.
198	if is_tier0_package(p.name):	1✔
199	continue	×
200	if not p.latest_succeeded:	1✔
201	pkgs.discard(p.name)	1✔
202	continue	1✔
203	if p.latest.id != p.latest_succeeded.id:	1✔
204	pkgs.discard(p.name)	1✔
205	return pkgs	1✔
206
207
208	def get_monthly_rebuild_regressions(	1✔
209	project_owner: str,
210	project_name: str,
211	start_time: datetime.datetime,
212	copr_pkgs: list[CoprPkg],
213	) -> set[str]:
214	"""Returns the list of packages that failed to build in the most recent
215	rebuild, but built successfully in the previous rebuild.
216
217	Args:
218	start_time (datetime.datetime): The start time of the most recent mass
219	rebuild. This needs to be a time
220	before the most recent mass rebuild
221	and after the previous one.
222	copr_pkgs (list[dict]): List of built packages for the COPR project.
223
224	Returns:
225	set[str]: List of packages that regressed in the most recent rebuilt.
226
227	Example:
228
229	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
230	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
231	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
232	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
233	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
234	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
235	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
236	>>> project_owner = "@fedora-llvm-team"
237	>>> project_name = "fedora41-clang-20"
238	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
239	>>> print(regressions)
240	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
241
242	"""
243	pkgs = []	1✔
244	for p in copr_pkgs:	1✔
245	if not p.latest:	1✔
246	continue	×
247
248	# Don't report regressions if there are still builds in progress
249	if p.latest.is_in_progress():	1✔
250	continue	1✔
251
252	if not p.latest_succeeded:	1✔
253	if is_tier0_package(p.name):	1✔
254	pkgs.append(p.get_regression_info(project_owner, project_name))	×
255	continue	1✔
256	if p.latest.id == p.latest_succeeded.id:	1✔
257	continue	1✔
258	# latest is a successful build, but this doesn't mean it failed.
259	# It could be in progress.
260	if p.latest.state != "failed":	1✔
261	continue	1✔
262	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
263	continue	1✔
264	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
265	return pkgs	1✔
266
267
268	def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:	1✔
269	for p in pkgs:	×
270	p["failed_chroots"] = []	×
271	for c in p["chroots"]:	×
272	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
273	if result["state"] == "failed":	×
274	p["failed_chroots"].append(c)	×
275
276
277	def start_rebuild(	1✔
278	project_owner: str,
279	project_name: str,
280	copr_client: copr.v3.Client,
281	pkgs: set[str],
282	snapshot_project_name: str,
283	):
284
285	print("START", pkgs, "END")	×
286	# Update the rebuild project to use the latest snapshot
287	copr_client.project_proxy.edit(	×
288	project_owner,
289	project_name,
290	additional_repos=[
291	"copr://tstellar/fedora-clang-default-cc",
292	f"copr://@fedora-llvm-team/{snapshot_project_name}",
293	],
294	)
295
296	buildopts = {	×
297	"background": True,
298	}
299	logging.info("Rebuilding", len(pkgs), "packages")	×
300	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
301	default_commitish = "f41"	×
302	for p in pkgs:	×
303	logging.info("Rebuild", p)	×
304	try:	×
305	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
306	build_info = koji_session.getBuild(build["build_id"])	×
307	commitish = build_info["source"].split("#")[1]	×
308	except:	×
309	logging.warn(	×
310	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
311	)
312	commitish = default_commitish	×
313
314	copr_client.build_proxy.create_from_distgit(	×
315	project_owner, project_name, p, commitish, buildopts=buildopts
316	)
317
318
319	def select_snapshot_project(	1✔
320	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
321	) -> str \| None:
322	project_owner = "@fedora-llvm-team"	×
323	for i in range(max_lookback_days):	×
324	chroots = set()	×
325	day = datetime.date.today() - datetime.timedelta(days=i)	×
326	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
327	logging.info("Trying:", project_name)	×
328	try:	×
329	p = copr_client.project_proxy.get(project_owner, project_name)	×
330	if not p:	×
331	continue	×
332	pkgs = copr_client.build_proxy.get_list(	×
333	project_owner, project_name, "llvm", status="succeeded"
334	)
335	for pkg in pkgs:	×
336	chroots.update(pkg["chroots"])	×
337
338	logging.info(project_name, chroots)	×
339	if all(t in chroots for t in target_chroots):	×
340	logging.info("PASS", project_name)	×
341	return project_name	×
342	except:	×
343	continue	×
344	logging.warn("FAIL")	×
345	return None	×
346
347
348	def create_new_project(	1✔
349	project_owner: str,
350	project_name: str,
351	copr_client: copr.v3.Client,
352	target_chroots: list[str],
353	):
354	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
355	for c in target_chroots:	×
356	copr_client.project_chroot_proxy.edit(	×
357	project_owner,
358	project_name,
359	c,
360	additional_packages=["fedora-clang-default-cc"],
361	with_opts=["toolchain_clang", "clang_lto"],
362	)
363
364	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
NEW 365	m = re.search("[0-9]+$", project_name)	×
NEW 366	if not m:	×
NEW 367	raise Exception(f"Invalid project name: {project_name}")	×
NEW 368	return datetime.datetime.fromisoformat(m.group(0)).date()	×
369
370
371	def find_midpoint_project(copr_client: copr.v3.Client, good: str, bad: str, chroot: str	1✔
372	):
NEW 373	good_date = extract_date_from_project(good)	×
NEW 374	bad_date = extract_date_from_project(bad)	×
NEW 375	days = (bad_date - good_date).days	×
NEW 376	mid_date = good_date + datetime.timedelta(days=days / 2)	×
NEW 377	increment = 0	×
NEW 378	while mid_date != good_date and mid_date != bad_date:	×
NEW 379	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
NEW 380	owner = mid_project.split("/")[0]	×
NEW 381	project = mid_project.split("/")[1]	×
NEW 382	try:	×
NEW 383	for builds in copr_client.build_proxy.get_list(	×
384	owner, project, "llvm", "succeeded"
385	):
NEW 386	if chroot in builds["chroots"]:	×
NEW 387	return mid_project	×
NEW 388	except:	×
NEW 389	pass	×
390
NEW 391	increment = increment * -1	×
NEW 392	if increment < 0:	×
NEW 393	increment -= 1	×
394	else:
NEW 395	increment += 1	×
NEW 396	mid_date += datetime.timedelta(days=increment)	×
397
NEW 398	return good	×
399
400
401	def main():	1✔
402
UNCOV 403	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
UNCOV 404	parser = argparse.ArgumentParser()	×
405	parser.add_argument(	×
406	"command",
407	type=str,
408	choices=[
409	"rebuild",
410	"get-regressions",
411	"get-snapshot-date",
412	"rebuild-in-progress",
413	"bisect",
414	],
415	)
UNCOV 416	parser.add_argument(	×
417	"--start-date", type=str, help="Any ISO date format is accepted"
418	)
UNCOV 419	parser.add_argument("--chroot", type=str)	×
UNCOV 420	parser.add_argument("--good", type=str)	×
NEW 421	parser.add_argument("--bad", type=str)	×
422
NEW 423	args = parser.parse_args()	×
424	copr_client = copr.v3.Client.create_from_config_file()	×
425
426	os_name = "fedora-41"	×
UNCOV 427	clang_version = "21"	×
428	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
429	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
430	project_owner = "@fedora-llvm-team"	×
431	project_name = f"{os_name}-clang-{clang_version}"	×
432
433	if args.command == "rebuild":	×
UNCOV 434	exclusions = get_exclusions()	×
435	pkgs = get_pkgs(exclusions)	×
436	print(pkgs)	×
437	try:	×
438	copr_client.project_proxy.get(project_owner, project_name)	×
439	copr_pkgs = CoprPkg.get_packages_from_copr(	×
440	project_owner, project_name, copr_client
441	)
UNCOV 442	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
UNCOV 443	except:	×
444	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
445	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
446	start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)	×
447	elif args.command == "get-regressions":	×
448	start_time = datetime.datetime.fromisoformat(args.start_date)	×
449	copr_pkgs = CoprPkg.get_packages_from_copr(	×
450	project_owner, project_name, copr_client
451	)
UNCOV 452	pkg_failures = get_monthly_rebuild_regressions(	×
453	project_owner, project_name, start_time, copr_pkgs
454	)
UNCOV 455	get_chroot_results(pkg_failures, copr_client)	×
456	# Delete attributes we don't need to print
457	for p in pkg_failures:	×
UNCOV 458	for k in ["fail_id", "chroots"]:	×
459	del p[k]	×
460
461	print(json.dumps(pkg_failures))	×
UNCOV 462	elif args.command == "get-snapshot-date":	×
463	project = copr_client.project_proxy.get(project_owner, project_name)	×
464	for repo in project["additional_repos"]:	×
465	match = re.match(	×
466	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
467	)
UNCOV 468	if match:	×
UNCOV 469	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
470	return	×
471	elif args.command == "rebuild-in-progress":	×
472	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
473	"packages"
474	]:
UNCOV 475	for c in pkg["chroots"]:	×
UNCOV 476	build = CoprBuild(pkg["chroots"][c])	×
477	if build.is_in_progress():	×
478	sys.exit(0)	×
479	sys.exit(1)	×
480	elif args.command == "bisect":	×
481	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
482
483
484	if __name__ == "__main__":	1✔
UNCOV 485	main()	×

fedora-llvm-team / llvm-snapshots / 14312622006

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous