14312286031

Committed 07 Apr 2025 02:46PM UTC coverage: 59.138% (-0.9%) from 60.078%

Build # 14312286031

Build Type

Pull #1242

github

Committed by

web-flow

Commit Message

Merge 90819cbee into 5e4823df3

Pull Request Pull Request #1242: bisect.sh: Use pre-built binaries to bisect failures

Run Details

1 of 28 new or added lines in 1 file covered. (3.57%)

16 existing lines in 1 file now uncovered.

1084 of 1833 relevant lines covered (59.14%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.65

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Set

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):

    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(self, project_owner, project_name):
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        return {
            "name": self.name,
            "fail_id": self.latest.id,
            "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
            "chroots": self.latest.chroots,
        }

    @property
    def latest(self) -> CoprBuild:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[set]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> set[str]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:
    for p in pkgs:
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
):

    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:
            logging.warn(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:
            continue
    logging.warn("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
):
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )

def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client,
    good: str,
    bad: str,
    chroot: str
):
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split('/')[0]
        project = mid_project.split('/')[1]
        try:
            for builds in copr_client.build_proxy.get_list(owner, project, 'llvm', "succeeded"):
                if chroot in builds['chroots']:
                    return mid_project
        except:
            pass
        
        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main():

    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect"
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument(
        "--chroot", type=str
    )
    parser.add_argument(
        "--good", type=str
    )
    parser.add_argument(
        "--bad", type=str
    )

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "21"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(pkg_failures, copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Set	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	import koji	1✔
13	from munch import Munch	1✔
14
15
16	def is_tier0_package(pkg: str) -> bool:	1✔
17	return pkg in [	1✔
18	"dotnet6.0",
19	"dotnet7.0",
20	"dotnet8.0",
21	"dotnet9.0",
22	"qemu-kvm", # RHEL name
23	"qemu", # Fedora name
24	"golang",
25	"wasi-lbc",
26	]
27
28
29	class CoprBuild(Munch):	1✔
30	pass	1✔
31
32	def is_in_progress(self) -> bool:	1✔
33	return self.state not in [	1✔
34	"succeeded",
35	"forked",
36	"skipped",
37	"failed",
38	"canceled",
39	]
40
41
42	class CoprPkg(Munch):	1✔
43
44	@classmethod	1✔
45	def get_packages_from_copr(	1✔
46	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
47	) -> list["CoprPkg"]:
48	return [	×
49	CoprPkg(p)
50	for p in copr_client.package_proxy.get_list(
51	project_owner,
52	project_name,
53	with_latest_succeeded_build=True,
54	with_latest_build=True,
55	)
56	]
57
58	def get_build(self, name: str) -> CoprBuild:	1✔
59	if "builds" not in self:	1✔
60	return None	×
61	if name not in self.builds:	1✔
62	return None	×
63	build = self.builds[name]	1✔
64	if not build:	1✔
65	return None	1✔
66	return CoprBuild(build)	1✔
67
68	def get_regression_info(self, project_owner, project_name):	1✔
69	owner_url = project_owner	1✔
70	if owner_url[0] == "@":	1✔
71	owner_url = f"g/{owner_url[1:]}"	1✔
72	return {	1✔
73	"name": self.name,
74	"fail_id": self.latest.id,
75	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
76	"chroots": self.latest.chroots,
77	}
78
79	@property	1✔
80	def latest(self) -> CoprBuild:	1✔
81	return self.get_build("latest")	1✔
82
83	@property	1✔
84	def latest_succeeded(self) -> CoprBuild:	1✔
85	return self.get_build("latest_succeeded")	1✔
86
87
88	def load_tests(loader, tests, ignore):	1✔
89	"""We want unittest to pick up all of our doctests
90
91	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
92	See https://stackoverflow.com/a/27171468
93	"""
94	import doctest	×
95
96	tests.addTests(doctest.DocTestSuite())	×
97	return tests	×
98
99
100	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
101	"""Filters out LLVM packages and returns the rest.
102
103	Args:
104	pkgs (set[str]): List of package names
105
106	Returns:
107	set[str]: List of package names without LLVM packages
108
109	Example:
110
111	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
112	>>> filtered=list(filter_llvm_pkgs(pkgs))
113	>>> filtered.sort()
114	>>> print(filtered)
115	['firefox', 'libreoffice']
116
117	"""
118	llvm_pkgs = {	1✔
119	"llvm",
120	"clang",
121	"llvm-bolt",
122	"libomp",
123	"compiler-rt",
124	"lld",
125	"lldb",
126	"polly",
127	"libcxx",
128	"libclc",
129	"flang",
130	"mlir",
131	}
132	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
133	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
134
135
136	def get_exclusions() -> set[str]:	1✔
137	"""
138	This returns a list of packages we don't want to test.
139	"""
140	return set()	×
141
142
143	def get_pkgs(exclusions: set[str]) -> set[set]:	1✔
144	base = dnf.Base()	×
145	conf = base.conf	×
146	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
147	base.repos.add_new_repo(	×
148	f"{c}-source",
149	conf,
150	baseurl=[
151	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
152	],
153	)
154	repos = base.repos.get_matching("*")	×
155	repos.disable()	×
156	repos = base.repos.get_matching("-source")	×
157	repos.enable()	×
158
159	base.fill_sack()	×
160	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
161	q = q.available()	×
162	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
163	pkgs = [p.name for p in list(q)]	×
164	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
165
166
167	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
168	"""Returns the list of packages that should be built in the next rebuild.
169	It will select all the packages that built successfully during the last
170	rebuild.
171
172	Args:
173	pkgs (set[str]): A list of every package that should be considered for
174	the rebuild.
175	copr_pkgs (list[dist]): A list containing the latest build results from
176	the COPR project.
177
178	Returns:
179	set[str]: List of packages that should be rebuilt.
180
181	Example:
182
183	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
184	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
185	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
186	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
187	>>> pkgs = { "b", "c", "d"}
188	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
189	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
190	>>> print(rebuild_pkgs)
191	{'d'}
192	"""
193
194	for p in copr_pkgs:	1✔
195	if p.name not in pkgs:	1✔
196	continue	1✔
197	# Always build tier0 packges.
198	if is_tier0_package(p.name):	1✔
199	continue	×
200	if not p.latest_succeeded:	1✔
201	pkgs.discard(p.name)	1✔
202	continue	1✔
203	if p.latest.id != p.latest_succeeded.id:	1✔
204	pkgs.discard(p.name)	1✔
205	return pkgs	1✔
206
207
208	def get_monthly_rebuild_regressions(	1✔
209	project_owner: str,
210	project_name: str,
211	start_time: datetime.datetime,
212	copr_pkgs: list[CoprPkg],
213	) -> set[str]:
214	"""Returns the list of packages that failed to build in the most recent
215	rebuild, but built successfully in the previous rebuild.
216
217	Args:
218	start_time (datetime.datetime): The start time of the most recent mass
219	rebuild. This needs to be a time
220	before the most recent mass rebuild
221	and after the previous one.
222	copr_pkgs (list[dict]): List of built packages for the COPR project.
223
224	Returns:
225	set[str]: List of packages that regressed in the most recent rebuilt.
226
227	Example:
228
229	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
230	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
231	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
232	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
233	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
234	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
235	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
236	>>> project_owner = "@fedora-llvm-team"
237	>>> project_name = "fedora41-clang-20"
238	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
239	>>> print(regressions)
240	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
241
242	"""
243	pkgs = []	1✔
244	for p in copr_pkgs:	1✔
245	if not p.latest:	1✔
246	continue	×
247
248	# Don't report regressions if there are still builds in progress
249	if p.latest.is_in_progress():	1✔
250	continue	1✔
251
252	if not p.latest_succeeded:	1✔
253	if is_tier0_package(p.name):	1✔
254	pkgs.append(p.get_regression_info(project_owner, project_name))	×
255	continue	1✔
256	if p.latest.id == p.latest_succeeded.id:	1✔
257	continue	1✔
258	# latest is a successful build, but this doesn't mean it failed.
259	# It could be in progress.
260	if p.latest.state != "failed":	1✔
261	continue	1✔
262	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
263	continue	1✔
264	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
265	return pkgs	1✔
266
267
268	def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:	1✔
269	for p in pkgs:	×
270	p["failed_chroots"] = []	×
271	for c in p["chroots"]:	×
272	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
273	if result["state"] == "failed":	×
274	p["failed_chroots"].append(c)	×
275
276
277	def start_rebuild(	1✔
278	project_owner: str,
279	project_name: str,
280	copr_client: copr.v3.Client,
281	pkgs: set[str],
282	snapshot_project_name: str,
283	):
284
285	print("START", pkgs, "END")	×
286	# Update the rebuild project to use the latest snapshot
287	copr_client.project_proxy.edit(	×
288	project_owner,
289	project_name,
290	additional_repos=[
291	"copr://tstellar/fedora-clang-default-cc",
292	f"copr://@fedora-llvm-team/{snapshot_project_name}",
293	],
294	)
295
296	buildopts = {	×
297	"background": True,
298	}
299	logging.info("Rebuilding", len(pkgs), "packages")	×
300	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
301	default_commitish = "f41"	×
302	for p in pkgs:	×
303	logging.info("Rebuild", p)	×
304	try:	×
305	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
306	build_info = koji_session.getBuild(build["build_id"])	×
307	commitish = build_info["source"].split("#")[1]	×
308	except:	×
309	logging.warn(	×
310	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
311	)
312	commitish = default_commitish	×
313
314	copr_client.build_proxy.create_from_distgit(	×
315	project_owner, project_name, p, commitish, buildopts=buildopts
316	)
317
318
319	def select_snapshot_project(	1✔
320	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
321	) -> str \| None:
322	project_owner = "@fedora-llvm-team"	×
323	for i in range(max_lookback_days):	×
324	chroots = set()	×
325	day = datetime.date.today() - datetime.timedelta(days=i)	×
326	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
327	logging.info("Trying:", project_name)	×
328	try:	×
329	p = copr_client.project_proxy.get(project_owner, project_name)	×
330	if not p:	×
331	continue	×
332	pkgs = copr_client.build_proxy.get_list(	×
333	project_owner, project_name, "llvm", status="succeeded"
334	)
335	for pkg in pkgs:	×
336	chroots.update(pkg["chroots"])	×
337
338	logging.info(project_name, chroots)	×
339	if all(t in chroots for t in target_chroots):	×
340	logging.info("PASS", project_name)	×
341	return project_name	×
342	except:	×
343	continue	×
344	logging.warn("FAIL")	×
345	return None	×
346
347
348	def create_new_project(	1✔
349	project_owner: str,
350	project_name: str,
351	copr_client: copr.v3.Client,
352	target_chroots: list[str],
353	):
354	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
355	for c in target_chroots:	×
356	copr_client.project_chroot_proxy.edit(	×
357	project_owner,
358	project_name,
359	c,
360	additional_packages=["fedora-clang-default-cc"],
361	with_opts=["toolchain_clang", "clang_lto"],
362	)
363
364	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
NEW 365	m = re.search("[0-9]+$", project_name)	×
NEW 366	if not m:	×
NEW 367	raise Exception(f"Invalid project name: {project_name}")	×
NEW 368	return datetime.datetime.fromisoformat(m.group(0)).date()	×
369
370
371	def find_midpoint_project(	1✔
372	copr_client: copr.v3.Client,
373	good: str,
374	bad: str,
375	chroot: str
376	):
NEW 377	good_date = extract_date_from_project(good)	×
NEW 378	bad_date = extract_date_from_project(bad)	×
NEW 379	days = (bad_date - good_date).days	×
NEW 380	mid_date = good_date + datetime.timedelta(days=days / 2)	×
NEW 381	increment = 0	×
NEW 382	while mid_date != good_date and mid_date != bad_date:	×
NEW 383	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
NEW 384	owner = mid_project.split('/')[0]	×
NEW 385	project = mid_project.split('/')[1]	×
NEW 386	try:	×
NEW 387	for builds in copr_client.build_proxy.get_list(owner, project, 'llvm', "succeeded"):	×
NEW 388	if chroot in builds['chroots']:	×
NEW 389	return mid_project	×
NEW 390	except:	×
NEW 391	pass	×
392
NEW 393	increment = increment * -1	×
NEW 394	if increment < 0:	×
NEW 395	increment -= 1	×
396	else:
NEW 397	increment += 1	×
NEW 398	mid_date += datetime.timedelta(days=increment)	×
399
NEW 400	return good	×
401
402
403	def main():	1✔
404
405	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
406	parser = argparse.ArgumentParser()	×
407	parser.add_argument(	×
408	"command",
409	type=str,
410	choices=[
411	"rebuild",
412	"get-regressions",
413	"get-snapshot-date",
414	"rebuild-in-progress",
415	"bisect"
416	],
417	)
418	parser.add_argument(	×
419	"--start-date", type=str, help="Any ISO date format is accepted"
420	)
NEW 421	parser.add_argument(	×
422	"--chroot", type=str
423	)
424	parser.add_argument(	×
425	"--good", type=str
426	)
UNCOV 427	parser.add_argument(	×
428	"--bad", type=str
429	)
430
431	args = parser.parse_args()	×
432	copr_client = copr.v3.Client.create_from_config_file()	×
433
UNCOV 434	os_name = "fedora-41"	×
435	clang_version = "21"	×
436	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
437	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
438	project_owner = "@fedora-llvm-team"	×
439	project_name = f"{os_name}-clang-{clang_version}"	×
440
441	if args.command == "rebuild":	×
UNCOV 442	exclusions = get_exclusions()	×
UNCOV 443	pkgs = get_pkgs(exclusions)	×
444	print(pkgs)	×
445	try:	×
446	copr_client.project_proxy.get(project_owner, project_name)	×
447	copr_pkgs = CoprPkg.get_packages_from_copr(	×
448	project_owner, project_name, copr_client
449	)
450	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
451	except:	×
UNCOV 452	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
UNCOV 453	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
454	start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)	×
UNCOV 455	elif args.command == "get-regressions":	×
UNCOV 456	start_time = datetime.datetime.fromisoformat(args.start_date)	×
457	copr_pkgs = CoprPkg.get_packages_from_copr(	×
458	project_owner, project_name, copr_client
459	)
460	pkg_failures = get_monthly_rebuild_regressions(	×
461	project_owner, project_name, start_time, copr_pkgs
462	)
463	get_chroot_results(pkg_failures, copr_client)	×
464	# Delete attributes we don't need to print
465	for p in pkg_failures:	×
466	for k in ["fail_id", "chroots"]:	×
467	del p[k]	×
468
UNCOV 469	print(json.dumps(pkg_failures))	×
470	elif args.command == "get-snapshot-date":	×
471	project = copr_client.project_proxy.get(project_owner, project_name)	×
472	for repo in project["additional_repos"]:	×
473	match = re.match(	×
474	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
475	)
UNCOV 476	if match:	×
477	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
478	return	×
479	elif args.command == "rebuild-in-progress":	×
480	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
481	"packages"
482	]:
NEW 483	for c in pkg["chroots"]:	×
UNCOV 484	build = CoprBuild(pkg["chroots"][c])	×
UNCOV 485	if build.is_in_progress():	×
UNCOV 486	sys.exit(0)	×
487	sys.exit(1)	×
UNCOV 488	elif args.command == "bisect":	×
UNCOV 489	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
490
491
492	if __name__ == "__main__":	1✔
UNCOV 493	main()	×

fedora-llvm-team / llvm-snapshots / 14312286031

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous