14848547666

Committed 05 May 2025 11:46PM UTC coverage: 58.877% (+0.05%) from 58.829%

Build # 14848547666

Build Type

Pull #1384

github

Committed by

web-flow

Commit Message

Merge 5b30990a4 into 2f4beadd5

Pull Request Pull Request #1384: rebuilder: Start using rawhide chroots

Run Details

1 of 12 new or added lines in 1 file covered. (8.33%)

1 existing line in 1 file now uncovered.

1237 of 2101 relevant lines covered (58.88%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

36.13

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
import unittest
from typing import Any

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
class CoprBuild(Munch):  # type: ignore[misc]
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
class CoprPkg(Munch):  # type: ignore[misc]
    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild | None:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(
        self, project_owner: str, project_name: str
    ) -> dict[str, Any] | None:
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        latest = self.latest
        if latest is not None:
            return {
                "name": self.name,
                "fail_id": latest.id,
                "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
                "chroots": latest.chroots,
            }
        return None

    @property
    def latest(self) -> CoprBuild | None:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild | None:
        return self.get_build("latest_succeeded")


def load_tests(
    loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
) -> unittest.TestSuite:
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    standard_tests.addTests(doctest.DocTestSuite())
    return standard_tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[str]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest is not None and p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> list[dict[str, Any] | None]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(
    pkgs: list[dict[str, Any] | None], copr_client: copr.v3.Client
) -> None:
    for p in pkgs:
        if p is None:
            continue
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def build_pkg(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkg,
    koji_server="https://koji.fedoraproject.org/kojihub",
    default_commitish="rawhide",
    build_tag="f43",
    distgit="fedora",
    chroots=None
):

    buildopts = {"background": True, "chroots": chroots}

    koji_session = koji.ClientSession(koji_server)
    try:
        build = koji_session.getLatestBuilds(tag=build_tag, package=pkg)[0]
        build_info = koji_session.getBuild(build["build_id"])
        commitish = build_info["source"].split("#")[1]
    except:  # noqa: E722
        logging.warn(
            "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
        )
        commitish = default_commitish

    copr_client.build_proxy.create_from_distgit(
        project_owner,
        project_name,
        pkg,
        commitish,
        buildopts=buildopts,
        distgit=distgit
    )


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
    chroots: list[str]
) -> None:
    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    logging.info("Rebuilding", len(pkgs), "packages")
    for p in pkgs:
        build_pkg(project_owner, project_name, copr_client, p, chroots=chroots)


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:  # noqa: E722
            continue
    logging.warning("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
) -> None:
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
) -> str:
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:  # noqa: E722
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main() -> None:
    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-rawhide"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = "clang-monthly-fedora-rebuild"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:  # noqa: E722
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        if snapshot_project is not None:
            start_rebuild(
                project_owner,
                project_name,
                copr_client,
                pkgs,
                snapshot_project,
                target_chroots
            )
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(list(pkg_failures), copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            if p is None:
                continue
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	import unittest	1✔
8	from typing import Any	1✔
9
10	import copr.v3	1✔
11	import dnf	1✔
12	import hawkey	1✔
13	import koji	1✔
14	from munch import Munch	1✔
15
16
17	def is_tier0_package(pkg: str) -> bool:	1✔
18	return pkg in [	1✔
19	"dotnet6.0",
20	"dotnet7.0",
21	"dotnet8.0",
22	"dotnet9.0",
23	"qemu-kvm", # RHEL name
24	"qemu", # Fedora name
25	"golang",
26	"wasi-lbc",
27	]
28
29
30	# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
31	class CoprBuild(Munch): # type: ignore[misc]	1✔
32	pass	1✔
33
34	def is_in_progress(self) -> bool:	1✔
35	return self.state not in [	1✔
36	"succeeded",
37	"forked",
38	"skipped",
39	"failed",
40	"canceled",
41	]
42
43
44	# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
45	class CoprPkg(Munch): # type: ignore[misc]	1✔
46	@classmethod	1✔
47	def get_packages_from_copr(	1✔
48	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
49	) -> list["CoprPkg"]:
50	return [	×
51	CoprPkg(p)
52	for p in copr_client.package_proxy.get_list(
53	project_owner,
54	project_name,
55	with_latest_succeeded_build=True,
56	with_latest_build=True,
57	)
58	]
59
60	def get_build(self, name: str) -> CoprBuild \| None:	1✔
61	if "builds" not in self:	1✔
62	return None	×
63	if name not in self.builds:	1✔
64	return None	×
65	build = self.builds[name]	1✔
66	if not build:	1✔
67	return None	1✔
68	return CoprBuild(build)	1✔
69
70	def get_regression_info(	1✔
71	self, project_owner: str, project_name: str
72	) -> dict[str, Any] \| None:
73	owner_url = project_owner	1✔
74	if owner_url[0] == "@":	1✔
75	owner_url = f"g/{owner_url[1:]}"	1✔
76	latest = self.latest	1✔
77	if latest is not None:	1✔
78	return {	1✔
79	"name": self.name,
80	"fail_id": latest.id,
81	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
82	"chroots": latest.chroots,
83	}
84	return None	×
85
86	@property	1✔
87	def latest(self) -> CoprBuild \| None:	1✔
88	return self.get_build("latest")	1✔
89
90	@property	1✔
91	def latest_succeeded(self) -> CoprBuild \| None:	1✔
92	return self.get_build("latest_succeeded")	1✔
93
94
95	def load_tests(	1✔
96	loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
97	) -> unittest.TestSuite:
98	"""We want unittest to pick up all of our doctests
99
100	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
101	See https://stackoverflow.com/a/27171468
102	"""
103	import doctest	×
104
105	standard_tests.addTests(doctest.DocTestSuite())	×
106	return standard_tests	×
107
108
109	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
110	"""Filters out LLVM packages and returns the rest.
111
112	Args:
113	pkgs (set[str]): List of package names
114
115	Returns:
116	set[str]: List of package names without LLVM packages
117
118	Example:
119
120	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
121	>>> filtered=list(filter_llvm_pkgs(pkgs))
122	>>> filtered.sort()
123	>>> print(filtered)
124	['firefox', 'libreoffice']
125
126	"""
127	llvm_pkgs = {	1✔
128	"llvm",
129	"clang",
130	"llvm-bolt",
131	"libomp",
132	"compiler-rt",
133	"lld",
134	"lldb",
135	"polly",
136	"libcxx",
137	"libclc",
138	"flang",
139	"mlir",
140	}
141	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
142	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
143
144
145	def get_exclusions() -> set[str]:	1✔
146	"""
147	This returns a list of packages we don't want to test.
148	"""
149	return set()	×
150
151
152	def get_pkgs(exclusions: set[str]) -> set[str]:	1✔
153	base = dnf.Base()	×
154	conf = base.conf	×
155	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
156	base.repos.add_new_repo(	×
157	f"{c}-source",
158	conf,
159	baseurl=[
160	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
161	],
162	)
163	repos = base.repos.get_matching("*")	×
164	repos.disable()	×
165	repos = base.repos.get_matching("-source")	×
166	repos.enable()	×
167
168	base.fill_sack()	×
169	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
170	q = q.available()	×
171	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
172	pkgs = [p.name for p in list(q)]	×
173	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
174
175
176	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
177	"""Returns the list of packages that should be built in the next rebuild.
178	It will select all the packages that built successfully during the last
179	rebuild.
180
181	Args:
182	pkgs (set[str]): A list of every package that should be considered for
183	the rebuild.
184	copr_pkgs (list[dist]): A list containing the latest build results from
185	the COPR project.
186
187	Returns:
188	set[str]: List of packages that should be rebuilt.
189
190	Example:
191
192	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
193	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
194	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
195	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
196	>>> pkgs = { "b", "c", "d"}
197	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
198	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
199	>>> print(rebuild_pkgs)
200	{'d'}
201	"""
202
203	for p in copr_pkgs:	1✔
204	if p.name not in pkgs:	1✔
205	continue	1✔
206	# Always build tier0 packges.
207	if is_tier0_package(p.name):	1✔
208	continue	×
209	if not p.latest_succeeded:	1✔
210	pkgs.discard(p.name)	1✔
211	continue	1✔
212	if p.latest is not None and p.latest.id != p.latest_succeeded.id:	1✔
213	pkgs.discard(p.name)	1✔
214	return pkgs	1✔
215
216
217	def get_monthly_rebuild_regressions(	1✔
218	project_owner: str,
219	project_name: str,
220	start_time: datetime.datetime,
221	copr_pkgs: list[CoprPkg],
222	) -> list[dict[str, Any] \| None]:
223	"""Returns the list of packages that failed to build in the most recent
224	rebuild, but built successfully in the previous rebuild.
225
226	Args:
227	start_time (datetime.datetime): The start time of the most recent mass
228	rebuild. This needs to be a time
229	before the most recent mass rebuild
230	and after the previous one.
231	copr_pkgs (list[dict]): List of built packages for the COPR project.
232
233	Returns:
234	set[str]: List of packages that regressed in the most recent rebuilt.
235
236	Example:
237
238	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
239	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
240	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
241	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
242	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
243	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
244	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
245	>>> project_owner = "@fedora-llvm-team"
246	>>> project_name = "fedora41-clang-20"
247	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
248	>>> print(regressions)
249	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
250
251	"""
252	pkgs = []	1✔
253	for p in copr_pkgs:	1✔
254	if not p.latest:	1✔
255	continue	×
256
257	# Don't report regressions if there are still builds in progress
258	if p.latest.is_in_progress():	1✔
259	continue	1✔
260
261	if not p.latest_succeeded:	1✔
262	if is_tier0_package(p.name):	1✔
263	pkgs.append(p.get_regression_info(project_owner, project_name))	×
264	continue	1✔
265	if p.latest.id == p.latest_succeeded.id:	1✔
266	continue	1✔
267	# latest is a successful build, but this doesn't mean it failed.
268	# It could be in progress.
269	if p.latest.state != "failed":	1✔
270	continue	1✔
271	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
272	continue	1✔
273	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
274	return pkgs	1✔
275
276
277	def get_chroot_results(	1✔
278	pkgs: list[dict[str, Any] \| None], copr_client: copr.v3.Client
279	) -> None:
280	for p in pkgs:	×
281	if p is None:	×
282	continue	×
283	p["failed_chroots"] = []	×
284	for c in p["chroots"]:	×
285	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
286	if result["state"] == "failed":	×
287	p["failed_chroots"].append(c)	×
288
289
290	def build_pkg(	1✔
291	project_owner: str,
292	project_name: str,
293	copr_client: copr.v3.Client,
294	pkg,
295	koji_server="https://koji.fedoraproject.org/kojihub",
296	default_commitish="rawhide",
297	build_tag="f43",
298	distgit="fedora",
299	chroots=None
300	):
301
NEW 302	buildopts = {"background": True, "chroots": chroots}	×
303
NEW 304	koji_session = koji.ClientSession(koji_server)	×
NEW 305	try:	×
NEW 306	build = koji_session.getLatestBuilds(tag=build_tag, package=pkg)[0]	×
NEW 307	build_info = koji_session.getBuild(build["build_id"])	×
NEW 308	commitish = build_info["source"].split("#")[1]	×
NEW 309	except: # noqa: E722	×
NEW 310	logging.warn(	×
311	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
312	)
NEW 313	commitish = default_commitish	×
314
NEW 315	copr_client.build_proxy.create_from_distgit(	×
316	project_owner,
317	project_name,
318	pkg,
319	commitish,
320	buildopts=buildopts,
321	distgit=distgit
322	)
323
324
325	def start_rebuild(	1✔
326	project_owner: str,
327	project_name: str,
328	copr_client: copr.v3.Client,
329	pkgs: set[str],
330	snapshot_project_name: str,
331	chroots: list[str]
332	) -> None:
333	print("START", pkgs, "END")	×
334	# Update the rebuild project to use the latest snapshot
335	copr_client.project_proxy.edit(	×
336	project_owner,
337	project_name,
338	additional_repos=[
339	"copr://tstellar/fedora-clang-default-cc",
340	f"copr://@fedora-llvm-team/{snapshot_project_name}",
341	],
342	)
343
344	logging.info("Rebuilding", len(pkgs), "packages")	×
UNCOV 345	for p in pkgs:	×
NEW 346	build_pkg(project_owner, project_name, copr_client, p, chroots=chroots)	×
347
348
349	def select_snapshot_project(	1✔
350	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
351	) -> str \| None:
352	project_owner = "@fedora-llvm-team"	×
353	for i in range(max_lookback_days):	×
354	chroots = set()	×
355	day = datetime.date.today() - datetime.timedelta(days=i)	×
356	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
357	logging.info("Trying:", project_name)	×
358	try:	×
359	p = copr_client.project_proxy.get(project_owner, project_name)	×
360	if not p:	×
361	continue	×
362	pkgs = copr_client.build_proxy.get_list(	×
363	project_owner, project_name, "llvm", status="succeeded"
364	)
365	for pkg in pkgs:	×
366	chroots.update(pkg["chroots"])	×
367
368	logging.info(project_name, chroots)	×
369	if all(t in chroots for t in target_chroots):	×
370	logging.info("PASS", project_name)	×
371	return project_name	×
372	except: # noqa: E722	×
373	continue	×
374	logging.warning("FAIL")	×
375	return None	×
376
377
378	def create_new_project(	1✔
379	project_owner: str,
380	project_name: str,
381	copr_client: copr.v3.Client,
382	target_chroots: list[str],
383	) -> None:
384	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
385	for c in target_chroots:	×
386	copr_client.project_chroot_proxy.edit(	×
387	project_owner,
388	project_name,
389	c,
390	additional_packages=["fedora-clang-default-cc"],
391	with_opts=["toolchain_clang", "clang_lto"],
392	)
393
394
395	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
396	m = re.search("[0-9]+$", project_name)	×
397	if not m:	×
398	raise Exception(f"Invalid project name: {project_name}")	×
399	return datetime.datetime.fromisoformat(m.group(0)).date()	×
400
401
402	def find_midpoint_project(	1✔
403	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
404	) -> str:
405	good_date = extract_date_from_project(good)	×
406	bad_date = extract_date_from_project(bad)	×
407	days = (bad_date - good_date).days	×
408	mid_date = good_date + datetime.timedelta(days=days / 2)	×
409	increment = 0	×
410	while mid_date != good_date and mid_date != bad_date:	×
411	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
412	owner = mid_project.split("/")[0]	×
413	project = mid_project.split("/")[1]	×
414	try:	×
415	for builds in copr_client.build_proxy.get_list(	×
416	owner, project, "llvm", "succeeded"
417	):
418	if chroot in builds["chroots"]:	×
419	return mid_project	×
420	except: # noqa: E722	×
421	pass	×
422
423	increment = increment * -1	×
424	if increment < 0:	×
425	increment -= 1	×
426	else:
427	increment += 1	×
428	mid_date += datetime.timedelta(days=increment)	×
429
430	return good	×
431
432
433	def main() -> None:	1✔
434	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
435	parser = argparse.ArgumentParser()	×
436	parser.add_argument(	×
437	"command",
438	type=str,
439	choices=[
440	"rebuild",
441	"get-regressions",
442	"get-snapshot-date",
443	"rebuild-in-progress",
444	"bisect",
445	],
446	)
447	parser.add_argument(	×
448	"--start-date", type=str, help="Any ISO date format is accepted"
449	)
450	parser.add_argument("--chroot", type=str)	×
451	parser.add_argument("--good", type=str)	×
452	parser.add_argument("--bad", type=str)	×
453
454	args = parser.parse_args()	×
455	copr_client = copr.v3.Client.create_from_config_file()	×
456
457	os_name = "fedora-rawhide"	×
458	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
459	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
460	project_owner = "@fedora-llvm-team"	×
461	project_name = "clang-monthly-fedora-rebuild"	×
462
463	if args.command == "rebuild":	×
464	exclusions = get_exclusions()	×
465	pkgs = get_pkgs(exclusions)	×
466	print(pkgs)	×
467	try:	×
468	copr_client.project_proxy.get(project_owner, project_name)	×
469	copr_pkgs = CoprPkg.get_packages_from_copr(	×
470	project_owner, project_name, copr_client
471	)
472	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
473	except: # noqa: E722	×
474	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
475	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
476	if snapshot_project is not None:	×
477	start_rebuild(	×
478	project_owner,
479	project_name,
480	copr_client,
481	pkgs,
482	snapshot_project,
483	target_chroots
484	)
485	elif args.command == "get-regressions":	×
486	start_time = datetime.datetime.fromisoformat(args.start_date)	×
487	copr_pkgs = CoprPkg.get_packages_from_copr(	×
488	project_owner, project_name, copr_client
489	)
490	pkg_failures = get_monthly_rebuild_regressions(	×
491	project_owner, project_name, start_time, copr_pkgs
492	)
493	get_chroot_results(list(pkg_failures), copr_client)	×
494	# Delete attributes we don't need to print
495	for p in pkg_failures:	×
496	if p is None:	×
497	continue	×
498	for k in ["fail_id", "chroots"]:	×
499	del p[k]	×
500
501	print(json.dumps(pkg_failures))	×
502	elif args.command == "get-snapshot-date":	×
503	project = copr_client.project_proxy.get(project_owner, project_name)	×
504	for repo in project["additional_repos"]:	×
505	match = re.match(	×
506	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
507	)
508	if match:	×
509	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
510	return	×
511	elif args.command == "rebuild-in-progress":	×
512	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
513	"packages"
514	]:
515	for c in pkg["chroots"]:	×
516	build = CoprBuild(pkg["chroots"][c])	×
517	if build.is_in_progress():	×
518	sys.exit(0)	×
519	sys.exit(1)	×
520	elif args.command == "bisect":	×
521	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
522
523
524	if __name__ == "__main__":	1✔
525	main()	×

fedora-llvm-team / llvm-snapshots / 14848547666

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous