14848474876

Committed 05 May 2025 11:40PM UTC coverage: 58.877% (+0.05%) from 58.829%

Build # 14848474876

Build Type

Pull #1384

github

Committed by

web-flow

Commit Message

Merge a6a1be0e3 into 2f4beadd5

Pull Request Pull Request #1384: rebuilder: Start using rawhide chroots

Run Details

1 of 12 new or added lines in 1 file covered. (8.33%)

1 existing line in 1 file now uncovered.

1237 of 2101 relevant lines covered (58.88%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

36.13

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
import unittest
from typing import Any

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
class CoprBuild(Munch):  # type: ignore[misc]
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
class CoprPkg(Munch):  # type: ignore[misc]
    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild | None:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(
        self, project_owner: str, project_name: str
    ) -> dict[str, Any] | None:
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        latest = self.latest
        if latest is not None:
            return {
                "name": self.name,
                "fail_id": latest.id,
                "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
                "chroots": latest.chroots,
            }
        return None

    @property
    def latest(self) -> CoprBuild | None:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild | None:
        return self.get_build("latest_succeeded")


def load_tests(
    loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
) -> unittest.TestSuite:
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    standard_tests.addTests(doctest.DocTestSuite())
    return standard_tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[str]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest is not None and p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> list[dict[str, Any] | None]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(
    pkgs: list[dict[str, Any] | None], copr_client: copr.v3.Client
) -> None:
    for p in pkgs:
        if p is None:
            continue
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def build_pkg(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkg,
    koji_server = "https://koji.fedoraproject.org/kojihub",
    default_commitish = 'rawhide',
    build_tag = 'f43',
    distgit = 'fedora',
    chroots = None
):

    buildopts = {
        "background": True,
        "chroots": chroots
    }

    koji_session = koji.ClientSession(koji_server)
    try:
        build = koji_session.getLatestBuilds(tag=build_tag, package=pkg)[0]
        build_info = koji_session.getBuild(build["build_id"])
        commitish = build_info["source"].split("#")[1]
    except:
        logging.warn(
            "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
        )
        commitish = default_commitish

    copr_client.build_proxy.create_from_distgit(
        project_owner, project_name, pkg, commitish, buildopts=buildopts, distgit=distgit
    )


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
    chroots : list[str]
) -> None:
    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    logging.info("Rebuilding", len(pkgs), "packages")
    for p in pkgs:
        build_pkg(project_owner, project_name, copr_client, p, chroots = chroots)


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:  # noqa: E722
            continue
    logging.warning("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
) -> None:
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def extract_date_from_project(project_name: str) -> datetime.date:
    m = re.search("[0-9]+$", project_name)
    if not m:
        raise Exception(f"Invalid project name: {project_name}")
    return datetime.datetime.fromisoformat(m.group(0)).date()


def find_midpoint_project(
    copr_client: copr.v3.Client, good: str, bad: str, chroot: str
) -> str:
    good_date = extract_date_from_project(good)
    bad_date = extract_date_from_project(bad)
    days = (bad_date - good_date).days
    mid_date = good_date + datetime.timedelta(days=days / 2)
    increment = 0
    while mid_date != good_date and mid_date != bad_date:
        mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)
        owner = mid_project.split("/")[0]
        project = mid_project.split("/")[1]
        try:
            for builds in copr_client.build_proxy.get_list(
                owner, project, "llvm", "succeeded"
            ):
                if chroot in builds["chroots"]:
                    return mid_project
        except:  # noqa: E722
            pass

        increment = increment * -1
        if increment < 0:
            increment -= 1
        else:
            increment += 1
        mid_date += datetime.timedelta(days=increment)

    return good


def main() -> None:
    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
            "bisect",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )
    parser.add_argument("--chroot", type=str)
    parser.add_argument("--good", type=str)
    parser.add_argument("--bad", type=str)

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-rawhide"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = "clang-monthly-fedora-rebuild"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:  # noqa: E722
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        if snapshot_project is not None:
            start_rebuild(
                project_owner, project_name, copr_client, pkgs, snapshot_project, target_chroots
            )
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(list(pkg_failures), copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            if p is None:
                continue
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)
    elif args.command == "bisect":
        print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	import unittest	1✔
8	from typing import Any	1✔
9
10	import copr.v3	1✔
11	import dnf	1✔
12	import hawkey	1✔
13	import koji	1✔
14	from munch import Munch	1✔
15
16
17	def is_tier0_package(pkg: str) -> bool:	1✔
18	return pkg in [	1✔
19	"dotnet6.0",
20	"dotnet7.0",
21	"dotnet8.0",
22	"dotnet9.0",
23	"qemu-kvm", # RHEL name
24	"qemu", # Fedora name
25	"golang",
26	"wasi-lbc",
27	]
28
29
30	# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
31	class CoprBuild(Munch): # type: ignore[misc]	1✔
32	pass	1✔
33
34	def is_in_progress(self) -> bool:	1✔
35	return self.state not in [	1✔
36	"succeeded",
37	"forked",
38	"skipped",
39	"failed",
40	"canceled",
41	]
42
43
44	# In order to remove the type: ignore[misc] check for this ticket: see https://github.com/Infinidat/munch/issues/84
45	class CoprPkg(Munch): # type: ignore[misc]	1✔
46	@classmethod	1✔
47	def get_packages_from_copr(	1✔
48	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
49	) -> list["CoprPkg"]:
50	return [	×
51	CoprPkg(p)
52	for p in copr_client.package_proxy.get_list(
53	project_owner,
54	project_name,
55	with_latest_succeeded_build=True,
56	with_latest_build=True,
57	)
58	]
59
60	def get_build(self, name: str) -> CoprBuild \| None:	1✔
61	if "builds" not in self:	1✔
62	return None	×
63	if name not in self.builds:	1✔
64	return None	×
65	build = self.builds[name]	1✔
66	if not build:	1✔
67	return None	1✔
68	return CoprBuild(build)	1✔
69
70	def get_regression_info(	1✔
71	self, project_owner: str, project_name: str
72	) -> dict[str, Any] \| None:
73	owner_url = project_owner	1✔
74	if owner_url[0] == "@":	1✔
75	owner_url = f"g/{owner_url[1:]}"	1✔
76	latest = self.latest	1✔
77	if latest is not None:	1✔
78	return {	1✔
79	"name": self.name,
80	"fail_id": latest.id,
81	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{latest.id}/",
82	"chroots": latest.chroots,
83	}
84	return None	×
85
86	@property	1✔
87	def latest(self) -> CoprBuild \| None:	1✔
88	return self.get_build("latest")	1✔
89
90	@property	1✔
91	def latest_succeeded(self) -> CoprBuild \| None:	1✔
92	return self.get_build("latest_succeeded")	1✔
93
94
95	def load_tests(	1✔
96	loader: unittest.TestLoader, standard_tests: unittest.TestSuite, pattern: str
97	) -> unittest.TestSuite:
98	"""We want unittest to pick up all of our doctests
99
100	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
101	See https://stackoverflow.com/a/27171468
102	"""
103	import doctest	×
104
105	standard_tests.addTests(doctest.DocTestSuite())	×
106	return standard_tests	×
107
108
109	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
110	"""Filters out LLVM packages and returns the rest.
111
112	Args:
113	pkgs (set[str]): List of package names
114
115	Returns:
116	set[str]: List of package names without LLVM packages
117
118	Example:
119
120	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
121	>>> filtered=list(filter_llvm_pkgs(pkgs))
122	>>> filtered.sort()
123	>>> print(filtered)
124	['firefox', 'libreoffice']
125
126	"""
127	llvm_pkgs = {	1✔
128	"llvm",
129	"clang",
130	"llvm-bolt",
131	"libomp",
132	"compiler-rt",
133	"lld",
134	"lldb",
135	"polly",
136	"libcxx",
137	"libclc",
138	"flang",
139	"mlir",
140	}
141	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
142	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
143
144
145	def get_exclusions() -> set[str]:	1✔
146	"""
147	This returns a list of packages we don't want to test.
148	"""
149	return set()	×
150
151
152	def get_pkgs(exclusions: set[str]) -> set[str]:	1✔
153	base = dnf.Base()	×
154	conf = base.conf	×
155	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
156	base.repos.add_new_repo(	×
157	f"{c}-source",
158	conf,
159	baseurl=[
160	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
161	],
162	)
163	repos = base.repos.get_matching("*")	×
164	repos.disable()	×
165	repos = base.repos.get_matching("-source")	×
166	repos.enable()	×
167
168	base.fill_sack()	×
169	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
170	q = q.available()	×
171	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
172	pkgs = [p.name for p in list(q)]	×
173	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
174
175
176	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
177	"""Returns the list of packages that should be built in the next rebuild.
178	It will select all the packages that built successfully during the last
179	rebuild.
180
181	Args:
182	pkgs (set[str]): A list of every package that should be considered for
183	the rebuild.
184	copr_pkgs (list[dist]): A list containing the latest build results from
185	the COPR project.
186
187	Returns:
188	set[str]: List of packages that should be rebuilt.
189
190	Example:
191
192	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
193	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
194	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
195	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
196	>>> pkgs = { "b", "c", "d"}
197	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
198	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
199	>>> print(rebuild_pkgs)
200	{'d'}
201	"""
202
203	for p in copr_pkgs:	1✔
204	if p.name not in pkgs:	1✔
205	continue	1✔
206	# Always build tier0 packges.
207	if is_tier0_package(p.name):	1✔
208	continue	×
209	if not p.latest_succeeded:	1✔
210	pkgs.discard(p.name)	1✔
211	continue	1✔
212	if p.latest is not None and p.latest.id != p.latest_succeeded.id:	1✔
213	pkgs.discard(p.name)	1✔
214	return pkgs	1✔
215
216
217	def get_monthly_rebuild_regressions(	1✔
218	project_owner: str,
219	project_name: str,
220	start_time: datetime.datetime,
221	copr_pkgs: list[CoprPkg],
222	) -> list[dict[str, Any] \| None]:
223	"""Returns the list of packages that failed to build in the most recent
224	rebuild, but built successfully in the previous rebuild.
225
226	Args:
227	start_time (datetime.datetime): The start time of the most recent mass
228	rebuild. This needs to be a time
229	before the most recent mass rebuild
230	and after the previous one.
231	copr_pkgs (list[dict]): List of built packages for the COPR project.
232
233	Returns:
234	set[str]: List of packages that regressed in the most recent rebuilt.
235
236	Example:
237
238	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
239	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
240	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
241	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
242	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
243	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
244	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
245	>>> project_owner = "@fedora-llvm-team"
246	>>> project_name = "fedora41-clang-20"
247	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
248	>>> print(regressions)
249	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
250
251	"""
252	pkgs = []	1✔
253	for p in copr_pkgs:	1✔
254	if not p.latest:	1✔
255	continue	×
256
257	# Don't report regressions if there are still builds in progress
258	if p.latest.is_in_progress():	1✔
259	continue	1✔
260
261	if not p.latest_succeeded:	1✔
262	if is_tier0_package(p.name):	1✔
263	pkgs.append(p.get_regression_info(project_owner, project_name))	×
264	continue	1✔
265	if p.latest.id == p.latest_succeeded.id:	1✔
266	continue	1✔
267	# latest is a successful build, but this doesn't mean it failed.
268	# It could be in progress.
269	if p.latest.state != "failed":	1✔
270	continue	1✔
271	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
272	continue	1✔
273	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
274	return pkgs	1✔
275
276
277	def get_chroot_results(	1✔
278	pkgs: list[dict[str, Any] \| None], copr_client: copr.v3.Client
279	) -> None:
280	for p in pkgs:	×
281	if p is None:	×
282	continue	×
283	p["failed_chroots"] = []	×
284	for c in p["chroots"]:	×
285	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
286	if result["state"] == "failed":	×
287	p["failed_chroots"].append(c)	×
288
289
290	def build_pkg(	1✔
291	project_owner: str,
292	project_name: str,
293	copr_client: copr.v3.Client,
294	pkg,
295	koji_server = "https://koji.fedoraproject.org/kojihub",
296	default_commitish = 'rawhide',
297	build_tag = 'f43',
298	distgit = 'fedora',
299	chroots = None
300	):
301
NEW 302	buildopts = {	×
303	"background": True,
304	"chroots": chroots
305	}
306
NEW 307	koji_session = koji.ClientSession(koji_server)	×
NEW 308	try:	×
NEW 309	build = koji_session.getLatestBuilds(tag=build_tag, package=pkg)[0]	×
NEW 310	build_info = koji_session.getBuild(build["build_id"])	×
NEW 311	commitish = build_info["source"].split("#")[1]	×
NEW 312	except:	×
NEW 313	logging.warn(	×
314	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
315	)
NEW 316	commitish = default_commitish	×
317
NEW 318	copr_client.build_proxy.create_from_distgit(	×
319	project_owner, project_name, pkg, commitish, buildopts=buildopts, distgit=distgit
320	)
321
322
323	def start_rebuild(	1✔
324	project_owner: str,
325	project_name: str,
326	copr_client: copr.v3.Client,
327	pkgs: set[str],
328	snapshot_project_name: str,
329	chroots : list[str]
330	) -> None:
331	print("START", pkgs, "END")	×
332	# Update the rebuild project to use the latest snapshot
333	copr_client.project_proxy.edit(	×
334	project_owner,
335	project_name,
336	additional_repos=[
337	"copr://tstellar/fedora-clang-default-cc",
338	f"copr://@fedora-llvm-team/{snapshot_project_name}",
339	],
340	)
341
342	logging.info("Rebuilding", len(pkgs), "packages")	×
UNCOV 343	for p in pkgs:	×
NEW 344	build_pkg(project_owner, project_name, copr_client, p, chroots = chroots)	×
345
346
347	def select_snapshot_project(	1✔
348	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
349	) -> str \| None:
350	project_owner = "@fedora-llvm-team"	×
351	for i in range(max_lookback_days):	×
352	chroots = set()	×
353	day = datetime.date.today() - datetime.timedelta(days=i)	×
354	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
355	logging.info("Trying:", project_name)	×
356	try:	×
357	p = copr_client.project_proxy.get(project_owner, project_name)	×
358	if not p:	×
359	continue	×
360	pkgs = copr_client.build_proxy.get_list(	×
361	project_owner, project_name, "llvm", status="succeeded"
362	)
363	for pkg in pkgs:	×
364	chroots.update(pkg["chroots"])	×
365
366	logging.info(project_name, chroots)	×
367	if all(t in chroots for t in target_chroots):	×
368	logging.info("PASS", project_name)	×
369	return project_name	×
370	except: # noqa: E722	×
371	continue	×
372	logging.warning("FAIL")	×
373	return None	×
374
375
376	def create_new_project(	1✔
377	project_owner: str,
378	project_name: str,
379	copr_client: copr.v3.Client,
380	target_chroots: list[str],
381	) -> None:
382	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
383	for c in target_chroots:	×
384	copr_client.project_chroot_proxy.edit(	×
385	project_owner,
386	project_name,
387	c,
388	additional_packages=["fedora-clang-default-cc"],
389	with_opts=["toolchain_clang", "clang_lto"],
390	)
391
392
393	def extract_date_from_project(project_name: str) -> datetime.date:	1✔
394	m = re.search("[0-9]+$", project_name)	×
395	if not m:	×
396	raise Exception(f"Invalid project name: {project_name}")	×
397	return datetime.datetime.fromisoformat(m.group(0)).date()	×
398
399
400	def find_midpoint_project(	1✔
401	copr_client: copr.v3.Client, good: str, bad: str, chroot: str
402	) -> str:
403	good_date = extract_date_from_project(good)	×
404	bad_date = extract_date_from_project(bad)	×
405	days = (bad_date - good_date).days	×
406	mid_date = good_date + datetime.timedelta(days=days / 2)	×
407	increment = 0	×
408	while mid_date != good_date and mid_date != bad_date:	×
409	mid_project = re.sub("[0-9]+$", mid_date.strftime("%Y%m%d"), good)	×
410	owner = mid_project.split("/")[0]	×
411	project = mid_project.split("/")[1]	×
412	try:	×
413	for builds in copr_client.build_proxy.get_list(	×
414	owner, project, "llvm", "succeeded"
415	):
416	if chroot in builds["chroots"]:	×
417	return mid_project	×
418	except: # noqa: E722	×
419	pass	×
420
421	increment = increment * -1	×
422	if increment < 0:	×
423	increment -= 1	×
424	else:
425	increment += 1	×
426	mid_date += datetime.timedelta(days=increment)	×
427
428	return good	×
429
430
431	def main() -> None:	1✔
432	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
433	parser = argparse.ArgumentParser()	×
434	parser.add_argument(	×
435	"command",
436	type=str,
437	choices=[
438	"rebuild",
439	"get-regressions",
440	"get-snapshot-date",
441	"rebuild-in-progress",
442	"bisect",
443	],
444	)
445	parser.add_argument(	×
446	"--start-date", type=str, help="Any ISO date format is accepted"
447	)
448	parser.add_argument("--chroot", type=str)	×
449	parser.add_argument("--good", type=str)	×
450	parser.add_argument("--bad", type=str)	×
451
452	args = parser.parse_args()	×
453	copr_client = copr.v3.Client.create_from_config_file()	×
454
455	os_name = "fedora-rawhide"	×
456	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
457	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
458	project_owner = "@fedora-llvm-team"	×
459	project_name = "clang-monthly-fedora-rebuild"	×
460
461	if args.command == "rebuild":	×
462	exclusions = get_exclusions()	×
463	pkgs = get_pkgs(exclusions)	×
464	print(pkgs)	×
465	try:	×
466	copr_client.project_proxy.get(project_owner, project_name)	×
467	copr_pkgs = CoprPkg.get_packages_from_copr(	×
468	project_owner, project_name, copr_client
469	)
470	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
471	except: # noqa: E722	×
472	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
473	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
474	if snapshot_project is not None:	×
475	start_rebuild(	×
476	project_owner, project_name, copr_client, pkgs, snapshot_project, target_chroots
477	)
478	elif args.command == "get-regressions":	×
479	start_time = datetime.datetime.fromisoformat(args.start_date)	×
480	copr_pkgs = CoprPkg.get_packages_from_copr(	×
481	project_owner, project_name, copr_client
482	)
483	pkg_failures = get_monthly_rebuild_regressions(	×
484	project_owner, project_name, start_time, copr_pkgs
485	)
486	get_chroot_results(list(pkg_failures), copr_client)	×
487	# Delete attributes we don't need to print
488	for p in pkg_failures:	×
489	if p is None:	×
490	continue	×
491	for k in ["fail_id", "chroots"]:	×
492	del p[k]	×
493
494	print(json.dumps(pkg_failures))	×
495	elif args.command == "get-snapshot-date":	×
496	project = copr_client.project_proxy.get(project_owner, project_name)	×
497	for repo in project["additional_repos"]:	×
498	match = re.match(	×
499	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
500	)
501	if match:	×
502	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
503	return	×
504	elif args.command == "rebuild-in-progress":	×
505	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
506	"packages"
507	]:
508	for c in pkg["chroots"]:	×
509	build = CoprBuild(pkg["chroots"][c])	×
510	if build.is_in_progress():	×
511	sys.exit(0)	×
512	sys.exit(1)	×
513	elif args.command == "bisect":	×
514	print(find_midpoint_project(copr_client, args.good, args.bad, args.chroot))	×
515
516
517	if __name__ == "__main__":	1✔
518	main()	×

fedora-llvm-team / llvm-snapshots / 14848474876

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous