13396366936

Committed 18 Feb 2025 05:18PM UTC coverage: 38.335% (+0.01%) from 38.325%

Build # 13396366936

Build Type

push

github

Committed by

web-flow

Commit Message

rebuilder: Pull from latest tagged build instead of latest dist-git (#1082)

Run Details

1 of 11 new or added lines in 1 file covered. (9.09%)

1 existing line in 1 file now uncovered.

9779 of 25509 relevant lines covered (38.34%)

0.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

40.2

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Set

import copr.v3
import dnf
import hawkey
import koji
from munch import Munch


def is_tier0_package(pkg: str) -> bool:
    return pkg in [
        "dotnet6.0",
        "dotnet7.0",
        "dotnet8.0",
        "dotnet9.0",
        "qemu-kvm",  # RHEL name
        "qemu",  # Fedora name
        "golang",
        "wasi-lbc",
    ]


class CoprBuild(Munch):
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):

    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    def get_regression_info(self, project_owner, project_name):
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        return {
            "name": self.name,
            "fail_id": self.latest.id,
            "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
            "chroots": self.latest.chroots,
        }

    @property
    def latest(self) -> CoprBuild:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[set]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        # Always build tier0 packges.
        if is_tier0_package(p.name):
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> set[str]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            if is_tier0_package(p.name):
                pkgs.append(p.get_regression_info(project_owner, project_name))
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        pkgs.append(p.get_regression_info(project_owner, project_name))
    return pkgs


def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:
    for p in pkgs:
        p["failed_chroots"] = []
        for c in p["chroots"]:
            result = copr_client.build_chroot_proxy.get(p["fail_id"], c)
            if result["state"] == "failed":
                p["failed_chroots"].append(c)


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
):

    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    default_commitish = "f41"
    for p in pkgs:
        logging.info("Rebuild", p)
        try:
            build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]
            build_info = koji_session.getBuild(build["build_id"])
            commitish = build_info["source"].split("#")[1]
        except:
            logging.warn(
                "Could not determine git commit for latest build of {p}.  Defaulting to {default_commitish}."
            )
            commitish = default_commitish

        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, commitish, buildopts=buildopts
        )
        break


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:
            continue
    logging.warn("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
):
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def main():

    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "20"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        get_chroot_results(pkg_failures, copr_client)
        # Delete attributes we don't need to print
        for p in pkg_failures:
            for k in ["fail_id", "chroots"]:
                del p[k]

        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Set	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	import koji	1✔
13	from munch import Munch	1✔
14
15
16	def is_tier0_package(pkg: str) -> bool:	1✔
17	return pkg in [	1✔
18	"dotnet6.0",
19	"dotnet7.0",
20	"dotnet8.0",
21	"dotnet9.0",
22	"qemu-kvm", # RHEL name
23	"qemu", # Fedora name
24	"golang",
25	"wasi-lbc",
26	]
27
28
29	class CoprBuild(Munch):	1✔
30	pass	1✔
31
32	def is_in_progress(self) -> bool:	1✔
33	return self.state not in [	1✔
34	"succeeded",
35	"forked",
36	"skipped",
37	"failed",
38	"canceled",
39	]
40
41
42	class CoprPkg(Munch):	1✔
43
44	@classmethod	1✔
45	def get_packages_from_copr(	1✔
46	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
47	) -> list["CoprPkg"]:
48	return [	×
49	CoprPkg(p)
50	for p in copr_client.package_proxy.get_list(
51	project_owner,
52	project_name,
53	with_latest_succeeded_build=True,
54	with_latest_build=True,
55	)
56	]
57
58	def get_build(self, name: str) -> CoprBuild:	1✔
59	if "builds" not in self:	1✔
60	return None	×
61	if name not in self.builds:	1✔
62	return None	×
63	build = self.builds[name]	1✔
64	if not build:	1✔
65	return None	1✔
66	return CoprBuild(build)	1✔
67
68	def get_regression_info(self, project_owner, project_name):	1✔
69	owner_url = project_owner	1✔
70	if owner_url[0] == "@":	1✔
71	owner_url = f"g/{owner_url[1:]}"	1✔
72	return {	1✔
73	"name": self.name,
74	"fail_id": self.latest.id,
75	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{self.latest.id}/",
76	"chroots": self.latest.chroots,
77	}
78
79	@property	1✔
80	def latest(self) -> CoprBuild:	1✔
81	return self.get_build("latest")	1✔
82
83	@property	1✔
84	def latest_succeeded(self) -> CoprBuild:	1✔
85	return self.get_build("latest_succeeded")	1✔
86
87
88	def load_tests(loader, tests, ignore):	1✔
89	"""We want unittest to pick up all of our doctests
90
91	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
92	See https://stackoverflow.com/a/27171468
93	"""
94	import doctest	×
95
96	tests.addTests(doctest.DocTestSuite())	×
97	return tests	×
98
99
100	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
101	"""Filters out LLVM packages and returns the rest.
102
103	Args:
104	pkgs (set[str]): List of package names
105
106	Returns:
107	set[str]: List of package names without LLVM packages
108
109	Example:
110
111	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
112	>>> filtered=list(filter_llvm_pkgs(pkgs))
113	>>> filtered.sort()
114	>>> print(filtered)
115	['firefox', 'libreoffice']
116
117	"""
118	llvm_pkgs = {	1✔
119	"llvm",
120	"clang",
121	"llvm-bolt",
122	"libomp",
123	"compiler-rt",
124	"lld",
125	"lldb",
126	"polly",
127	"libcxx",
128	"libclc",
129	"flang",
130	"mlir",
131	}
132	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
133	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
134
135
136	def get_exclusions() -> set[str]:	1✔
137	"""
138	This returns a list of packages we don't want to test.
139	"""
140	return set()	×
141
142
143	def get_pkgs(exclusions: set[str]) -> set[set]:	1✔
144	base = dnf.Base()	×
145	conf = base.conf	×
146	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
147	base.repos.add_new_repo(	×
148	f"{c}-source",
149	conf,
150	baseurl=[
151	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
152	],
153	)
154	repos = base.repos.get_matching("*")	×
155	repos.disable()	×
156	repos = base.repos.get_matching("-source")	×
157	repos.enable()	×
158
159	base.fill_sack()	×
160	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
161	q = q.available()	×
162	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
163	pkgs = [p.name for p in list(q)]	×
164	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
165
166
167	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
168	"""Returns the list of packages that should be built in the next rebuild.
169	It will select all the packages that built successfully during the last
170	rebuild.
171
172	Args:
173	pkgs (set[str]): A list of every package that should be considered for
174	the rebuild.
175	copr_pkgs (list[dist]): A list containing the latest build results from
176	the COPR project.
177
178	Returns:
179	set[str]: List of packages that should be rebuilt.
180
181	Example:
182
183	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
184	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
185	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
186	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
187	>>> pkgs = { "b", "c", "d"}
188	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
189	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
190	>>> print(rebuild_pkgs)
191	{'d'}
192	"""
193
194	for p in copr_pkgs:	1✔
195	if p.name not in pkgs:	1✔
196	continue	1✔
197	# Always build tier0 packges.
198	if is_tier0_package(p.name):	1✔
199	continue	×
200	if not p.latest_succeeded:	1✔
201	pkgs.discard(p.name)	1✔
202	continue	1✔
203	if p.latest.id != p.latest_succeeded.id:	1✔
204	pkgs.discard(p.name)	1✔
205	return pkgs	1✔
206
207
208	def get_monthly_rebuild_regressions(	1✔
209	project_owner: str,
210	project_name: str,
211	start_time: datetime.datetime,
212	copr_pkgs: list[CoprPkg],
213	) -> set[str]:
214	"""Returns the list of packages that failed to build in the most recent
215	rebuild, but built successfully in the previous rebuild.
216
217	Args:
218	start_time (datetime.datetime): The start time of the most recent mass
219	rebuild. This needs to be a time
220	before the most recent mass rebuild
221	and after the previous one.
222	copr_pkgs (list[dict]): List of built packages for the COPR project.
223
224	Returns:
225	set[str]: List of packages that regressed in the most recent rebuilt.
226
227	Example:
228
229	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
230	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : None } }
231	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
232	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
233	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1, "chroots" : [] } , "latest_succeeded" : { "id" : 1 } } }
234	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321, "chroots" : ["x86_64", "ppc64le", "s390x", "aarch64"] } , "latest_succeeded" : { "id" : 1 } } }
235	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
236	>>> project_owner = "@fedora-llvm-team"
237	>>> project_name = "fedora41-clang-20"
238	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
239	>>> print(regressions)
240	[{'name': 'f', 'fail_id': 2, 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/', 'chroots': ['x86_64', 'ppc64le', 's390x', 'aarch64']}]
241
242	"""
243	pkgs = []	1✔
244	for p in copr_pkgs:	1✔
245	if not p.latest:	1✔
246	continue	×
247
248	# Don't report regressions if there are still builds in progress
249	if p.latest.is_in_progress():	1✔
250	continue	1✔
251
252	if not p.latest_succeeded:	1✔
253	if is_tier0_package(p.name):	1✔
254	pkgs.append(p.get_regression_info(project_owner, project_name))	×
255	continue	1✔
256	if p.latest.id == p.latest_succeeded.id:	1✔
257	continue	1✔
258	# latest is a successful build, but this doesn't mean it failed.
259	# It could be in progress.
260	if p.latest.state != "failed":	1✔
261	continue	1✔
262	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
263	continue	1✔
264	pkgs.append(p.get_regression_info(project_owner, project_name))	1✔
265	return pkgs	1✔
266
267
268	def get_chroot_results(pkgs: list[dict], copr_client: copr.v3.Client) -> None:	1✔
269	for p in pkgs:	×
270	p["failed_chroots"] = []	×
271	for c in p["chroots"]:	×
272	result = copr_client.build_chroot_proxy.get(p["fail_id"], c)	×
273	if result["state"] == "failed":	×
274	p["failed_chroots"].append(c)	×
275
276
277	def start_rebuild(	1✔
278	project_owner: str,
279	project_name: str,
280	copr_client: copr.v3.Client,
281	pkgs: set[str],
282	snapshot_project_name: str,
283	):
284
285	print("START", pkgs, "END")	×
286	# Update the rebuild project to use the latest snapshot
287	copr_client.project_proxy.edit(	×
288	project_owner,
289	project_name,
290	additional_repos=[
291	"copr://tstellar/fedora-clang-default-cc",
292	f"copr://@fedora-llvm-team/{snapshot_project_name}",
293	],
294	)
295
296	buildopts = {	×
297	"background": True,
298	}
299	logging.info("Rebuilding", len(pkgs), "packages")	×
NEW 300	koji_session = koji.ClientSession("https://koji.fedoraproject.org/kojihub")	×
NEW 301	default_commitish = "f41"	×
302	for p in pkgs:	×
303	logging.info("Rebuild", p)	×
NEW 304	try:	×
NEW 305	build = koji_session.getLatestBuilds(tag="f41-updates", package=p)[0]	×
NEW 306	build_info = koji_session.getBuild(build["build_id"])	×
NEW 307	commitish = build_info["source"].split("#")[1]	×
NEW 308	except:	×
NEW 309	logging.warn(	×
310	"Could not determine git commit for latest build of {p}. Defaulting to {default_commitish}."
311	)
NEW 312	commitish = default_commitish	×
313
UNCOV 314	copr_client.build_proxy.create_from_distgit(	×
315	project_owner, project_name, p, commitish, buildopts=buildopts
316	)
NEW 317	break	×
318
319
320	def select_snapshot_project(	1✔
321	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
322	) -> str \| None:
323	project_owner = "@fedora-llvm-team"	×
324	for i in range(max_lookback_days):	×
325	chroots = set()	×
326	day = datetime.date.today() - datetime.timedelta(days=i)	×
327	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
328	logging.info("Trying:", project_name)	×
329	try:	×
330	p = copr_client.project_proxy.get(project_owner, project_name)	×
331	if not p:	×
332	continue	×
333	pkgs = copr_client.build_proxy.get_list(	×
334	project_owner, project_name, "llvm", status="succeeded"
335	)
336	for pkg in pkgs:	×
337	chroots.update(pkg["chroots"])	×
338
339	logging.info(project_name, chroots)	×
340	if all(t in chroots for t in target_chroots):	×
341	logging.info("PASS", project_name)	×
342	return project_name	×
343	except:	×
344	continue	×
345	logging.warn("FAIL")	×
346	return None	×
347
348
349	def create_new_project(	1✔
350	project_owner: str,
351	project_name: str,
352	copr_client: copr.v3.Client,
353	target_chroots: list[str],
354	):
355	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
356	for c in target_chroots:	×
357	copr_client.project_chroot_proxy.edit(	×
358	project_owner,
359	project_name,
360	c,
361	additional_packages=["fedora-clang-default-cc"],
362	with_opts=["toolchain_clang", "clang_lto"],
363	)
364
365
366	def main():	1✔
367
368	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
369	parser = argparse.ArgumentParser()	×
370	parser.add_argument(	×
371	"command",
372	type=str,
373	choices=[
374	"rebuild",
375	"get-regressions",
376	"get-snapshot-date",
377	"rebuild-in-progress",
378	],
379	)
380	parser.add_argument(	×
381	"--start-date", type=str, help="Any ISO date format is accepted"
382	)
383
384	args = parser.parse_args()	×
385	copr_client = copr.v3.Client.create_from_config_file()	×
386
387	os_name = "fedora-41"	×
388	clang_version = "20"	×
389	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
390	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
391	project_owner = "@fedora-llvm-team"	×
392	project_name = f"{os_name}-clang-{clang_version}"	×
393
394	if args.command == "rebuild":	×
395	exclusions = get_exclusions()	×
396	pkgs = get_pkgs(exclusions)	×
397	print(pkgs)	×
398	try:	×
399	copr_client.project_proxy.get(project_owner, project_name)	×
400	copr_pkgs = CoprPkg.get_packages_from_copr(	×
401	project_owner, project_name, copr_client
402	)
403	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
404	except:	×
405	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
406	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
407	start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)	×
408	elif args.command == "get-regressions":	×
409	start_time = datetime.datetime.fromisoformat(args.start_date)	×
410	copr_pkgs = CoprPkg.get_packages_from_copr(	×
411	project_owner, project_name, copr_client
412	)
413	pkg_failures = get_monthly_rebuild_regressions(	×
414	project_owner, project_name, start_time, copr_pkgs
415	)
416	get_chroot_results(pkg_failures, copr_client)	×
417	# Delete attributes we don't need to print
418	for p in pkg_failures:	×
419	for k in ["fail_id", "chroots"]:	×
420	del p[k]	×
421
422	print(json.dumps(pkg_failures))	×
423	elif args.command == "get-snapshot-date":	×
424	project = copr_client.project_proxy.get(project_owner, project_name)	×
425	for repo in project["additional_repos"]:	×
426	match = re.match(	×
427	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
428	)
429	if match:	×
430	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
431	return	×
432	elif args.command == "rebuild-in-progress":	×
433	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
434	"packages"
435	]:
436	for c in pkg["chroots"]:	×
437	build = CoprBuild(pkg["chroots"][c])	×
438	if build.is_in_progress():	×
439	sys.exit(0)	×
440	sys.exit(1)	×
441
442
443	if __name__ == "__main__":	1✔
444	main()	×

fedora-llvm-team / llvm-snapshots / 13396366936

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous