13027618083

Committed 29 Jan 2025 08:47AM UTC coverage: 38.322% (+0.01%) from 38.31%

Build # 13027618083

Build Type

push

github

Committed by

kwk

Commit Message

[debug] Add logging to github client setup

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

87 existing lines in 2 files now uncovered.

9759 of 25466 relevant lines covered (38.32%)

0.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

42.6

/scripts/rebuilder.py

import argparse
import datetime
import json
import logging
import re
import sys
from typing import Set

import copr.v3
import dnf
import hawkey
from munch import Munch


class CoprBuild(Munch):
    pass

    def is_in_progress(self) -> bool:
        return self.state not in [
            "succeeded",
            "forked",
            "skipped",
            "failed",
            "canceled",
        ]


class CoprPkg(Munch):

    @classmethod
    def get_packages_from_copr(
        cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
    ) -> list["CoprPkg"]:
        return [
            CoprPkg(p)
            for p in copr_client.package_proxy.get_list(
                project_owner,
                project_name,
                with_latest_succeeded_build=True,
                with_latest_build=True,
            )
        ]

    def get_build(self, name: str) -> CoprBuild:
        if "builds" not in self:
            return None
        if name not in self.builds:
            return None
        build = self.builds[name]
        if not build:
            return None
        return CoprBuild(build)

    @property
    def latest(self) -> CoprBuild:
        return self.get_build("latest")

    @property
    def latest_succeeded(self) -> CoprBuild:
        return self.get_build("latest_succeeded")


def load_tests(loader, tests, ignore):
    """We want unittest to pick up all of our doctests

    See https://docs.python.org/3/library/unittest.html#load-tests-protocol
    See https://stackoverflow.com/a/27171468
    """
    import doctest

    tests.addTests(doctest.DocTestSuite())
    return tests


def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:
    """Filters out LLVM packages and returns the rest.

    Args:
        pkgs (set[str]): List of package names

    Returns:
        set[str]: List of package names without LLVM packages

    Example:

    >>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
    >>> filtered=list(filter_llvm_pkgs(pkgs))
    >>> filtered.sort()
    >>> print(filtered)
    ['firefox', 'libreoffice']

    """
    llvm_pkgs = {
        "llvm",
        "clang",
        "llvm-bolt",
        "libomp",
        "compiler-rt",
        "lld",
        "lldb",
        "polly",
        "libcxx",
        "libclc",
        "flang",
        "mlir",
    }
    llvm_pkg_pattern = rf"({'|'.join(llvm_pkgs)})[0-9]*$"
    return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}


def get_exclusions() -> set[str]:
    """
    This returns a list of packages we don't want to test.
    """
    return set()


def get_pkgs(exclusions: set[str]) -> set[set]:
    base = dnf.Base()
    conf = base.conf
    for c in "AppStream", "BaseOS", "CRB", "Extras":
        base.repos.add_new_repo(
            f"{c}-source",
            conf,
            baseurl=[
                f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
            ],
        )
    repos = base.repos.get_matching("*")
    repos.disable()
    repos = base.repos.get_matching("*-source*")
    repos.enable()

    base.fill_sack()
    q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)
    q = q.available()
    q = q.filter(requires=["clang", "gcc", "gcc-c++"])
    pkgs = [p.name for p in list(q)]
    return filter_llvm_pkgs(set(pkgs)) - exclusions


def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:
    """Returns the list of packages that should be built in the next rebuild.
        It will select all the packages that built successfully during the last
        rebuild.

    Args:
        pkgs (set[str]): A list of every package that should be considered for
                        the rebuild.
        copr_pkgs (list[dist]): A list containing the latest build results from
                                the COPR project.

    Returns:
        set[str]: List of packages that should be rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
    >>> pkgs = { "b", "c", "d"}
    >>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
    >>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
    >>> print(rebuild_pkgs)
    {'d'}
    """

    for p in copr_pkgs:
        if p.name not in pkgs:
            continue
        if not p.latest_succeeded:
            pkgs.discard(p.name)
            continue
        if p.latest.id != p.latest_succeeded.id:
            pkgs.discard(p.name)
    return pkgs


def get_monthly_rebuild_regressions(
    project_owner: str,
    project_name: str,
    start_time: datetime.datetime,
    copr_pkgs: list[CoprPkg],
) -> set[str]:
    """Returns the list of packages that failed to build in the most recent
       rebuild, but built successfully in the previous rebuild.

    Args:
        start_time (datetime.datetime): The start time of the most recent mass
                                        rebuild.  This needs to be a time
                                        before the most recent mass rebuild
                                        and after the previous one.
        copr_pkgs (list[dict]): List of built packages for the COPR project.

    Returns:
        set[str]: List of packages that regressed in the most recent rebuilt.

    Example:

    >>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321 } , "latest_succeeded" : None } }
    >>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : None } }
    >>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } }
    >>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } }
    >>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1 } , "latest_succeeded" : { "id" : 1 } } }
    >>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } }
    >>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
    >>> project_owner = "@fedora-llvm-team"
    >>> project_name = "fedora41-clang-20"
    >>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
    >>> print(regressions)
    [{'name': 'f', 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/'}]

    """
    pkgs = []
    for p in copr_pkgs:
        if not p.latest:
            continue

        # Don't report regressions if there are still builds in progress
        if p.latest.is_in_progress():
            continue

        if not p.latest_succeeded:
            continue
        if p.latest.id == p.latest_succeeded.id:
            continue
        # latest is a successful build, but this doesn't mean it failed.
        # It could be in progress.
        if p.latest.state != "failed":
            continue
        if int(p.latest.submitted_on) < start_time.timestamp():
            continue
        owner_url = project_owner
        if owner_url[0] == "@":
            owner_url = f"g/{owner_url[1:]}"
        pkgs.append(
            {
                "name": p.name,
                "url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{p.latest.id}/",
            }
        )
    return pkgs


def start_rebuild(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    pkgs: set[str],
    snapshot_project_name: str,
):

    print("START", pkgs, "END")
    # Update the rebuild project to use the latest snapshot
    copr_client.project_proxy.edit(
        project_owner,
        project_name,
        additional_repos=[
            "copr://tstellar/fedora-clang-default-cc",
            f"copr://@fedora-llvm-team/{snapshot_project_name}",
        ],
    )

    buildopts = {
        "background": True,
    }
    logging.info("Rebuilding", len(pkgs), "packages")
    for p in pkgs:
        logging.info("Rebuild", p)
        copr_client.build_proxy.create_from_distgit(
            project_owner, project_name, p, "f41", buildopts=buildopts
        )


def select_snapshot_project(
    copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
) -> str | None:
    project_owner = "@fedora-llvm-team"
    for i in range(max_lookback_days):
        chroots = set()
        day = datetime.date.today() - datetime.timedelta(days=i)
        project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")
        logging.info("Trying:", project_name)
        try:
            p = copr_client.project_proxy.get(project_owner, project_name)
            if not p:
                continue
            pkgs = copr_client.build_proxy.get_list(
                project_owner, project_name, "llvm", status="succeeded"
            )
            for pkg in pkgs:
                chroots.update(pkg["chroots"])

            logging.info(project_name, chroots)
            if all(t in chroots for t in target_chroots):
                logging.info("PASS", project_name)
                return project_name
        except:
            continue
    logging.warn("FAIL")
    return None


def create_new_project(
    project_owner: str,
    project_name: str,
    copr_client: copr.v3.Client,
    target_chroots: list[str],
):
    copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)
    for c in target_chroots:
        copr_client.project_chroot_proxy.edit(
            project_owner,
            project_name,
            c,
            additional_packages=["fedora-clang-default-cc"],
            with_opts=["toolchain_clang", "clang_lto"],
        )


def main():

    logging.basicConfig(filename="rebuilder.log", level=logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "command",
        type=str,
        choices=[
            "rebuild",
            "get-regressions",
            "get-snapshot-date",
            "rebuild-in-progress",
        ],
    )
    parser.add_argument(
        "--start-date", type=str, help="Any ISO date format is accepted"
    )

    args = parser.parse_args()
    copr_client = copr.v3.Client.create_from_config_file()

    os_name = "fedora-41"
    clang_version = "20"
    target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]
    target_chroots = [f"{os_name}-{a}" for a in target_arches]
    project_owner = "@fedora-llvm-team"
    project_name = f"{os_name}-clang-{clang_version}"

    if args.command == "rebuild":
        exclusions = get_exclusions()
        pkgs = get_pkgs(exclusions)
        print(pkgs)
        try:
            copr_client.project_proxy.get(project_owner, project_name)
            copr_pkgs = CoprPkg.get_packages_from_copr(
                project_owner, project_name, copr_client
            )
            pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
        except:
            create_new_project(project_owner, project_name, copr_client, target_chroots)
        snapshot_project = select_snapshot_project(copr_client, target_chroots)
        start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)
    elif args.command == "get-regressions":
        start_time = datetime.datetime.fromisoformat(args.start_date)
        copr_pkgs = CoprPkg.get_packages_from_copr(
            project_owner, project_name, copr_client
        )
        pkg_failures = get_monthly_rebuild_regressions(
            project_owner, project_name, start_time, copr_pkgs
        )
        print(json.dumps(pkg_failures))
    elif args.command == "get-snapshot-date":
        project = copr_client.project_proxy.get(project_owner, project_name)
        for repo in project["additional_repos"]:
            match = re.match(
                r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
            )
            if match:
                print(datetime.datetime.fromisoformat(match.group(1)).isoformat())
                return
    elif args.command == "rebuild-in-progress":
        for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[
            "packages"
        ]:
            for c in pkg["chroots"]:
                build = CoprBuild(pkg["chroots"][c])
                if build.is_in_progress():
                    sys.exit(0)
        sys.exit(1)


if __name__ == "__main__":
    main()

1	import argparse	1✔
2	import datetime	1✔
3	import json	1✔
4	import logging	1✔
5	import re	1✔
6	import sys	1✔
7	from typing import Set	1✔
8
9	import copr.v3	1✔
10	import dnf	1✔
11	import hawkey	1✔
12	from munch import Munch	1✔
13
14
15	class CoprBuild(Munch):	1✔
16	pass	1✔
17
18	def is_in_progress(self) -> bool:	1✔
19	return self.state not in [	1✔
20	"succeeded",
21	"forked",
22	"skipped",
23	"failed",
24	"canceled",
25	]
26
27
28	class CoprPkg(Munch):	1✔
29
30	@classmethod	1✔
31	def get_packages_from_copr(	1✔
32	cls, project_owner: str, project_name: str, copr_client: copr.v3.Client
33	) -> list["CoprPkg"]:
34	return [	×
35	CoprPkg(p)
36	for p in copr_client.package_proxy.get_list(
37	project_owner,
38	project_name,
39	with_latest_succeeded_build=True,
40	with_latest_build=True,
41	)
42	]
43
44	def get_build(self, name: str) -> CoprBuild:	1✔
45	if "builds" not in self:	1✔
46	return None	×
47	if name not in self.builds:	1✔
48	return None	×
49	build = self.builds[name]	1✔
50	if not build:	1✔
51	return None	1✔
52	return CoprBuild(build)	1✔
53
54	@property	1✔
55	def latest(self) -> CoprBuild:	1✔
56	return self.get_build("latest")	1✔
57
58	@property	1✔
59	def latest_succeeded(self) -> CoprBuild:	1✔
60	return self.get_build("latest_succeeded")	1✔
61
62
63	def load_tests(loader, tests, ignore):	1✔
64	"""We want unittest to pick up all of our doctests
65
66	See https://docs.python.org/3/library/unittest.html#load-tests-protocol
67	See https://stackoverflow.com/a/27171468
68	"""
69	import doctest	×
70
71	tests.addTests(doctest.DocTestSuite())	×
72	return tests	×
73
74
75	def filter_llvm_pkgs(pkgs: set[str]) -> set[str]:	1✔
76	"""Filters out LLVM packages and returns the rest.
77
78	Args:
79	pkgs (set[str]): List of package names
80
81	Returns:
82	set[str]: List of package names without LLVM packages
83
84	Example:
85
86	>>> pkgs={'firefox', 'llvm99', 'libreoffice', 'clang18', 'compiler-rt'}
87	>>> filtered=list(filter_llvm_pkgs(pkgs))
88	>>> filtered.sort()
89	>>> print(filtered)
90	['firefox', 'libreoffice']
91
92	"""
93	llvm_pkgs = {	1✔
94	"llvm",
95	"clang",
96	"llvm-bolt",
97	"libomp",
98	"compiler-rt",
99	"lld",
100	"lldb",
101	"polly",
102	"libcxx",
103	"libclc",
104	"flang",
105	"mlir",
106	}
107	llvm_pkg_pattern = rf"({'\|'.join(llvm_pkgs)})[0-9]*$"	1✔
108	return {pkg for pkg in pkgs if not re.match(llvm_pkg_pattern, pkg)}	1✔
109
110
111	def get_exclusions() -> set[str]:	1✔
112	"""
113	This returns a list of packages we don't want to test.
114	"""
115	return set()	×
116
117
118	def get_pkgs(exclusions: set[str]) -> set[set]:	1✔
119	base = dnf.Base()	×
120	conf = base.conf	×
121	for c in "AppStream", "BaseOS", "CRB", "Extras":	×
122	base.repos.add_new_repo(	×
123	f"{c}-source",
124	conf,
125	baseurl=[
126	f"https://odcs.fedoraproject.org/composes/production/latest-Fedora-ELN/compose/{c}/source/tree/"
127	],
128	)
129	repos = base.repos.get_matching("*")	×
130	repos.disable()	×
131	repos = base.repos.get_matching("-source")	×
132	repos.enable()	×
133
134	base.fill_sack()	×
135	q = base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES)	×
136	q = q.available()	×
137	q = q.filter(requires=["clang", "gcc", "gcc-c++"])	×
138	pkgs = [p.name for p in list(q)]	×
139	return filter_llvm_pkgs(set(pkgs)) - exclusions	×
140
141
142	def get_monthly_rebuild_packages(pkgs: set[str], copr_pkgs: list[CoprPkg]) -> set[str]:	1✔
143	"""Returns the list of packages that should be built in the next rebuild.
144	It will select all the packages that built successfully during the last
145	rebuild.
146
147	Args:
148	pkgs (set[str]): A list of every package that should be considered for
149	the rebuild.
150	copr_pkgs (list[dist]): A list containing the latest build results from
151	the COPR project.
152
153	Returns:
154	set[str]: List of packages that should be rebuilt.
155
156	Example:
157
158	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : { "id" : 1 } } }
159	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1 } , "latest_succeeded" : None } }
160	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 1 } } }
161	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2 } , "latest_succeeded" : { "id" : 2 } } }
162	>>> pkgs = { "b", "c", "d"}
163	>>> copr_pkgs = [CoprPkg(p) for p in [a, b, c, d]]
164	>>> rebuild_pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)
165	>>> print(rebuild_pkgs)
166	{'d'}
167	"""
168
169	for p in copr_pkgs:	1✔
170	if p.name not in pkgs:	1✔
171	continue	1✔
172	if not p.latest_succeeded:	1✔
173	pkgs.discard(p.name)	1✔
174	continue	1✔
175	if p.latest.id != p.latest_succeeded.id:	1✔
176	pkgs.discard(p.name)	1✔
177	return pkgs	1✔
178
179
180	def get_monthly_rebuild_regressions(	1✔
181	project_owner: str,
182	project_name: str,
183	start_time: datetime.datetime,
184	copr_pkgs: list[CoprPkg],
185	) -> set[str]:
186	"""Returns the list of packages that failed to build in the most recent
187	rebuild, but built successfully in the previous rebuild.
188
189	Args:
190	start_time (datetime.datetime): The start time of the most recent mass
191	rebuild. This needs to be a time
192	before the most recent mass rebuild
193	and after the previous one.
194	copr_pkgs (list[dict]): List of built packages for the COPR project.
195
196	Returns:
197	set[str]: List of packages that regressed in the most recent rebuilt.
198
199	Example:
200
201	>>> a = {"name" : "a", "builds" : { "latest" : { "id" : 1, "state" : "running", "submitted_on" : 1731457321 } , "latest_succeeded" : None } }
202	>>> b = {"name" : "b", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : None } }
203	>>> c = {"name" : "c", "builds" : { "latest" : { "id" : 1, "state" : "succeeded", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } }
204	>>> d = {"name" : "d", "builds" : { "latest" : { "id" : 2, "state" : "canceled", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } }
205	>>> e = {"name" : "e", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1 } , "latest_succeeded" : { "id" : 1 } } }
206	>>> f = {"name" : "f", "builds" : { "latest" : { "id" : 2, "state" : "failed", "submitted_on" : 1731457321 } , "latest_succeeded" : { "id" : 1 } } }
207	>>> copr_pkgs= [CoprPkg(p) for p in [ a, b, c, d, e, f ]]
208	>>> project_owner = "@fedora-llvm-team"
209	>>> project_name = "fedora41-clang-20"
210	>>> regressions = get_monthly_rebuild_regressions(project_owner, project_name, datetime.datetime.fromisoformat("2024-11-11"), copr_pkgs)
211	>>> print(regressions)
212	[{'name': 'f', 'url': 'https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/fedora41-clang-20/build/2/'}]
213
214	"""
215	pkgs = []	1✔
216	for p in copr_pkgs:	1✔
217	if not p.latest:	1✔
218	continue	×
219
220	# Don't report regressions if there are still builds in progress
221	if p.latest.is_in_progress():	1✔
222	continue	1✔
223
224	if not p.latest_succeeded:	1✔
225	continue	1✔
226	if p.latest.id == p.latest_succeeded.id:	1✔
227	continue	1✔
228	# latest is a successful build, but this doesn't mean it failed.
229	# It could be in progress.
230	if p.latest.state != "failed":	1✔
231	continue	1✔
232	if int(p.latest.submitted_on) < start_time.timestamp():	1✔
233	continue	1✔
234	owner_url = project_owner	1✔
235	if owner_url[0] == "@":	1✔
236	owner_url = f"g/{owner_url[1:]}"	1✔
237	pkgs.append(	1✔
238	{
239	"name": p.name,
240	"url": f"https://copr.fedorainfracloud.org/coprs/{owner_url}/{project_name}/build/{p.latest.id}/",
241	}
242	)
243	return pkgs	1✔
244
245
246	def start_rebuild(	1✔
247	project_owner: str,
248	project_name: str,
249	copr_client: copr.v3.Client,
250	pkgs: set[str],
251	snapshot_project_name: str,
252	):
253
UNCOV 254	print("START", pkgs, "END")	×
255	# Update the rebuild project to use the latest snapshot
UNCOV 256	copr_client.project_proxy.edit(	×
257	project_owner,
258	project_name,
259	additional_repos=[
260	"copr://tstellar/fedora-clang-default-cc",
261	f"copr://@fedora-llvm-team/{snapshot_project_name}",
262	],
263	)
264
265	buildopts = {	×
266	"background": True,
267	}
268	logging.info("Rebuilding", len(pkgs), "packages")	×
UNCOV 269	for p in pkgs:	×
UNCOV 270	logging.info("Rebuild", p)	×
UNCOV 271	copr_client.build_proxy.create_from_distgit(	×
272	project_owner, project_name, p, "f41", buildopts=buildopts
273	)
274
275
276	def select_snapshot_project(	1✔
277	copr_client: copr.v3.Client, target_chroots: list[str], max_lookback_days: int = 14
278	) -> str \| None:
279	project_owner = "@fedora-llvm-team"	×
280	for i in range(max_lookback_days):	×
281	chroots = set()	×
282	day = datetime.date.today() - datetime.timedelta(days=i)	×
283	project_name = day.strftime("llvm-snapshots-big-merge-%Y%m%d")	×
284	logging.info("Trying:", project_name)	×
285	try:	×
286	p = copr_client.project_proxy.get(project_owner, project_name)	×
UNCOV 287	if not p:	×
UNCOV 288	continue	×
289	pkgs = copr_client.build_proxy.get_list(	×
290	project_owner, project_name, "llvm", status="succeeded"
291	)
292	for pkg in pkgs:	×
293	chroots.update(pkg["chroots"])	×
294
295	logging.info(project_name, chroots)	×
296	if all(t in chroots for t in target_chroots):	×
297	logging.info("PASS", project_name)	×
298	return project_name	×
299	except:	×
UNCOV 300	continue	×
UNCOV 301	logging.warn("FAIL")	×
UNCOV 302	return None	×
303
304
305	def create_new_project(	1✔
306	project_owner: str,
307	project_name: str,
308	copr_client: copr.v3.Client,
309	target_chroots: list[str],
310	):
UNCOV 311	copr_client.project_proxy.add(project_owner, project_name, chroots=target_chroots)	×
UNCOV 312	for c in target_chroots:	×
UNCOV 313	copr_client.project_chroot_proxy.edit(	×
314	project_owner,
315	project_name,
316	c,
317	additional_packages=["fedora-clang-default-cc"],
318	with_opts=["toolchain_clang", "clang_lto"],
319	)
320
321
322	def main():	1✔
323
UNCOV 324	logging.basicConfig(filename="rebuilder.log", level=logging.INFO)	×
UNCOV 325	parser = argparse.ArgumentParser()	×
UNCOV 326	parser.add_argument(	×
327	"command",
328	type=str,
329	choices=[
330	"rebuild",
331	"get-regressions",
332	"get-snapshot-date",
333	"rebuild-in-progress",
334	],
335	)
UNCOV 336	parser.add_argument(	×
337	"--start-date", type=str, help="Any ISO date format is accepted"
338	)
339
340	args = parser.parse_args()	×
341	copr_client = copr.v3.Client.create_from_config_file()	×
342
343	os_name = "fedora-41"	×
344	clang_version = "20"	×
345	target_arches = ["aarch64", "ppc64le", "s390x", "x86_64"]	×
UNCOV 346	target_chroots = [f"{os_name}-{a}" for a in target_arches]	×
347	project_owner = "@fedora-llvm-team"	×
348	project_name = f"{os_name}-clang-{clang_version}"	×
349
350	if args.command == "rebuild":	×
351	exclusions = get_exclusions()	×
352	pkgs = get_pkgs(exclusions)	×
353	print(pkgs)	×
UNCOV 354	try:	×
UNCOV 355	copr_client.project_proxy.get(project_owner, project_name)	×
356	copr_pkgs = CoprPkg.get_packages_from_copr(	×
357	project_owner, project_name, copr_client
358	)
359	pkgs = get_monthly_rebuild_packages(pkgs, copr_pkgs)	×
360	except:	×
361	create_new_project(project_owner, project_name, copr_client, target_chroots)	×
362	snapshot_project = select_snapshot_project(copr_client, target_chroots)	×
363	start_rebuild(project_owner, project_name, copr_client, pkgs, snapshot_project)	×
UNCOV 364	elif args.command == "get-regressions":	×
UNCOV 365	start_time = datetime.datetime.fromisoformat(args.start_date)	×
366	copr_pkgs = CoprPkg.get_packages_from_copr(	×
367	project_owner, project_name, copr_client
368	)
369	pkg_failures = get_monthly_rebuild_regressions(	×
370	project_owner, project_name, start_time, copr_pkgs
371	)
372	print(json.dumps(pkg_failures))	×
373	elif args.command == "get-snapshot-date":	×
UNCOV 374	project = copr_client.project_proxy.get(project_owner, project_name)	×
UNCOV 375	for repo in project["additional_repos"]:	×
376	match = re.match(	×
377	r"copr://@fedora-llvm-team/llvm-snapshots-big-merge-([0-9]+)$", repo
378	)
379	if match:	×
380	print(datetime.datetime.fromisoformat(match.group(1)).isoformat())	×
UNCOV 381	return	×
UNCOV 382	elif args.command == "rebuild-in-progress":	×
383	for pkg in copr_client.monitor_proxy.monitor(project_owner, project_name)[	×
384	"packages"
385	]:
386	for c in pkg["chroots"]:	×
387	build = CoprBuild(pkg["chroots"][c])	×
UNCOV 388	if build.is_in_progress():	×
UNCOV 389	sys.exit(0)	×
UNCOV 390	sys.exit(1)	×
391
392
393	if __name__ == "__main__":	1✔
UNCOV 394	main()	×

fedora-llvm-team / llvm-snapshots / 13027618083

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous