• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

fedora-llvm-team / llvm-snapshots / 8639054350

10 Apr 2024 11:19PM UTC coverage: 86.941% (-5.1%) from 92.045%
8639054350

push

github

kwk
update comment body as fast as we can

0 of 5 new or added lines in 1 file covered. (0.0%)

57 existing lines in 1 file now uncovered.

972 of 1118 relevant lines covered (86.94%)

0.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

22.62
/snapshot_manager/snapshot_manager/snapshot_manager.py
1
"""
2
SnapshotManager
3
"""
4

5
import datetime
1✔
6
import logging
1✔
7
import os
1✔
8
import re
1✔
9

10
import github.Issue
1✔
11

12
import snapshot_manager.build_status as build_status
1✔
13
import snapshot_manager.config as config
1✔
14
import snapshot_manager.copr_util as copr_util
1✔
15
import snapshot_manager.github_graphql as ghgql
1✔
16
import snapshot_manager.github_util as github_util
1✔
17
import snapshot_manager.testing_farm_util as tf
1✔
18
import snapshot_manager.util as util
1✔
19

20

21
class SnapshotManager:
1✔
22

23
    def __init__(self, config: config.Config = config.Config()):
1✔
24
        self.config = config
×
25
        self.copr = copr_util.CoprClient(config=config)
×
26
        self.github = github_util.GithubClient(config=config)
×
27

28
    def check_todays_builds(self):
1✔
29
        """This method is driven from the config settings"""
30
        issue, _ = self.github.create_or_get_todays_github_issue(
31
            maintainer_handle=self.config.maintainer_handle,
32
            creator=self.config.creator_handle,
33
        )
34
        # if issue.state == "closed":
35
        #     logging.info(
36
        #         f"Issue {issue.html_url} was already closed. Not doing anything."
37
        #     )
38
        #     return
39

40
        all_chroots = self.copr.get_copr_chroots()
41

42
        logging.info("Get build states from copr")
43
        states = self.copr.get_build_states_from_copr_monitor(
44
            copr_ownername=self.config.copr_ownername,
45
            copr_projectname=self.config.copr_projectname,
46
        )
47

48
        logging.info("Filter states by chroot of interest")
49
        states = [
50
            state for state in states if state.chroot in self.copr.get_copr_chroots()
51
        ]
52

53
        logging.info("Augment the states with information from the build logs")
54
        states = [state.augment_with_error() for state in states]
55

56
        comment_body = issue.body
57

58
        logging.info("Add a build matrix")
59
        build_status_matrix = build_status.markdown_build_status_matrix(
60
            chroots=all_chroots,
61
            packages=self.config.packages,
62
            build_states=states,
63
        )
64

65
        logging.info("Get ordered list of errors to the issue's body comment")
66
        errors = build_status.list_only_errors(states=states)
67

68
        logging.info("Recover testing-farm requests")
69
        requests = tf.TestingFarmRequest.parse(comment_body)
70
        if requests is None:
71
            requests = dict()
72

73
        # Migrate recovered requests without build IDs.
74
        # Just assign the build IDs of the current chroot respectively.
75
        for chroot in requests:
76
            if requests[chroot].copr_build_ids == []:
77
                logging.info(
78
                    f"Migrating request ID {requests[chroot].request_id} to get copr build IDs"
79
                )
80
                requests[chroot].copr_build_ids = [
81
                    state.build_id for state in states if state.chroot == chroot
82
                ]
83

84
        # Immediately update issue comment and maybe later we update it again:
85
        logging.info("Update issue comment body")
86
        comment_body = f"""
NEW
87
{self.github.initial_comment}
×
NEW
88
{build_status_matrix}
×
NEW
89
{tf.TestingFarmRequest.dict_to_html_comment(requests)}
×
NEW
90
"""
×
NEW
91
        issue.edit(body=comment_body)
×
92

UNCOV
93
        logging.info("Filter testing-farm requests by chroot of interest")
×
UNCOV
94
        new_requests = dict()
×
UNCOV
95
        for chroot in requests:
×
UNCOV
96
            if chroot in self.copr.get_copr_chroots():
×
UNCOV
97
                new_requests[chroot] = requests[chroot]
×
UNCOV
98
        requests = new_requests
×
99

100
        # Hide/Unhide all unused/used chroot error comments if they belong to a chroot we don't care about
UNCOV
101
        for comment in issue.get_comments():
×
UNCOV
102
            logging.info("checking for <!--ERRORS_FOR_CHROOT/")
×
UNCOV
103
            match = re.search(r"<!--ERRORS_FOR_CHROOT/(.*)-->", comment.body)
×
UNCOV
104
            if match:
×
UNCOV
105
                chroot = match.group(1)
×
UNCOV
106
                logging.info(f"found match: chroot={chroot}")
×
UNCOV
107
                if chroot not in self.copr.get_copr_chroots():
×
UNCOV
108
                    logging.info("minimizing comment")
×
UNCOV
109
                    self.github.minimize_comment_as_outdated(comment)
×
UNCOV
110
                else:
×
UNCOV
111
                    logging.info("unminimizing comment")
×
UNCOV
112
                    self.github.unminimize_comment(comment)
×
113

114
        # logging.info(f"Update the issue comment body")
115
        # # See https://github.com/fedora-llvm-team/llvm-snapshots/issues/205#issuecomment-1902057639
116
        # max_length = 65536
117
        # logging.info(f"Checking for maximum length of comment body: {max_length}")
118
        # if len(comment_body) >= max_length:
119
        #     logging.info(
120
        #         f"Github only allows {max_length} characters on a comment body and we have reached {len(comment_body)} characters."
121
        #     )
122

UNCOV
123
        self.handle_labels(issue=issue, all_chroots=all_chroots, errors=errors)
×
124

UNCOV
125
        failed_test_cases: list[tf.FailedTestCase] = []
×
126

UNCOV
127
        for chroot in all_chroots:
×
128
            # Create or update a comment for each chroot that has errors and render
129
            errors_for_this_chroot = [
130
                error for error in errors if error.chroot == chroot
131
            ]
UNCOV
132
            marker = f"<!--ERRORS_FOR_CHROOT/{chroot}-->"
×
UNCOV
133
            if errors_for_this_chroot is not None and len(errors_for_this_chroot) > 0:
×
134
                comment = self.github.create_or_update_comment(
135
                    issue=issue,
136
                    marker=marker,
137
                    comment_body=f"""{marker}
138
<h3>Errors found in Copr builds on <code>{chroot}</code></h3>
139
{build_status.render_as_markdown(errors_for_this_chroot)}
140
""",
141
                )
142
                build_status_matrix = build_status_matrix.replace(
143
                    chroot,
144
                    f'{chroot}<br /> :x: <a href="{comment.html_url}">Copr build(s) failed</a>',
145
                )
UNCOV
146
            else:
×
147
                # Hide any outdated comments
UNCOV
148
                comment = self.github.get_comment(issue=issue, marker=marker)
×
UNCOV
149
                if comment is not None:
×
UNCOV
150
                    self.github.minimize_comment_as_outdated(comment)
×
151

UNCOV
152
        for chroot in all_chroots:
×
153
            # Check if we can ignore the chroot because it is not supported by testing-farm
UNCOV
154
            if not tf.TestingFarmRequest.is_chroot_supported(chroot):
×
155
                # see https://docs.testing-farm.io/Testing%20Farm/0.1/test-environment.html#_supported_architectures
156
                logging.debug(
157
                    f"Ignoring chroot {chroot} because testing-farm doesn't support it."
158
                )
UNCOV
159
                continue
×
160

UNCOV
161
            in_testing = f"{self.config.label_prefix_in_testing}{chroot}"
×
UNCOV
162
            tested_on = f"{self.config.label_prefix_tested_on}{chroot}"
×
UNCOV
163
            failed_on = f"{self.config.label_prefix_failed_on}{chroot}"
×
164

165
            # Gather build IDs associated with this chroot.
166
            # We'll attach them a new testing-farm request, and for a recovered
167
            # request we'll check if they still match the current ones.
168
            current_copr_build_ids = [
169
                state.build_id for state in states if state.chroot == chroot
170
            ]
171

172
            # Check if we need to invalidate a recovered testing-farm requests.
173
            # Background: It can be that we have old testing-farm request IDs in the issue comment.
174
            # But if a package was re-build and failed, the old request ID for that chroot is invalid.
175
            # To compensate for this scenario that we saw on April 1st 2024 btw., we're gonna
176
            # delete any request that has a different set of Copr build IDs associated with it.
UNCOV
177
            if chroot in requests:
×
UNCOV
178
                recovered_request = requests[chroot]
×
UNCOV
179
                if set(recovered_request.copr_build_ids) != set(current_copr_build_ids):
×
180
                    logging.info(
181
                        f"Recovered request ({recovered_request.request_id}) invalid (build IDs changed):\\nRecovered: {recovered_request.copr_build_ids}\\nCurrent: {current_copr_build_ids}"
182
                    )
183
                    self.github.flip_test_label(
184
                        issue=issue, chroot=chroot, new_label=None
185
                    )
UNCOV
186
                    del requests[chroot]
×
187

UNCOV
188
            logging.info(f"Check if all builds in chroot {chroot} have succeeded")
×
189
            builds_succeeded = self.copr.has_all_good_builds(
190
                copr_ownername=self.config.copr_ownername,
191
                copr_projectname=self.config.copr_projectname,
192
                required_chroots=[chroot],
193
                required_packages=self.config.packages,
194
                states=states,
195
            )
196

UNCOV
197
            if not builds_succeeded:
×
UNCOV
198
                continue
×
199

UNCOV
200
            logging.info(f"All builds in chroot {chroot} have succeeded!")
×
201

202
            # Check for current status of testing-farm request
UNCOV
203
            if chroot in requests:
×
UNCOV
204
                request = requests[chroot]
×
UNCOV
205
                watch_result, artifacts_url = request.watch()
×
206

UNCOV
207
                html = tf.render_html(request, watch_result, artifacts_url)
×
208
                build_status_matrix = build_status_matrix.replace(
209
                    chroot,
210
                    f"{chroot}<br />{html}",
211
                )
212

213
                logging.info(
214
                    f"Chroot {chroot} testing-farm watch result: {watch_result} (URL: {artifacts_url})"
215
                )
216

UNCOV
217
                if watch_result.is_error:
×
218
                    # Fetch all failed test cases for this request
219
                    failed_test_cases.extend(
220
                        request.fetch_failed_test_cases(
221
                            artifacts_url_origin=artifacts_url
222
                        )
223
                    )
UNCOV
224
                    self.github.flip_test_label(issue, chroot, failed_on)
×
UNCOV
225
                elif watch_result == tf.TestingFarmWatchResult.TESTS_PASSED:
×
UNCOV
226
                    self.github.flip_test_label(issue, chroot, tested_on)
×
UNCOV
227
                else:
×
UNCOV
228
                    self.github.flip_test_label(issue, chroot, in_testing)
×
UNCOV
229
            else:
×
UNCOV
230
                logging.info(f"Starting tests for chroot {chroot}")
×
231
                request = tf.TestingFarmRequest.make(
232
                    chroot=chroot,
233
                    config=self.config,
234
                    issue=issue,
235
                    copr_build_ids=current_copr_build_ids,
236
                )
UNCOV
237
                logging.info(f"Request ID: {request.request_id}")
×
UNCOV
238
                requests[chroot] = request
×
UNCOV
239
                self.github.flip_test_label(issue, chroot, in_testing)
×
240

241
            # Create or update a comment for testing-farm results display
UNCOV
242
            if len(failed_test_cases) > 0:
×
243
                self.github.create_or_update_comment(
244
                    issue=issue,
245
                    marker=tf.results_html_comment(),
246
                    comment_body=tf.FailedTestCase.render_list_as_markdown(
247
                        failed_test_cases
248
                    ),
249
                )
250

251
        logging.info("Constructing issue comment body")
252
        comment_body = f"""
253
{self.github.initial_comment}
254
{build_status_matrix}
255
{tf.TestingFarmRequest.dict_to_html_comment(requests)}
256
"""
257
        issue.edit(body=comment_body)
258

259
        logging.info("Checking if issue can be closed")
260
        # issue.update()
261
        tested_chroot_labels = [
262
            label.name
263
            for label in issue.labels
264
            if label.name.startswith("{self.config.label_prefix_tested_on}")
265
        ]
266
        required_chroot_abels = [
267
            "{self.config.label_prefix_tested_on}{chroot}"
268
            for chroot in all_chroots
269
            if tf.TestingFarmRequest.is_chroot_supported(chroot)
270
        ]
271
        if set(tested_chroot_labels) == set(required_chroot_abels):
272
            msg = f"@{self.config.maintainer_handle}, all required packages have been successfully built and tested on all required chroots. We'll close this issue for you now as completed. Congratulations!"
273
            logging.info(msg)
274
            issue.create_comment(body=msg)
275
            issue.edit(state="closed", state_reason="completed")
276
            # TODO(kwk): Promotion of issue goes here.
277
        else:
278
            logging.info("Cannot close issue yet.")
279

280
        logging.info(f"Updated today's issue: {issue.html_url}")
281

282
    def handle_labels(
1✔
283
        self,
284
        issue: github.Issue.Issue,
1✔
285
        all_chroots: list[str],
1✔
286
        errors: build_status.BuildStateList,
1✔
287
    ):
288
        logging.info("Gather labels based on the errors we've found")
289
        error_labels = list({f"error/{err.err_cause}" for err in errors})
290
        project_labels = list({f"project/{err.package_name}" for err in errors})
291
        os_labels = list({f"os/{err.os}" for err in errors})
292
        arch_labels = list({f"arch/{err.arch}" for err in errors})
293
        strategy_labels = [f"strategy/{self.config.build_strategy}"]
294
        other_labels: list[str] = []
295
        if errors is None and len(errors) > 0:
296
            other_labels.append("broken_snapshot_detected")
297

298
        logging.info("Create labels")
299
        self.github.create_labels(
300
            labels=["broken_snapshot_detected"], color="F46696", prefix=""
301
        )
302
        self.github.create_labels_for_error_causes(error_labels)
303
        self.github.create_labels_for_oses(os_labels)
304
        self.github.create_labels_for_projects(project_labels)
305
        self.github.create_labels_for_archs(arch_labels)
306
        self.github.create_labels_for_strategies(strategy_labels)
307
        self.github.create_labels_for_in_testing(all_chroots)
308
        self.github.create_labels_for_tested_on(all_chroots)
309
        self.github.create_labels_for_failed_on(all_chroots)
310

311
        # Remove old labels from issue if they no longer apply. This is great
312
        # for restarted builds for example to make all builds green and be able
313
        # to promote this snapshot.
314

315
        labels_to_be_removed: list[str] = []
316
        old_error_labels = self.github.get_error_label_names_on_issue(issue=issue)
317
        old_project_labels = self.github.get_project_label_names_on_issue(issue=issue)
318
        old_arch_labels = self.github.get_arch_label_names_on_issue(issue=issue)
319

320
        labels_to_be_removed.extend(set(old_error_labels) - set(error_labels))
321
        labels_to_be_removed.extend(set(old_project_labels) - set(project_labels))
322
        labels_to_be_removed.extend(set(old_arch_labels) - set(arch_labels))
323

324
        for label in labels_to_be_removed:
325
            logging.info(f"Removing label that no longer applies: {label}")
326
            issue.remove_from_labels(label=label)
327

328
        # Labels must be added or removed manually in order to not remove manually added labels :/
329
        labels_to_add = (
330
            error_labels
331
            + project_labels
332
            + os_labels
333
            + arch_labels
334
            + strategy_labels
335
            + other_labels
336
        )
337
        logging.info(f"Adding label: {labels_to_add}")
338
        issue.add_to_labels(*labels_to_add)
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc