#5852

Committed 22 May 2026 02:17PM UTC coverage: 23.275% (+1.8%) from 21.433%

Build # #5852

Build Type

push

coveralls-python

Committed by

jgraham

Commit Message

Add web platform features rules

Bugs with the `web-feature` keyword are managed by the bot so that
their lifetimes matches the lifetime of the related
web-feature (specified in the user-story field) and are closed once
the web feature is marked as implemented in Firefox.

This initial change implements the following functionality:
* Automatically handling renames of existing web features
* Removing `web-feature` labels in the user story which don't
correspond to a real web-feature.
* Updating links to external resources on web-features bugs.
* Reopening web-features bugs that are closed when the corresponding
feature isn't marked as supported.
* Closing bugs once the corresponding web-feature is marked as supported.

Coverage Stats

517 of 3146 branches covered (16.43%)

204 of 294 new or added lines in 1 file covered. (69.39%)

1 existing line in 1 file now uncovered.

2229 of 9577 relevant lines covered (23.27%)

0.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.44

/bugbot/rules/web_platform_features.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

# mypy: disallow-untyped-defs
import re
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import Iterator
from dataclasses import dataclass, field
from enum import Enum, IntEnum
from typing import (
    Any,
    Generic,
    Iterable,
    Mapping,
    MutableMapping,
    Optional,
    Sequence,
    TypeVar,
)
from urllib import parse

from google.cloud import bigquery

from bugbot import gcp
from bugbot.bzcleaner import Bug, BzCleaner

Json = None | str | int | float | Sequence["Json"] | Mapping[str, "Json"]


def parse_user_story(
    user_story: str,
) -> Iterator[tuple[str, Optional[str], Optional[str]]]:
    """Parse the user story assuming it's lines of the form key: value.

    If there isn't a colon in the line we simply set value to the full line."""
    user_story_re = re.compile(r"^\s*([^\s]+)\s*:\s*(.*)")
    for line in user_story.splitlines():
        key = None
        value = None
        m = user_story_re.match(line)
        if m is not None:
            maybe_key, maybe_value = m.groups()
            if maybe_value:
                key = maybe_key
                value = maybe_value
        yield line, key, value


class UserStoryChangeType(IntEnum):
    APPEND = 1
    REPLACE = 2
    DELETE = 3


@dataclass(frozen=True)
class UserStoryChange:
    field: str
    type: UserStoryChangeType
    old_value: Optional[str] = None
    new_value: Optional[str] = None


class Resolution(Enum):
    NONE = ""
    FIXED = "FIXED"
    DUPLICATE = "DUPLICATE"


@dataclass
class AddRemoveChange:
    add: list[str] = field(default_factory=list)
    remove: list[str] = field(default_factory=list)

    def to_json(self) -> Optional[Json]:
        if self.add is None and self.remove is None:
            return None
        rv = {}
        if self.add:
            rv["add"] = self.add
        if self.remove:
            rv["remove"] = self.remove
        return rv

    def __bool__(self) -> bool:
        return bool(self.add or self.remove)


@dataclass
class BugChanges:
    keywords: Optional[AddRemoveChange] = None
    see_also: Optional[AddRemoveChange] = None
    whiteboard: Optional[str] = None
    user_story: Optional[str] = None
    status: Optional[str] = None
    resolution: Optional[str] = None
    comment: Optional[str] = None

    def __bool__(self) -> bool:
        if any(add_remove_field for add_remove_field in [self.keywords, self.see_also]):
            return True
        return any(
            string_field is not None
            for string_field in [
                self.whiteboard,
                self.user_story,
                self.status,
                self.resolution,
                self.comment,
            ]
        )

    def to_json(self) -> Json:
        rv: dict[str, Json] = {}
        for add_remove_field, name in [
            (self.keywords, "keywords"),
            (self.see_also, "see_also"),
        ]:
            if add_remove_field is not None:
                value = add_remove_field.to_json()
                if value:
                    rv[name] = value

        for value, name in [
            (self.whiteboard, "whiteboard"),
            (self.status, "status"),
            (self.resolution, "resolution"),
            (self.user_story, "cf_user_story"),
        ]:
            if value is not None:
                rv[name] = value

        if self.comment is not None:
            rv["comment"] = {"body": self.comment}

        return rv


@dataclass
class BugUpdate:
    keywords: dict[str, bool] = field(default_factory=dict)
    see_also: dict[str, bool] = field(default_factory=dict)
    user_story: list[UserStoryChange] = field(default_factory=list)
    comment: list[str] = field(default_factory=list)
    comment_when_unchanged: bool = False
    resolve: Optional[Resolution] = None

    def update_keywords(self, current_keywords: set[str]) -> AddRemoveChange:
        return AddRemoveChange(
            add=[
                keyword
                for keyword, add_keyword in self.keywords.items()
                if add_keyword and keyword not in current_keywords
            ],
            remove=[
                keyword
                for keyword, add_keyword in self.keywords.items()
                if not add_keyword and keyword in current_keywords
            ],
        )

    def update_see_also(
        self, current_url: str, current_see_also: list[str]
    ) -> AddRemoveChange:
        add = []
        remove = []
        has_links = [current_url] + current_see_also
        has_link_keys = url_keys(has_links)
        expected_link_keys = url_keys(self.see_also.keys())

        for key, urls in expected_link_keys.items():
            for url in urls:
                add_url = self.see_also[url]
                if add_url and key not in has_link_keys:
                    add.append(url)
                elif not add_url and key in has_link_keys:
                    remove.append(url)

        return AddRemoveChange(add=add, remove=remove)

    def update_user_story(self, user_story: str) -> Optional[str]:
        new_user_story = []
        user_story_updates = defaultdict(list)
        for change in self.user_story:
            user_story_updates[change.field].append(change)

        has_updates = False
        applied_changes = set()

        for line, key, value in parse_user_story(user_story):
            if key is None or value is None:
                new_user_story.append(line)
                continue

            output_line: Optional[tuple[str, str]] = (key, value)
            if key in user_story_updates:
                changes = user_story_updates[key]
                current_value = value.strip()
                for change in changes:
                    if change in applied_changes:
                        continue
                    if current_value == change.old_value:
                        applied_changes.add(change)
                        if change.type == UserStoryChangeType.DELETE:
                            output_line = None
                            has_updates = True
                        elif change.type == UserStoryChangeType.REPLACE:
                            assert change.new_value is not None
                            output_line = (key, change.new_value)
                            has_updates = True
                    elif (
                        change.type == UserStoryChangeType.APPEND
                        and current_value == change.new_value
                    ):
                        # If we are going to append a value that's already there
                        # do nothing
                        applied_changes.add(change)
            if output_line is not None:
                new_user_story.append(f"{output_line[0]}:{output_line[1]}")

        for changes in user_story_updates.values():
            for change in changes:
                if change not in applied_changes:
                    if change.type == UserStoryChangeType.DELETE:
                        # Tried to delete a key that doesn't exist, do nothing
                        pass
                    elif change.type == UserStoryChangeType.REPLACE:
                        # Tried to replace a key that doesn't exist, do nothing
                        pass
                    elif change.type == UserStoryChangeType.APPEND:
                        new_user_story.append(f"{change.field}:{change.new_value}")
                        has_updates = True

        if has_updates:
            return "\n".join(new_user_story)

        return None

    def into_changes(self, bug: Bug) -> BugChanges:
        changes = BugChanges()

        if self.keywords:
            changes.keywords = self.update_keywords(set(bug["keywords"]))
        if self.see_also:
            changes.see_also = self.update_see_also(bug["url"], bug["see_also"])
        if self.user_story:
            changes.user_story = self.update_user_story(bug["cf_user_story"])

        if self.resolve:
            if bug["resolution"] != self.resolve.value:
                changes.status = (
                    "REOPENED" if self.resolve == Resolution.NONE else "RESOLVED"
                )
                changes.resolution = self.resolve.value

        if self.comment and (changes or self.comment_when_unchanged):
            changes.comment = "\n\n".join(self.comment)

        return changes


def url_keys(urls: Iterable[str]) -> Mapping[tuple[str, str], list[str]]:
    """Group URLs by a key consisting of their hostname and path"""
    rv: dict[tuple[str, str], list[str]] = {}
    for url in urls:
        try:
            parsed = parse.urlparse(url)
            if parsed.hostname is None:
                continue
            key = (parsed.hostname, parsed.path)
            if key not in rv:
                rv[key] = []
            rv[key].append(url)
        except ValueError:
            pass
    return rv


@dataclass
class FeatureData:
    feature: str
    supported_browsers: set[str]
    sp_issue: Optional[int]
    spec_url: set[str]

    def is_supported(self) -> bool:
        return {"firefox", "firefox_android"}.issubset(self.supported_browsers)


@dataclass
class FeatureBug:
    """Bug that represents a web-feature"""

    resolution: str
    keywords: list[str]
    url: Optional[str]
    whiteboard: str
    see_also: set[str]
    user_story: Mapping[str, str | list[str]]
    features: Mapping[str, FeatureData]

    def is_supported(self) -> bool:
        return all(feature.is_supported() for feature in self.features.values())

    def expected_keywords(self) -> set[str]:
        rv = set()
        if "[platform-feature]" in self.whiteboard:
            rv.add("web-feature")
        if not self.is_supported():
            for feature in self.features.values():
                if {"chrome", "chrome_android"}.issubset(feature.supported_browsers):
                    rv.add("parity-chrome")
                if {"safari", "safari_ios"}.issubset(feature.supported_browsers):
                    rv.add("parity-safari")
        return rv

    def missing_keywords(self) -> set[str]:
        return self.expected_keywords().difference(self.keywords)

    def expected_links(self) -> set[str]:
        links = set()
        for feature_name, feature in self.features.items():
            links.add(
                f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/"
            )
            if feature.sp_issue is not None:
                links.add(
                    f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}"
                )
            links |= feature.spec_url
        return links

    def missing_links(self) -> set[str]:
        rv = set()
        has_links = list(self.see_also)
        if self.url:
            has_links.append(self.url)
        has_link_keys = url_keys(has_links)
        expected_link_keys = url_keys(self.expected_links())

        for key, urls in expected_link_keys.items():
            if key not in has_link_keys:
                rv |= set(urls)
        return rv

    def remove_links(self) -> set[str]:
        rv = set()
        for link in self.see_also:
            if link.startswith(
                "https://web-platform-dx.github.io/web-features-explorer/features/"
            ) and not any(
                link.startswith(
                    f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/"
                )
                for feature_name in self.features.keys()
            ):
                rv.add(link)
        return rv


_DataType = TypeVar("_DataType")


class UpdateRule(ABC, Generic[_DataType]):
    """Rule for updating bugs based on BigQuery data"""

    def __init__(self, client: bigquery.Client):
        self.client = client

    @abstractmethod
    def get_data(self) -> _DataType:
        ...

    @abstractmethod
    def update(self, updates: MutableMapping[int, BugUpdate], data: _DataType) -> None:
        ...

    def run(self, updates: MutableMapping[int, BugUpdate]) -> None:
        data: _DataType = self.get_data()
        self.update(updates, data)


class FeatureRenames(UpdateRule):
    """Update web-feature marker for features that have been renamed"""

    def get_data(self) -> Mapping[int, list[tuple[str, str]]]:
        rv = defaultdict(list)
        query = """
    SELECT DISTINCT number, feature, redirect_target
    FROM `web_features.features_moved`
    JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs
      ON feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature"))"""

        for row in self.client.query(query):
            rv[row.number].append((row["feature"], row["redirect_target"]))

        return rv

    def update(
        self,
        updates: MutableMapping[int, BugUpdate],
        data: Mapping[int, list[tuple[str, str]]],
    ) -> None:
        for bug_id, renames in data.items():
            for old_name, new_name in renames:
                updates[bug_id].user_story.append(
                    UserStoryChange(
                        "web-feature", UserStoryChangeType.REPLACE, old_name, new_name
                    )
                )


class InvalidFeatures(UpdateRule):
    def get_data(self) -> Mapping[int, list[tuple[str, list[str]]]]:
        rv = defaultdict(list)
        query = """
    WITH
    missing_features AS (
      SELECT number, bug_feature as bug_feature
      FROM `webcompat_knowledge_base.bugzilla_bugs` AS bugs
      JOIN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature")) AS bug_feature
      LEFT JOIN `web_features.features_latest` AS features ON features.feature = bug_feature
      WHERE features.feature IS NULL
    ),

    suggestions AS (
      SELECT number, bug_feature, feature, EDIT_DISTANCE(feature, bug_feature) as distance
      FROM missing_features
      CROSS JOIN `web_features.features_latest`
    )

    SELECT number, bug_feature, ARRAY_AGG(STRUCT(feature as feature, distance) ORDER BY distance LIMIT 5) AS suggestions
    FROM suggestions
    WHERE distance < 5
    GROUP BY number, bug_feature
    """
        for row in self.client.query(query):
            rv[row.number].append(
                (
                    row.bug_feature,
                    [
                        suggestion["feature"]
                        for suggestion in sorted(
                            row.suggestions, key=lambda x: x["distance"]
                        )
                    ],
                )
            )

        return rv

    def update(
        self,
        updates: MutableMapping[int, BugUpdate],
        data: Mapping[int, list[tuple[str, list[str]]]],
    ) -> None:
        for bug_id, invalid_names in data.items():
            for invalid_name, suggestions in invalid_names:
                updates[bug_id].user_story.append(
                    UserStoryChange(
                        "web-feature", UserStoryChangeType.DELETE, invalid_name
                    )
                )
                options_links = [
                    f"[{suggestion}](https://web-platform-dx.github.io/web-features-explorer/features/{suggestion})"
                    for suggestion in suggestions
                ]
                # TODO: Consider adding a needinfo on someone (reporter? user that added this?)
                comment = f"{invalid_name} is not a valid web-feature id."
                if options_links:
                    comment += f" Consider one of the following possible ids: {', '.join(options_links)}."
                updates[bug_id].comment.append(comment)


class UpdateMetadata(UpdateRule):
    """Update existing web-feature bugs to ensure they have the correct metadata and status"""

    def get_data(self) -> Mapping[int, FeatureBug]:
        rv: dict[int, FeatureBug] = {}
        query = """
WITH
feature_bugs AS (
  SELECT
    number,
    ARRAY_AGG(STRUCT(
      feature,
      web_features.spec as spec_url,
      (SELECT ARRAY_AGG(browser) FROM UNNEST(web_features.support)) AS supported_browsers,
      sp_mozilla.issue as sp_issue
      )
    ) as features,
    LOGICAL_OR(
      bugs.resolution = "FIXED" AND
      ("firefox" NOT in UNNEST(web_features.support.browser) OR
       "firefox_android" NOT IN UNNEST(web_features.support.browser))
    ) as unsupported_closed_bug
  FROM `webcompat_knowledge_base.bugzilla_bugs` AS bugs
  JOIN `web_features.features_latest` AS web_features
    ON web_features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature"))
  LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla
    ON (`webcompat_knowledge_base.BUG_ID_FROM_BUGZILLA_URL`(sp_mozilla.bug) = bugs.number OR sp_mozilla.web_feature = feature)
  GROUP BY number
)

SELECT
    number,
    resolution,
    url,
    keywords,
    whiteboard,
    see_also,
    user_story,
    features
FROM feature_bugs
JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs USING(number)
WHERE
  ("web-feature" in UNNEST(keywords) OR whiteboard LIKE "%[platform-feature]%") AND
  (resolution = "" OR unsupported_closed_bug)
"""

        for row in self.client.query(query):
            rv[row.number] = FeatureBug(
                resolution=row.resolution,
                url=row.url,
                keywords=row.keywords,
                whiteboard=row.whiteboard,
                see_also=set(row.see_also),
                user_story=row.user_story,
                features={
                    feature["feature"]: FeatureData(
                        feature=feature["feature"],
                        spec_url=set(feature["spec_url"]),
                        supported_browsers=set(feature["supported_browsers"]),
                        sp_issue=feature["sp_issue"],
                    )
                    for feature in row.features
                },
            )

        return rv

    def update(
        self, updates: MutableMapping[int, BugUpdate], data: Mapping[int, FeatureBug]
    ) -> None:
        for bug_id, feature_bug in data.items():
            # Add any missing keywords
            # TODO: are there keywords we should remove too if they're invalid
            for keyword in feature_bug.missing_keywords():
                updates[bug_id].keywords[keyword] = True
            for link in feature_bug.missing_links():
                updates[bug_id].see_also[link] = True
            for link in feature_bug.remove_links():
                updates[bug_id].see_also[link] = False

            # Close bugs where the BCD status is fixed
            if (
                feature_bug.resolution == ""
                and "leave-open" not in feature_bug.keywords
                and feature_bug.is_supported()
            ):
                updates[bug_id].resolve = Resolution.FIXED

            # Reopen bugs where the BCD status is not fixed
            unsupported_features = [
                feature
                for feature in feature_bug.features.values()
                if not feature.is_supported()
            ]
            if feature_bug.resolution == "FIXED" and unsupported_features:
                updates[bug_id].resolve = Resolution.NONE
                feature_list = ", ".join(
                    f"{feature_name} ([definition file](https://github.com/web-platform-dx/web-features/blob/main/features/{feature_name}.yml.dist))"
                    for feature_name in unsupported_features
                )
                text = (
                    f"web-features {feature_list} are"
                    if len(unsupported_features) > 1
                    else f"web-feature {feature_list} is"
                )
                updates[bug_id].comment.append(
                    f"""Bug was resolved, but the {text} not yet marked as supported in Firefox.

Feature bugs are usually automatically closed once the corresponding web-features are marked as supported; this typically happens after the feature reaches release.
"""
                )


class WebPlatformFeatures(BzCleaner):
    def __init__(self) -> None:
        super().__init__()
        self.bug_updates: dict[int, BugUpdate] = defaultdict(BugUpdate)

    def description(self) -> str:
        return "Update web-features bugs"

    def filter_no_nag_keyword(self) -> bool:
        return False

    def has_default_products(self) -> bool:
        return False

    def columns(self) -> list[str]:
        return ["id", "summary", "changes", "whiteboard", "user_story"]

    def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]:
        bug_id_str = str(bug["id"])
        bug_id_int = int(bug["id"])

        if bug_id_int not in self.bug_updates:
            return None
        changes = self.bug_updates[bug_id_int].into_changes(bug)

        if changes:
            self.autofix_changes[bug_id_str] = changes.to_json()
            data[bug_id_str] = {
                "changes": changes,
                "whiteboard": bug["whiteboard"],
                "user_story": bug["cf_user_story"],
            }
            return bug

        return None

    def get_bz_params(self, date: str) -> dict[str, str | int | list[str] | list[int]]:
        fields = [
            "id",
            "url",
            "see_also",
            "keywords",
            "whiteboard",
            "cf_user_story",
            "status",
            "resolution",
        ]
        self.get_bug_updates()
        return {"include_fields": fields, "id": list(self.bug_updates.keys())}

    def get_bug_updates(self) -> None:
        project = "moz-fx-dev-dschubert-wckb"
        client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"])
        for update_rule in [
            FeatureRenames(client),
            InvalidFeatures(client),
            UpdateMetadata(client),
        ]:
            update_rule.run(self.bug_updates)


if __name__ == "__main__":
    WebPlatformFeatures().run()

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	# mypy: disallow-untyped-defs
6	import re	1✔
7	from abc import ABC, abstractmethod	1✔
8	from collections import defaultdict	1✔
9	from collections.abc import Iterator	1✔
10	from dataclasses import dataclass, field	1✔
11	from enum import Enum, IntEnum	1✔
12	from typing import (	1✔
13	Any,
14	Generic,
15	Iterable,
16	Mapping,
17	MutableMapping,
18	Optional,
19	Sequence,
20	TypeVar,
21	)
22	from urllib import parse	1✔
23
24	from google.cloud import bigquery	1✔
25
26	from bugbot import gcp	1✔
27	from bugbot.bzcleaner import Bug, BzCleaner	1✔
28
29	Json = None \| str \| int \| float \| Sequence["Json"] \| Mapping[str, "Json"]	1✔
30
31
32	def parse_user_story(	1✔
33	user_story: str,
34	) -> Iterator[tuple[str, Optional[str], Optional[str]]]:
35	"""Parse the user story assuming it's lines of the form key: value.
36
37	If there isn't a colon in the line we simply set value to the full line."""
38	user_story_re = re.compile(r"^\s([^\s]+)\s:\s(.)")	1✔
39	for line in user_story.splitlines():	1✔
40	key = None	1✔
41	value = None	1✔
42	m = user_story_re.match(line)	1✔
43	if m is not None:	1✔
44	maybe_key, maybe_value = m.groups()	1✔
45	if maybe_value:	1✔
46	key = maybe_key	1✔
47	value = maybe_value	1✔
48	yield line, key, value	1✔
49
50
51	class UserStoryChangeType(IntEnum):	1✔
52	APPEND = 1	1✔
53	REPLACE = 2	1✔
54	DELETE = 3	1✔
55
56
57	@dataclass(frozen=True)	1✔
58	class UserStoryChange:	1✔
59	field: str	1✔
60	type: UserStoryChangeType	1✔
61	old_value: Optional[str] = None	1✔
62	new_value: Optional[str] = None	1✔
63
64
65	class Resolution(Enum):	1✔
66	NONE = ""	1✔
67	FIXED = "FIXED"	1✔
68	DUPLICATE = "DUPLICATE"	1✔
69
70
71	@dataclass	1✔
72	class AddRemoveChange:	1✔
73	add: list[str] = field(default_factory=list)	1✔
74	remove: list[str] = field(default_factory=list)	1✔
75
76	def to_json(self) -> Optional[Json]:	1✔
77	if self.add is None and self.remove is None:	1!
NEW 78	return None	×
79	rv = {}	1✔
80	if self.add:	1✔
81	rv["add"] = self.add	1✔
82	if self.remove:	1✔
83	rv["remove"] = self.remove	1✔
84	return rv	1✔
85
86	def __bool__(self) -> bool:	1✔
87	return bool(self.add or self.remove)	1✔
88
89
90	@dataclass	1✔
91	class BugChanges:	1✔
92	keywords: Optional[AddRemoveChange] = None	1✔
93	see_also: Optional[AddRemoveChange] = None	1✔
94	whiteboard: Optional[str] = None	1✔
95	user_story: Optional[str] = None	1✔
96	status: Optional[str] = None	1✔
97	resolution: Optional[str] = None	1✔
98	comment: Optional[str] = None	1✔
99
100	def __bool__(self) -> bool:	1✔
101	if any(add_remove_field for add_remove_field in [self.keywords, self.see_also]):	1✔
102	return True	1✔
103	return any(	1✔
104	string_field is not None
105	for string_field in [
106	self.whiteboard,
107	self.user_story,
108	self.status,
109	self.resolution,
110	self.comment,
111	]
112	)
113
114	def to_json(self) -> Json:	1✔
115	rv: dict[str, Json] = {}	1✔
116	for add_remove_field, name in [	1✔
117	(self.keywords, "keywords"),
118	(self.see_also, "see_also"),
119	]:
120	if add_remove_field is not None:	1✔
121	value = add_remove_field.to_json()	1✔
122	if value:	1!
123	rv[name] = value	1✔
124
125	for value, name in [	1✔
126	(self.whiteboard, "whiteboard"),
127	(self.status, "status"),
128	(self.resolution, "resolution"),
129	(self.user_story, "cf_user_story"),
130	]:
131	if value is not None:	1✔
132	rv[name] = value	1✔
133
134	if self.comment is not None:	1✔
135	rv["comment"] = {"body": self.comment}	1✔
136
137	return rv	1✔
138
139
140	@dataclass	1✔
141	class BugUpdate:	1✔
142	keywords: dict[str, bool] = field(default_factory=dict)	1✔
143	see_also: dict[str, bool] = field(default_factory=dict)	1✔
144	user_story: list[UserStoryChange] = field(default_factory=list)	1✔
145	comment: list[str] = field(default_factory=list)	1✔
146	comment_when_unchanged: bool = False	1✔
147	resolve: Optional[Resolution] = None	1✔
148
149	def update_keywords(self, current_keywords: set[str]) -> AddRemoveChange:	1✔
150	return AddRemoveChange(	1✔
151	add=[
152	keyword
153	for keyword, add_keyword in self.keywords.items()
154	if add_keyword and keyword not in current_keywords
155	],
156	remove=[
157	keyword
158	for keyword, add_keyword in self.keywords.items()
159	if not add_keyword and keyword in current_keywords
160	],
161	)
162
163	def update_see_also(	1✔
164	self, current_url: str, current_see_also: list[str]
165	) -> AddRemoveChange:
166	add = []	1✔
167	remove = []	1✔
168	has_links = [current_url] + current_see_also	1✔
169	has_link_keys = url_keys(has_links)	1✔
170	expected_link_keys = url_keys(self.see_also.keys())	1✔
171
172	for key, urls in expected_link_keys.items():	1✔
173	for url in urls:	1✔
174	add_url = self.see_also[url]	1✔
175	if add_url and key not in has_link_keys:	1✔
176	add.append(url)	1✔
177	elif not add_url and key in has_link_keys:	1✔
178	remove.append(url)	1✔
179
180	return AddRemoveChange(add=add, remove=remove)	1✔
181
182	def update_user_story(self, user_story: str) -> Optional[str]:	1✔
183	new_user_story = []	1✔
184	user_story_updates = defaultdict(list)	1✔
185	for change in self.user_story:	1✔
186	user_story_updates[change.field].append(change)	1✔
187
188	has_updates = False	1✔
189	applied_changes = set()	1✔
190
191	for line, key, value in parse_user_story(user_story):	1✔
192	if key is None or value is None:	1✔
193	new_user_story.append(line)	1✔
194	continue	1✔
195
196	output_line: Optional[tuple[str, str]] = (key, value)	1✔
197	if key in user_story_updates:	1!
198	changes = user_story_updates[key]	1✔
199	current_value = value.strip()	1✔
200	for change in changes:	1✔
201	if change in applied_changes:	1✔
202	continue	1✔
203	if current_value == change.old_value:	1✔
204	applied_changes.add(change)	1✔
205	if change.type == UserStoryChangeType.DELETE:	1✔
206	output_line = None	1✔
207	has_updates = True	1✔
208	elif change.type == UserStoryChangeType.REPLACE:	1!
209	assert change.new_value is not None	1✔
210	output_line = (key, change.new_value)	1✔
211	has_updates = True	1✔
212	elif (	1✔
213	change.type == UserStoryChangeType.APPEND
214	and current_value == change.new_value
215	):
216	# If we are going to append a value that's already there
217	# do nothing
218	applied_changes.add(change)	1✔
219	if output_line is not None:	1✔
220	new_user_story.append(f"{output_line[0]}:{output_line[1]}")	1✔
221
222	for changes in user_story_updates.values():	1✔
223	for change in changes:	1✔
224	if change not in applied_changes:	1✔
225	if change.type == UserStoryChangeType.DELETE:	1!
226	# Tried to delete a key that doesn't exist, do nothing
NEW 227	pass	×
228	elif change.type == UserStoryChangeType.REPLACE:	1✔
229	# Tried to replace a key that doesn't exist, do nothing
230	pass	1✔
231	elif change.type == UserStoryChangeType.APPEND:	1!
232	new_user_story.append(f"{change.field}:{change.new_value}")	1✔
233	has_updates = True	1✔
234
235	if has_updates:	1!
236	return "\n".join(new_user_story)	1✔
237
NEW 238	return None	×
239
240	def into_changes(self, bug: Bug) -> BugChanges:	1✔
241	changes = BugChanges()	1✔
242
243	if self.keywords:	1✔
244	changes.keywords = self.update_keywords(set(bug["keywords"]))	1✔
245	if self.see_also:	1✔
246	changes.see_also = self.update_see_also(bug["url"], bug["see_also"])	1✔
247	if self.user_story:	1✔
248	changes.user_story = self.update_user_story(bug["cf_user_story"])	1✔
249
250	if self.resolve:	1✔
251	if bug["resolution"] != self.resolve.value:	1!
252	changes.status = (	1✔
253	"REOPENED" if self.resolve == Resolution.NONE else "RESOLVED"
254	)
255	changes.resolution = self.resolve.value	1✔
256
257	if self.comment and (changes or self.comment_when_unchanged):	1✔
258	changes.comment = "\n\n".join(self.comment)	1✔
259
260	return changes	1✔
261
262
263	def url_keys(urls: Iterable[str]) -> Mapping[tuple[str, str], list[str]]:	1✔
264	"""Group URLs by a key consisting of their hostname and path"""
265	rv: dict[tuple[str, str], list[str]] = {}	1✔
266	for url in urls:	1✔
267	try:	1✔
268	parsed = parse.urlparse(url)	1✔
269	if parsed.hostname is None:	1!
270	continue	×
271	key = (parsed.hostname, parsed.path)	1✔
272	if key not in rv:	1✔
273	rv[key] = []	1✔
274	rv[key].append(url)	1✔
275	except ValueError:	×
276	pass	×
277	return rv	1✔
278
279
280	@dataclass	1✔
281	class FeatureData:	1✔
282	feature: str	1✔
283	supported_browsers: set[str]	1✔
284	sp_issue: Optional[int]	1✔
285	spec_url: set[str]	1✔
286
287	def is_supported(self) -> bool:	1✔
NEW 288	return {"firefox", "firefox_android"}.issubset(self.supported_browsers)	×
289
290
291	@dataclass	1✔
292	class FeatureBug:	1✔
293	"""Bug that represents a web-feature"""
294
295	resolution: str	1✔
296	keywords: list[str]	1✔
297	url: Optional[str]	1✔
298	whiteboard: str	1✔
299	see_also: set[str]	1✔
300	user_story: Mapping[str, str \| list[str]]	1✔
301	features: Mapping[str, FeatureData]	1✔
302
303	def is_supported(self) -> bool:	1✔
NEW 304	return all(feature.is_supported() for feature in self.features.values())	×
305
306	def expected_keywords(self) -> set[str]:	1✔
NEW 307	rv = set()	×
NEW 308	if "[platform-feature]" in self.whiteboard:	×
NEW 309	rv.add("web-feature")	×
NEW 310	if not self.is_supported():	×
NEW 311	for feature in self.features.values():	×
NEW 312	if {"chrome", "chrome_android"}.issubset(feature.supported_browsers):	×
NEW 313	rv.add("parity-chrome")	×
NEW 314	if {"safari", "safari_ios"}.issubset(feature.supported_browsers):	×
NEW 315	rv.add("parity-safari")	×
NEW 316	return rv	×
317
318	def missing_keywords(self) -> set[str]:	1✔
NEW 319	return self.expected_keywords().difference(self.keywords)	×
320
321	def expected_links(self) -> set[str]:	1✔
NEW 322	links = set()	×
NEW 323	for feature_name, feature in self.features.items():	×
NEW 324	links.add(	×
325	f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/"
326	)
NEW 327	if feature.sp_issue is not None:	×
NEW 328	links.add(	×
329	f"https://github.com/mozilla/standards-positions/issues/{feature.sp_issue}"
330	)
NEW 331	links \|= feature.spec_url	×
NEW 332	return links	×
333
334	def missing_links(self) -> set[str]:	1✔
NEW 335	rv = set()	×
NEW 336	has_links = list(self.see_also)	×
NEW 337	if self.url:	×
NEW 338	has_links.append(self.url)	×
NEW 339	has_link_keys = url_keys(has_links)	×
NEW 340	expected_link_keys = url_keys(self.expected_links())	×
341
NEW 342	for key, urls in expected_link_keys.items():	×
NEW 343	if key not in has_link_keys:	×
NEW 344	rv \|= set(urls)	×
NEW 345	return rv	×
346
347	def remove_links(self) -> set[str]:	1✔
NEW 348	rv = set()	×
NEW 349	for link in self.see_also:	×
NEW 350	if link.startswith(	×
351	"https://web-platform-dx.github.io/web-features-explorer/features/"
352	) and not any(
353	link.startswith(
354	f"https://web-platform-dx.github.io/web-features-explorer/features/{feature_name}/"
355	)
356	for feature_name in self.features.keys()
357	):
NEW 358	rv.add(link)	×
NEW 359	return rv	×
360
361
362	_DataType = TypeVar("_DataType")	1✔
363
364
365	class UpdateRule(ABC, Generic[_DataType]):	1✔
366	"""Rule for updating bugs based on BigQuery data"""
367
368	def __init__(self, client: bigquery.Client):	1✔
NEW 369	self.client = client	×
370
371	@abstractmethod
372	def get_data(self) -> _DataType:
373	...
374
375	@abstractmethod
376	def update(self, updates: MutableMapping[int, BugUpdate], data: _DataType) -> None:
377	...
378
379	def run(self, updates: MutableMapping[int, BugUpdate]) -> None:	1✔
NEW 380	data: _DataType = self.get_data()	×
NEW 381	self.update(updates, data)	×
382
383
384	class FeatureRenames(UpdateRule):	1✔
385	"""Update web-feature marker for features that have been renamed"""
386
387	def get_data(self) -> Mapping[int, list[tuple[str, str]]]:	1✔
NEW 388	rv = defaultdict(list)	×
NEW 389	query = """	×
390	SELECT DISTINCT number, feature, redirect_target
391	FROM `web_features.features_moved`
392	JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs
393	ON feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature"))"""
394
NEW 395	for row in self.client.query(query):	×
NEW 396	rv[row.number].append((row["feature"], row["redirect_target"]))	×
397
NEW 398	return rv	×
399
400	def update(	1✔
401	self,
402	updates: MutableMapping[int, BugUpdate],
403	data: Mapping[int, list[tuple[str, str]]],
404	) -> None:
NEW 405	for bug_id, renames in data.items():	×
NEW 406	for old_name, new_name in renames:	×
NEW 407	updates[bug_id].user_story.append(	×
408	UserStoryChange(
409	"web-feature", UserStoryChangeType.REPLACE, old_name, new_name
410	)
411	)
412
413
414	class InvalidFeatures(UpdateRule):	1✔
415	def get_data(self) -> Mapping[int, list[tuple[str, list[str]]]]:	1✔
NEW 416	rv = defaultdict(list)	×
NEW 417	query = """	×
418	WITH
419	missing_features AS (
420	SELECT number, bug_feature as bug_feature
421	FROM `webcompat_knowledge_base.bugzilla_bugs` AS bugs
422	JOIN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature")) AS bug_feature
423	LEFT JOIN `web_features.features_latest` AS features ON features.feature = bug_feature
424	WHERE features.feature IS NULL
425	),
426
427	suggestions AS (
428	SELECT number, bug_feature, feature, EDIT_DISTANCE(feature, bug_feature) as distance
429	FROM missing_features
430	CROSS JOIN `web_features.features_latest`
431	)
432
433	SELECT number, bug_feature, ARRAY_AGG(STRUCT(feature as feature, distance) ORDER BY distance LIMIT 5) AS suggestions
434	FROM suggestions
435	WHERE distance < 5
436	GROUP BY number, bug_feature
437	"""
NEW 438	for row in self.client.query(query):	×
NEW 439	rv[row.number].append(	×
440	(
441	row.bug_feature,
442	[
443	suggestion["feature"]
444	for suggestion in sorted(
445	row.suggestions, key=lambda x: x["distance"]
446	)
447	],
448	)
449	)
450
NEW 451	return rv	×
452
453	def update(	1✔
454	self,
455	updates: MutableMapping[int, BugUpdate],
456	data: Mapping[int, list[tuple[str, list[str]]]],
457	) -> None:
NEW 458	for bug_id, invalid_names in data.items():	×
NEW 459	for invalid_name, suggestions in invalid_names:	×
NEW 460	updates[bug_id].user_story.append(	×
461	UserStoryChange(
462	"web-feature", UserStoryChangeType.DELETE, invalid_name
463	)
464	)
NEW 465	options_links = [	×
466	f"[{suggestion}](https://web-platform-dx.github.io/web-features-explorer/features/{suggestion})"
467	for suggestion in suggestions
468	]
469	# TODO: Consider adding a needinfo on someone (reporter? user that added this?)
NEW 470	comment = f"{invalid_name} is not a valid web-feature id."	×
NEW 471	if options_links:	×
NEW 472	comment += f" Consider one of the following possible ids: {', '.join(options_links)}."	×
NEW 473	updates[bug_id].comment.append(comment)	×
474
475
476	class UpdateMetadata(UpdateRule):	1✔
477	"""Update existing web-feature bugs to ensure they have the correct metadata and status"""
478
479	def get_data(self) -> Mapping[int, FeatureBug]:	1✔
NEW 480	rv: dict[int, FeatureBug] = {}	×
NEW 481	query = """	×
482	WITH
483	feature_bugs AS (
484	SELECT
485	number,
486	ARRAY_AGG(STRUCT(
487	feature,
488	web_features.spec as spec_url,
489	(SELECT ARRAY_AGG(browser) FROM UNNEST(web_features.support)) AS supported_browsers,
490	sp_mozilla.issue as sp_issue
491	)
492	) as features,
493	LOGICAL_OR(
494	bugs.resolution = "FIXED" AND
495	("firefox" NOT in UNNEST(web_features.support.browser) OR
496	"firefox_android" NOT IN UNNEST(web_features.support.browser))
497	) as unsupported_closed_bug
498	FROM `webcompat_knowledge_base.bugzilla_bugs` AS bugs
499	JOIN `web_features.features_latest` AS web_features
500	ON web_features.feature IN UNNEST(`webcompat_knowledge_base.EXTRACT_ARRAY`(bugs.user_story, "$.web-feature"))
501	LEFT JOIN `standards_positions.mozilla_standards_positions` AS sp_mozilla
502	ON (`webcompat_knowledge_base.BUG_ID_FROM_BUGZILLA_URL`(sp_mozilla.bug) = bugs.number OR sp_mozilla.web_feature = feature)
503	GROUP BY number
504	)
505
506	SELECT
507	number,
508	resolution,
509	url,
510	keywords,
511	whiteboard,
512	see_also,
513	user_story,
514	features
515	FROM feature_bugs
516	JOIN `webcompat_knowledge_base.bugzilla_bugs` AS bugs USING(number)
517	WHERE
518	("web-feature" in UNNEST(keywords) OR whiteboard LIKE "%[platform-feature]%") AND
519	(resolution = "" OR unsupported_closed_bug)
520	"""
521
NEW 522	for row in self.client.query(query):	×
NEW 523	rv[row.number] = FeatureBug(	×
524	resolution=row.resolution,
525	url=row.url,
526	keywords=row.keywords,
527	whiteboard=row.whiteboard,
528	see_also=set(row.see_also),
529	user_story=row.user_story,
530	features={
531	feature["feature"]: FeatureData(
532	feature=feature["feature"],
533	spec_url=set(feature["spec_url"]),
534	supported_browsers=set(feature["supported_browsers"]),
535	sp_issue=feature["sp_issue"],
536	)
537	for feature in row.features
538	},
539	)
540
NEW 541	return rv	×
542
543	def update(	1✔
544	self, updates: MutableMapping[int, BugUpdate], data: Mapping[int, FeatureBug]
545	) -> None:
NEW 546	for bug_id, feature_bug in data.items():	×
547	# Add any missing keywords
548	# TODO: are there keywords we should remove too if they're invalid
NEW 549	for keyword in feature_bug.missing_keywords():	×
NEW 550	updates[bug_id].keywords[keyword] = True	×
NEW 551	for link in feature_bug.missing_links():	×
NEW 552	updates[bug_id].see_also[link] = True	×
NEW 553	for link in feature_bug.remove_links():	×
NEW 554	updates[bug_id].see_also[link] = False	×
555
556	# Close bugs where the BCD status is fixed
NEW 557	if (	×
558	feature_bug.resolution == ""
559	and "leave-open" not in feature_bug.keywords
560	and feature_bug.is_supported()
561	):
NEW 562	updates[bug_id].resolve = Resolution.FIXED	×
563
564	# Reopen bugs where the BCD status is not fixed
NEW 565	unsupported_features = [	×
566	feature
567	for feature in feature_bug.features.values()
568	if not feature.is_supported()
569	]
NEW 570	if feature_bug.resolution == "FIXED" and unsupported_features:	×
NEW 571	updates[bug_id].resolve = Resolution.NONE	×
NEW 572	feature_list = ", ".join(	×
573	f"{feature_name} ([definition file](https://github.com/web-platform-dx/web-features/blob/main/features/{feature_name}.yml.dist))"
574	for feature_name in unsupported_features
575	)
NEW 576	text = (	×
577	f"web-features {feature_list} are"
578	if len(unsupported_features) > 1
579	else f"web-feature {feature_list} is"
580	)
NEW 581	updates[bug_id].comment.append(	×
582	f"""Bug was resolved, but the {text} not yet marked as supported in Firefox.
583
584	Feature bugs are usually automatically closed once the corresponding web-features are marked as supported; this typically happens after the feature reaches release.
585	"""
586	)
587
588
589	class WebPlatformFeatures(BzCleaner):	1✔
590	def __init__(self) -> None:	1✔
591	super().__init__()	1✔
592	self.bug_updates: dict[int, BugUpdate] = defaultdict(BugUpdate)	1✔
593
594	def description(self) -> str:	1✔
NEW 595	return "Update web-features bugs"	×
596
597	def filter_no_nag_keyword(self) -> bool:	1✔
598	return False	×
599
600	def has_default_products(self) -> bool:	1✔
601	return False	×
602
603	def columns(self) -> list[str]:	1✔
NEW 604	return ["id", "summary", "changes", "whiteboard", "user_story"]	×
605
606	def handle_bug(self, bug: Bug, data: dict[str, Any]) -> Optional[Bug]:	1✔
607	bug_id_str = str(bug["id"])	1✔
608	bug_id_int = int(bug["id"])	1✔
609
610	if bug_id_int not in self.bug_updates:	1!
NEW 611	return None	×
612	changes = self.bug_updates[bug_id_int].into_changes(bug)	1✔
613
614	if changes:	1!
615	self.autofix_changes[bug_id_str] = changes.to_json()	1✔
616	data[bug_id_str] = {	1✔
617	"changes": changes,
618	"whiteboard": bug["whiteboard"],
619	"user_story": bug["cf_user_story"],
620	}
621	return bug	1✔
622
623	return None	×
624
625	def get_bz_params(self, date: str) -> dict[str, str \| int \| list[str] \| list[int]]:	1✔
NEW 626	fields = [	×
627	"id",
628	"url",
629	"see_also",
630	"keywords",
631	"whiteboard",
632	"cf_user_story",
633	"status",
634	"resolution",
635	]
NEW 636	self.get_bug_updates()	×
NEW 637	return {"include_fields": fields, "id": list(self.bug_updates.keys())}	×
638
639	def get_bug_updates(self) -> None:	1✔
640	project = "moz-fx-dev-dschubert-wckb"	×
UNCOV 641	client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"])	×
NEW 642	for update_rule in [	×
643	FeatureRenames(client),
644	InvalidFeatures(client),
645	UpdateMetadata(client),
646	]:
NEW 647	update_rule.run(self.bug_updates)	×
648
649
650	if __name__ == "__main__":	1!
651	WebPlatformFeatures().run()	×

mozilla / relman-auto-nag / #5852

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous