8312886973

Committed 17 Mar 2024 04:13AM UTC coverage: 89.268% (-0.1%) from 89.364%

Build # 8312886973

Build Type

push

github

Committed by

snejus

Commit Message

internal: slightly clean up Album logic

Run Details

487 of 556 branches covered (87.59%)

Branch coverage included in aggregate %.

24 of 26 new or added lines in 1 file covered. (92.31%)

12 existing lines in 2 files now uncovered.

977 of 1084 relevant lines covered (90.13%)

13.42 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.21

/beetsplug/bandcamp/helpers.py

"""Module with a Helpers class that contains various static, independent functions."""

from __future__ import annotations

import itertools as it
import operator as op
import re
from functools import lru_cache, partial
from typing import Any, Dict, Iterable, List, NamedTuple, Pattern

from beets.autotag.hooks import AlbumInfo
from ordered_set import OrderedSet as ordset

from .genres_lookup import GENRES

JSONDict = Dict[str, Any]
DIGI_MEDIA = "Digital Media"
FORMAT_TO_MEDIA = {
    "VinylFormat": "Vinyl",
    "CDFormat": "CD",
    "CassetteFormat": "Cassette",
    "DigitalFormat": DIGI_MEDIA,
    "DVDFormat": "DVD",
    "USB Flash Drive": DIGI_MEDIA,
}


class MediaInfo(NamedTuple):
    album_id: str
    name: str
    title: str
    description: str


CATALOGNUM_CONSTRAINT = r"""(?<![]/@-])(\b
(?!\W|LC[ ]|VA[\d ]+|[EL]P[\W\d]|[^\n.]+[ ](?:20\d\d|VA[ \d]+)|(?i:vol|disc|number|rd-9))
{}
\b(?!["%]))"""
_cat_pat = CATALOGNUM_CONSTRAINT.format(
    r"""
(
      [A-Z][A-Z .]+\d{3}         # HANDS D300, CC ATOM 101
    | [A-Z-]{3,}\d+              # RIV4
    | [A-Z]{2,}[A-Z.$-]*\d{2,}   # HS11, USE202, HEY-101, LI$INGLE025
    | (?<!\w\W)[A-Z.]{2,}[ ]\d+  # OBS.CUR 9
    | [A-z]+-[A-z]+[ ]?\d+       # o-ton 119
    | \w+[A-z]0\d+               # 1ØPILLS018, fa036
    | [a-z]+(?:cd|lp|:)\d+       # ostgutlp45, reni:7
    | [A-z]+\d+-\d+              # P90-003
)
(?: # optionally followed by
      (?<=\d\d)-?[A-Z]+  # IBM001CD (needs at least two digits before the letter)
    | [.]\d+             # ISMVA002.1
)?
"""
)

LABEL_CATNUM = CATALOGNUM_CONSTRAINT.format(r"(?i:{}[ ]?[A-Z]*\d+[A-Z]*)")
CATNUM_PAT = {
    # preceded by some variation of 'Catalogue number:'
    "header": re.compile(r"^cat[\w .]+(?:number\b:?|:) ?(\w.+)$", re.I | re.M),
    # beginning or end of line
    "start_end": re.compile(rf"(^{_cat_pat}|{_cat_pat}$)", re.M | re.VERBOSE),
    # enclosed by parens or square brackets, but not ending with MIX
    "delimited": re.compile(rf"(?:[\[(])(?!.*MIX){_cat_pat}(?:[])]|$)", re.VERBOSE),
    # can possibly be followed up by a second catalogue number
    "anywhere": re.compile(rf"({_cat_pat}(\ /\ {_cat_pat})?)", re.VERBOSE),
}

PATTERNS: Dict[str, Pattern[str]] = {
    "split_artists": re.compile(r", - |, | (?:[x+/-]|//|vs|and)[.]? "),
    "meta": re.compile(r'.*"@id".*'),
    "ft": re.compile(
        r"""
        [ ]*                        # all preceding space
        ((?P<br>[\[(])|\b)          # bracket or word boundary
        (ft|feat|featuring|w/)[. ]  # any ft variation
        (
            # when it does not start with a bracket, do not allow " - " in it, otherwise
            # we may match full track name
            (?(br)|(?!.*[ ]-[ ].*))
            # anything but brackets or a slash, except for a slash preceded
            # by a non-space (can be part of artist or title)
            (?:[^]\[()/]|\S/)+
        )
        (?<!mix)\b    # does not end with "mix"
        (?(br)[]\)])  # if it started with a bracket, it must end with a closing bracket
        [ ]*          # trailing space
    """,
        re.I | re.VERBOSE,
    ),
    "track_alt": re.compile(
        r"^([A-J]{1,3}[12]?\.?\d|[AB]+(?=\W{2,}))(?:(?!-\w)[^\w(]|_)+", re.I + re.M
    ),
    "vinyl_name": re.compile(r"[1-5](?= ?(xLP|LP|x))|single|double|triple", re.I),
}
rm_strings = [
    "limited edition",
    r"^[EL]P( \d+)?",
    r"\((digital )?album\)",
    r"\(single\)",
    r"\Wvinyl\W|vinyl-only",
    "compiled by.*",
    r"[\[(]presented by.*",
    r"free download|\([^()]*\bfree(?!.*mix)[^()]*\)",
    r"(\W|\W )bonus( \w+)*",
    "Various -",
    r"CD ?\d+",
]

_remix_pat = r"(?P<remix>((?P<remixer>[^])]+) )?\b((re)?mix|edit|bootleg)\b[^])]*)"
CAMELCASE = re.compile(r"(?<=[a-z])(?=[A-Z])")


def split_artist_title(m: re.Match[str]) -> str:
    """See for yourself.

    https://examine-archive.bandcamp.com/album/va-examine-archive-international-sampler-xmn01
    """
    artist, title = m.groups()
    artist = CAMELCASE.sub(" ", artist)
    title = CAMELCASE.sub(" ", title)

    return f"{artist} - {title}"


# fmt: off
CLEAN_PATTERNS = [
    (re.compile(rf"(([\[(])|(^| ))\*?({'|'.join(rm_strings)})(?(2)[])]|([- ]|$))", re.I), ""),       # noqa
    (re.compile(r" -(\S)"), r" - \1"),                    # hi -bye          -> hi - bye
    (re.compile(r"(\S)- "), r"\1 - "),                    # hi- bye          -> hi - bye
    (re.compile(r"  +"), " "),                            # hi  bye          -> hi bye
    (re.compile(r"(- )?\( *"), "("),                      # hi - ( bye)      -> hi (bye)
    (re.compile(r" \)+|(\)+$)"), ")"),                    # hi (bye ))       -> hi (bye)
    (re.compile(r"- Reworked"), "(Reworked)"),            # bye - Reworked   -> bye (Reworked)    # noqa
    (re.compile(rf"(\({_remix_pat})$", re.I), r"\1)"),    # bye - (Some Mix  -> bye - (Some Mix)  # noqa
    (re.compile(rf"- *({_remix_pat})$", re.I), r"(\1)"),  # bye - Some Mix   -> bye (Some Mix)    # noqa
    (re.compile(r'(^|- )[“"]([^”"]+)[”"]( \(|$)'), r"\1\2\3"),   # "bye" -> bye; hi - "bye" -> hi - bye  # noqa
    (re.compile(r"\((the )?(remixes)\)", re.I), r"\2"),   # Album (Remixes)  -> Album Remixes     # noqa
    (re.compile(r"examine-.+CD\d+_([^_-]+)[_-](.*)"), split_artist_title),  # See https://examine-archive.bandcamp.com/album/va-examine-archive-international-sampler-xmn01 # noqa
]
# fmt: on


class Helpers:
    @staticmethod
    def get_label(meta: JSONDict) -> str:
        try:
            item = meta["albumRelease"][0]["recordLabel"]
        except (KeyError, IndexError):
            item = meta["publisher"]
        return item.get("name") or ""

    @staticmethod
    def get_vinyl_count(name: str) -> int:
        conv = {"single": 1, "double": 2, "triple": 3}
        for m in PATTERNS["vinyl_name"].finditer(name):
            count = m.group()
            return int(count) if count.isdigit() else conv[count.lower()]
        return 1

    @staticmethod
    def split_artists(artists: Iterable[str]) -> List[str]:
        """Split artists taking into account delimiters such as ',', '+', 'x', 'X' etc.
        Note: featuring artists are removed since they are not main artists.
        """
        no_ft_artists = (PATTERNS["ft"].sub("", a) for a in artists)
        split = map(PATTERNS["split_artists"].split, ordset(no_ft_artists))
        split_artists = ordset(map(str.strip, it.chain(*split))) - {"", "more"}

        for artist in list(split_artists):
            # ' & ' or ' X ' may be part of single artist name, so we need to be careful
            # here. We check whether any of the split artists appears on their own and
            # only split then
            for char in "X&":
                subartists = artist.split(f" {char} ")
                if len(subartists) > 1 and any(s in split_artists for s in subartists):
                    split_artists.discard(artist)  # type: ignore[attr-defined]
                    split_artists.update(subartists)  # type: ignore[attr-defined]
        return list(split_artists)

    @staticmethod
    @lru_cache(maxsize=None)
    def parse_catalognum(
        album="", disctitle="", description="", label="", artistitles=""
    ):
        # type: (str, str, str, str, str) -> str
        """Try getting the catalog number looking at text from various fields."""
        cases = [
            (CATNUM_PAT["header"], description),
            (CATNUM_PAT["anywhere"], disctitle),
            (CATNUM_PAT["anywhere"], album),
            (CATNUM_PAT["start_end"], description),
            (CATNUM_PAT["anywhere"], description),
        ]
        if label:
            pat = re.compile(LABEL_CATNUM.format(re.escape(label)), re.VERBOSE)
            cases.append((pat, "\n".join((album, disctitle, description))))

        def find(pat: Pattern[str], string: str) -> str:
            """Return the match if it is
            * not found in any of the track artists or titles
            * made of the label name when it has a space and is shorter than 6 chars
            """
            for m in pat.finditer(string):
                catnum = m.group(1).strip()
                if catnum.lower() not in artistitles:
                    if " " in catnum:
                        first = catnum.split()[0].lower()
                        if len(catnum) <= 5 and first not in label.lower():
                            continue
                    return catnum
            return ""

        try:
            return next(filter(None, it.starmap(find, cases)))
        except StopIteration:
            return ""

    @staticmethod
    def clean_name(name: str) -> str:
        """Both album and track names are cleaned using these patterns."""
        for pat, repl in CLEAN_PATTERNS:
            name = pat.sub(repl, name).strip()
        return name

    @staticmethod
    def get_genre(keywords, config, label):
        # type: (Iterable[str], JSONDict, str) -> Iterable[str]
        """Return a comma-delimited list of valid genres, using MB genres for reference.

        1. Exclude keywords that are label names, unless they are a valid MB genre

        2. Verify each keyword's (potential genre) validity w.r.t. the configured `mode`
          * classical: valid only if the _entire keyword_ matches a MB genre in the list
          * progressive: either above or if each of the words matches MB genre - since
            it is effectively a subgenre.
          * psychedelic: one of the above or if the last word is a valid MB genre.
            This allows to be flexible regarding the variety of potential genres while
            keeping away from spammy ones.

        3. Once we have the list of keywords that coming out of the mode filters,
           an additional filter is executed:
           * if a keyword is _part of another keyword_ (genre within a sub-genre),
             the more generic option gets excluded, for example,
             >>> get_genre(['house', 'garage house', 'glitch'], "classical")
             'garage house, glitch'
        """
        valid_mb_genre = partial(op.contains, GENRES)
        label_name = label.lower().replace(" ", "")

        def is_label_name(kw: str) -> bool:
            return kw.replace(" ", "") == label_name and not valid_mb_genre(kw)

        def is_included(kw: str) -> bool:
            return any(re.search(x, kw) for x in config["always_include"])

        def valid_for_mode(kw: str) -> bool:
            if config["mode"] == "classical":
                return valid_mb_genre(kw)

            words = map(str.strip, kw.split(" "))
            if config["mode"] == "progressive":
                return valid_mb_genre(kw) or all(map(valid_mb_genre, words))

            return valid_mb_genre(kw) or valid_mb_genre(list(words)[-1])

        unique_genres: ordset[str] = ordset()
        # expand badly delimited keywords
        split_kw = partial(re.split, r"[.] | #| - ")
        for kw in it.chain.from_iterable(map(split_kw, keywords)):
            # remove full stops and hashes and ensure the expected form of 'and'
            kw = re.sub("[.#]", "", str(kw)).replace("&", "and")
            if not is_label_name(kw) and (is_included(kw) or valid_for_mode(kw)):
                unique_genres.add(kw)

        def duplicate(genre: str) -> bool:
            """Return True if genre is contained within another genre or if,
            having removed spaces from every other, there is a duplicate found.
            It is done this way so that 'dark folk' is kept while 'darkfolk' is removed,
            and not the other way around.
            """
            others = unique_genres - {genre}
            others = others.union(x.replace(" ", "").replace("-", "") for x in others)  # type: ignore[attr-defined] # noqa
            return any(genre in x for x in others)

        return it.filterfalse(duplicate, unique_genres)

    @staticmethod
    def unpack_props(obj: JSONDict) -> JSONDict:
        """Add all 'additionalProperty'-ies to the parent dictionary."""
        for prop in obj.get("additionalProperty") or []:
            obj[prop["name"]] = prop["value"]
        return obj

    @staticmethod
    def get_media_formats(format_list: List[JSONDict]) -> List[MediaInfo]:
        """Return filtered Bandcamp media formats as a list of MediaInfo objects.
        Formats are filtered using the following fields,

        type_id item_type  type_name          musicReleaseFormat
                a          Digital            DigitalFormat   # digital album
                b          Digital            DigitalFormat   # discography
                t          Digital            DigitalFormat   # digital track
        0       p          Other
        1       p          Compact Disc (CD)  CDFormat
        2       p          Vinyl LP           VinylFormat
        3       p          Cassette           CassetteFormat
        4       p          DVD                DVDFormat
        5       p          USB Flash Drive
        10      p          Poster/Print
        11      p          T-Shirt/Apparel
        15      p          2 x Vinyl LP       VinylFormat
        16      p          7" Vinyl           VinylFormat
        17      p          Vinyl Box Set      VinylFormat
        18      p          Other Vinyl        VinylFormat
        19      p          T-Shirt/Shirt
        20      p          Sweater/Hoodie
        """

        def valid_format(obj: JSONDict) -> bool:
            return (
                {"name", "item_type"} < set(obj)
                # not a discography
                and obj["item_type"] != "b"
                # musicReleaseFormat format is given or it is a USB
                and ("musicReleaseFormat" in obj or obj["type_id"] == 5)
                # it is not a vinyl bundle
                and not (obj["item_type"] == "p" and "bundle" in obj["name"].lower())
            )

        formats = []
        for _format in filter(valid_format, map(Helpers.unpack_props, format_list)):
            formats.append(
                MediaInfo(
                    _format["@id"],
                    FORMAT_TO_MEDIA[
                        _format.get("musicReleaseFormat") or "DigitalFormat"
                    ],
                    _format["name"],
                    _format.get("description") or "",
                )
            )
        return formats

    @staticmethod
    def add_track_alts(album: AlbumInfo, comments: str) -> AlbumInfo:
        # using an ordered set here in case of duplicates
        track_alts = ordset(PATTERNS["track_alt"].findall(comments))

        @lru_cache(maxsize=None)
        def get_medium_total(medium: int) -> int:
            starts = {1: "AB", 2: "CD", 3: "EF", 4: "GH", 5: "IJ"}[medium]
            return len(re.findall(rf"^[{starts}]", "\n".join(track_alts), re.M))

        medium = 1
        medium_index = 1
        if len(track_alts) == len(album.tracks):
            for track, track_alt in zip(album.tracks, track_alts):
                track.track_alt = track_alt
                track.medium_index = medium_index
                track.medium = medium
                track.medium_total = get_medium_total(medium)
                if track.medium_index == track.medium_total:
                    medium += 1
                    medium_index = 1
                else:
                    medium_index += 1
        return album

1	"""Module with a Helpers class that contains various static, independent functions."""
2
3	from __future__ import annotations	15✔
4
5	import itertools as it	15✔
6	import operator as op	15✔
7	import re	15✔
8	from functools import lru_cache, partial	15✔
9	from typing import Any, Dict, Iterable, List, NamedTuple, Pattern	15✔
10
11	from beets.autotag.hooks import AlbumInfo	15✔
12	from ordered_set import OrderedSet as ordset	15✔
13
14	from .genres_lookup import GENRES	15✔
15
16	JSONDict = Dict[str, Any]	15✔
17	DIGI_MEDIA = "Digital Media"	15✔
18	FORMAT_TO_MEDIA = {	15✔
19	"VinylFormat": "Vinyl",
20	"CDFormat": "CD",
21	"CassetteFormat": "Cassette",
22	"DigitalFormat": DIGI_MEDIA,
23	"DVDFormat": "DVD",
24	"USB Flash Drive": DIGI_MEDIA,
25	}
26
27
28	class MediaInfo(NamedTuple):	15✔
29	album_id: str	15✔
30	name: str	15✔
31	title: str	15✔
32	description: str	15✔
33
34
35	CATALOGNUM_CONSTRAINT = r"""(?<![]/@-])(\b	15✔
36	(?!\W\|LC[ ]\|VA[\d ]+\|[EL]P[\W\d]\|[^\n.]+[ ](?:20\d\d\|VA[ \d]+)\|(?i:vol\|disc\|number\|rd-9))
37	{}
38	\b(?!["%]))"""
39	_cat_pat = CATALOGNUM_CONSTRAINT.format(	15✔
40	r"""
41	(
42	[A-Z][A-Z .]+\d{3} # HANDS D300, CC ATOM 101
43	\| [A-Z-]{3,}\d+ # RIV4
44	\| [A-Z]{2,}[A-Z.$-]*\d{2,} # HS11, USE202, HEY-101, LI$INGLE025
45	\| (?<!\w\W)[A-Z.]{2,}[ ]\d+ # OBS.CUR 9
46	\| [A-z]+-[A-z]+[ ]?\d+ # o-ton 119
47	\| \w+[A-z]0\d+ # 1ØPILLS018, fa036
48	\| [a-z]+(?:cd\|lp\|:)\d+ # ostgutlp45, reni:7
49	\| [A-z]+\d+-\d+ # P90-003
50	)
51	(?: # optionally followed by
52	(?<=\d\d)-?[A-Z]+ # IBM001CD (needs at least two digits before the letter)
53	\| [.]\d+ # ISMVA002.1
54	)?
55	"""
56	)
57
58	LABEL_CATNUM = CATALOGNUM_CONSTRAINT.format(r"(?i:{}[ ]?[A-Z]\d+[A-Z])")	15✔
59	CATNUM_PAT = {	15✔
60	# preceded by some variation of 'Catalogue number:'
61	"header": re.compile(r"^cat[\w .]+(?:number\b:?\|:) ?(\w.+)$", re.I \| re.M),
62	# beginning or end of line
63	"start_end": re.compile(rf"(^{_cat_pat}\|{_cat_pat}$)", re.M \| re.VERBOSE),
64	# enclosed by parens or square brackets, but not ending with MIX
65	"delimited": re.compile(rf"(?:[\[(])(?!.*MIX){_cat_pat}(?:[])]\|$)", re.VERBOSE),
66	# can possibly be followed up by a second catalogue number
67	"anywhere": re.compile(rf"({_cat_pat}(\ /\ {_cat_pat})?)", re.VERBOSE),
68	}
69
70	PATTERNS: Dict[str, Pattern[str]] = {	15✔
71	"split_artists": re.compile(r", - \|, \| (?:[x+/-]\|//\|vs\|and)[.]? "),
72	"meta": re.compile(r'."@id".'),
73	"ft": re.compile(
74	r"""
75	[ ]* # all preceding space
76	((?P<br>[\[(])\|\b) # bracket or word boundary
77	(ft\|feat\|featuring\|w/)[. ] # any ft variation
78	(
79	# when it does not start with a bracket, do not allow " - " in it, otherwise
80	# we may match full track name
81	(?(br)\|(?!.[ ]-[ ].))
82	# anything but brackets or a slash, except for a slash preceded
83	# by a non-space (can be part of artist or title)
84	(?:[^]\[()/]\|\S/)+
85	)
86	(?<!mix)\b # does not end with "mix"
87	(?(br)[]\)]) # if it started with a bracket, it must end with a closing bracket
88	[ ]* # trailing space
89	""",
90	re.I \| re.VERBOSE,
91	),
92	"track_alt": re.compile(
93	r"^([A-J]{1,3}[12]?\.?\d\|[AB]+(?=\W{2,}))(?:(?!-\w)[^\w(]\|_)+", re.I + re.M
94	),
95	"vinyl_name": re.compile(r"[1-5](?= ?(xLP\|LP\|x))\|single\|double\|triple", re.I),
96	}
97	rm_strings = [	15✔
98	"limited edition",
99	r"^[EL]P( \d+)?",
100	r"\((digital )?album\)",
101	r"\(single\)",
102	r"\Wvinyl\W\|vinyl-only",
103	"compiled by.*",
104	r"[\[(]presented by.*",
105	r"free download\|\([^()]\bfree(?!.mix)[^()]*\)",
106	r"(\W\|\W )bonus( \w+)*",
107	"Various -",
108	r"CD ?\d+",
109	]
110
111	_remix_pat = r"(?P<remix>((?P<remixer>[^])]+) )?\b((re)?mix\|edit\|bootleg)\b[^])]*)"	15✔
112	CAMELCASE = re.compile(r"(?<=[a-z])(?=[A-Z])")	15✔
113
114
115	def split_artist_title(m: re.Match[str]) -> str:	15✔
116	"""See for yourself.
117
118	https://examine-archive.bandcamp.com/album/va-examine-archive-international-sampler-xmn01
119	"""
120	artist, title = m.groups()	×
UNCOV 121	artist = CAMELCASE.sub(" ", artist)	×
UNCOV 122	title = CAMELCASE.sub(" ", title)	×
123
UNCOV 124	return f"{artist} - {title}"	×
125
126
127	# fmt: off
128	CLEAN_PATTERNS = [	15✔
129	(re.compile(rf"(([\[(])\|(^\| ))\*?({'\|'.join(rm_strings)})(?(2)[])]\|([- ]\|$))", re.I), ""), # noqa
130	(re.compile(r" -(\S)"), r" - \1"), # hi -bye -> hi - bye
131	(re.compile(r"(\S)- "), r"\1 - "), # hi- bye -> hi - bye
132	(re.compile(r" +"), " "), # hi bye -> hi bye
133	(re.compile(r"(- )?\( *"), "("), # hi - ( bye) -> hi (bye)
134	(re.compile(r" \)+\|(\)+$)"), ")"), # hi (bye )) -> hi (bye)
135	(re.compile(r"- Reworked"), "(Reworked)"), # bye - Reworked -> bye (Reworked) # noqa
136	(re.compile(rf"(\({_remix_pat})$", re.I), r"\1)"), # bye - (Some Mix -> bye - (Some Mix) # noqa
137	(re.compile(rf"- *({_remix_pat})$", re.I), r"(\1)"), # bye - Some Mix -> bye (Some Mix) # noqa
138	(re.compile(r'(^\|- )[“"]([^”"]+)[”"]( \(\|$)'), r"\1\2\3"), # "bye" -> bye; hi - "bye" -> hi - bye # noqa
139	(re.compile(r"\((the )?(remixes)\)", re.I), r"\2"), # Album (Remixes) -> Album Remixes # noqa
140	(re.compile(r"examine-.+CD\d+_([^_-]+)[_-](.*)"), split_artist_title), # See https://examine-archive.bandcamp.com/album/va-examine-archive-international-sampler-xmn01 # noqa
141	]
142	# fmt: on
143
144
145	class Helpers:	15✔
146	@staticmethod	15✔
147	def get_label(meta: JSONDict) -> str:	15✔
148	try:	15✔
149	item = meta["albumRelease"][0]["recordLabel"]	15✔
150	except (KeyError, IndexError):	15✔
151	item = meta["publisher"]	15✔
152	return item.get("name") or ""	15✔
153
154	@staticmethod	15✔
155	def get_vinyl_count(name: str) -> int:	15✔
156	conv = {"single": 1, "double": 2, "triple": 3}	15✔
157	for m in PATTERNS["vinyl_name"].finditer(name):	15✔
158	count = m.group()	15✔
159	return int(count) if count.isdigit() else conv[count.lower()]	15✔
160	return 1	15✔
161
162	@staticmethod	15✔
163	def split_artists(artists: Iterable[str]) -> List[str]:	15✔
164	"""Split artists taking into account delimiters such as ',', '+', 'x', 'X' etc.
165	Note: featuring artists are removed since they are not main artists.
166	"""
167	no_ft_artists = (PATTERNS["ft"].sub("", a) for a in artists)	15✔
168	split = map(PATTERNS["split_artists"].split, ordset(no_ft_artists))	15✔
169	split_artists = ordset(map(str.strip, it.chain(*split))) - {"", "more"}	15✔
170
171	for artist in list(split_artists):	15✔
172	# ' & ' or ' X ' may be part of single artist name, so we need to be careful
173	# here. We check whether any of the split artists appears on their own and
174	# only split then
175	for char in "X&":	15✔
176	subartists = artist.split(f" {char} ")	15✔
177	if len(subartists) > 1 and any(s in split_artists for s in subartists):	15✔
178	split_artists.discard(artist) # type: ignore[attr-defined]	15✔
179	split_artists.update(subartists) # type: ignore[attr-defined]	15✔
180	return list(split_artists)	15✔
181
182	@staticmethod	15✔
183	@lru_cache(maxsize=None)	15✔
184	def parse_catalognum(	15✔
185	album="", disctitle="", description="", label="", artistitles=""
186	):
187	# type: (str, str, str, str, str) -> str
188	"""Try getting the catalog number looking at text from various fields."""
189	cases = [	15✔
190	(CATNUM_PAT["header"], description),
191	(CATNUM_PAT["anywhere"], disctitle),
192	(CATNUM_PAT["anywhere"], album),
193	(CATNUM_PAT["start_end"], description),
194	(CATNUM_PAT["anywhere"], description),
195	]
196	if label:	15✔
197	pat = re.compile(LABEL_CATNUM.format(re.escape(label)), re.VERBOSE)	15✔
198	cases.append((pat, "\n".join((album, disctitle, description))))	15✔
199
200	def find(pat: Pattern[str], string: str) -> str:	15✔
201	"""Return the match if it is
202	* not found in any of the track artists or titles
203	* made of the label name when it has a space and is shorter than 6 chars
204	"""
205	for m in pat.finditer(string):	15✔
206	catnum = m.group(1).strip()	15✔
207	if catnum.lower() not in artistitles:	15✔
208	if " " in catnum:	15✔
209	first = catnum.split()[0].lower()	15✔
210	if len(catnum) <= 5 and first not in label.lower():	15!
UNCOV 211	continue	×
212	return catnum	15✔
213	return ""	15✔
214
215	try:	15✔
216	return next(filter(None, it.starmap(find, cases)))	15✔
217	except StopIteration:	15✔
218	return ""	15✔
219
220	@staticmethod	15✔
221	def clean_name(name: str) -> str:	15✔
222	"""Both album and track names are cleaned using these patterns."""
223	for pat, repl in CLEAN_PATTERNS:	15✔
224	name = pat.sub(repl, name).strip()	15✔
225	return name	15✔
226
227	@staticmethod	15✔
228	def get_genre(keywords, config, label):	15✔
229	# type: (Iterable[str], JSONDict, str) -> Iterable[str]
230	"""Return a comma-delimited list of valid genres, using MB genres for reference.
231
232	1. Exclude keywords that are label names, unless they are a valid MB genre
233
234	2. Verify each keyword's (potential genre) validity w.r.t. the configured `mode`
235	* classical: valid only if the _entire keyword_ matches a MB genre in the list
236	* progressive: either above or if each of the words matches MB genre - since
237	it is effectively a subgenre.
238	* psychedelic: one of the above or if the last word is a valid MB genre.
239	This allows to be flexible regarding the variety of potential genres while
240	keeping away from spammy ones.
241
242	3. Once we have the list of keywords that coming out of the mode filters,
243	an additional filter is executed:
244	* if a keyword is _part of another keyword_ (genre within a sub-genre),
245	the more generic option gets excluded, for example,
246	>>> get_genre(['house', 'garage house', 'glitch'], "classical")
247	'garage house, glitch'
248	"""
249	valid_mb_genre = partial(op.contains, GENRES)	15✔
250	label_name = label.lower().replace(" ", "")	15✔
251
252	def is_label_name(kw: str) -> bool:	15✔
253	return kw.replace(" ", "") == label_name and not valid_mb_genre(kw)	15✔
254
255	def is_included(kw: str) -> bool:	15✔
256	return any(re.search(x, kw) for x in config["always_include"])	15✔
257
258	def valid_for_mode(kw: str) -> bool:	15✔
259	if config["mode"] == "classical":	15✔
260	return valid_mb_genre(kw)	15✔
261
262	words = map(str.strip, kw.split(" "))	15✔
263	if config["mode"] == "progressive":	15✔
264	return valid_mb_genre(kw) or all(map(valid_mb_genre, words))	15✔
265
266	return valid_mb_genre(kw) or valid_mb_genre(list(words)[-1])	15✔
267
268	unique_genres: ordset[str] = ordset()	15✔
269	# expand badly delimited keywords
270	split_kw = partial(re.split, r"[.] \| #\| - ")	15✔
271	for kw in it.chain.from_iterable(map(split_kw, keywords)):	15✔
272	# remove full stops and hashes and ensure the expected form of 'and'
273	kw = re.sub("[.#]", "", str(kw)).replace("&", "and")	15✔
274	if not is_label_name(kw) and (is_included(kw) or valid_for_mode(kw)):	15✔
275	unique_genres.add(kw)	15✔
276
277	def duplicate(genre: str) -> bool:	15✔
278	"""Return True if genre is contained within another genre or if,
279	having removed spaces from every other, there is a duplicate found.
280	It is done this way so that 'dark folk' is kept while 'darkfolk' is removed,
281	and not the other way around.
282	"""
283	others = unique_genres - {genre}	15✔
284	others = others.union(x.replace(" ", "").replace("-", "") for x in others) # type: ignore[attr-defined] # noqa	15✔
285	return any(genre in x for x in others)	15✔
286
287	return it.filterfalse(duplicate, unique_genres)	15✔
288
289	@staticmethod	15✔
290	def unpack_props(obj: JSONDict) -> JSONDict:	15✔
291	"""Add all 'additionalProperty'-ies to the parent dictionary."""
292	for prop in obj.get("additionalProperty") or []:	15✔
293	obj[prop["name"]] = prop["value"]	15✔
294	return obj	15✔
295
296	@staticmethod	15✔
297	def get_media_formats(format_list: List[JSONDict]) -> List[MediaInfo]:	15✔
298	"""Return filtered Bandcamp media formats as a list of MediaInfo objects.
299	Formats are filtered using the following fields,
300
301	type_id item_type type_name musicReleaseFormat
302	a Digital DigitalFormat # digital album
303	b Digital DigitalFormat # discography
304	t Digital DigitalFormat # digital track
305	0 p Other
306	1 p Compact Disc (CD) CDFormat
307	2 p Vinyl LP VinylFormat
308	3 p Cassette CassetteFormat
309	4 p DVD DVDFormat
310	5 p USB Flash Drive
311	10 p Poster/Print
312	11 p T-Shirt/Apparel
313	15 p 2 x Vinyl LP VinylFormat
314	16 p 7" Vinyl VinylFormat
315	17 p Vinyl Box Set VinylFormat
316	18 p Other Vinyl VinylFormat
317	19 p T-Shirt/Shirt
318	20 p Sweater/Hoodie
319	"""
320
321	def valid_format(obj: JSONDict) -> bool:	15✔
322	return (	15✔
323	{"name", "item_type"} < set(obj)
324	# not a discography
325	and obj["item_type"] != "b"
326	# musicReleaseFormat format is given or it is a USB
327	and ("musicReleaseFormat" in obj or obj["type_id"] == 5)
328	# it is not a vinyl bundle
329	and not (obj["item_type"] == "p" and "bundle" in obj["name"].lower())
330	)
331
332	formats = []	15✔
333	for _format in filter(valid_format, map(Helpers.unpack_props, format_list)):	15✔
334	formats.append(	15✔
335	MediaInfo(
336	_format["@id"],
337	FORMAT_TO_MEDIA[
338	_format.get("musicReleaseFormat") or "DigitalFormat"
339	],
340	_format["name"],
341	_format.get("description") or "",
342	)
343	)
344	return formats	15✔
345
346	@staticmethod	15✔
347	def add_track_alts(album: AlbumInfo, comments: str) -> AlbumInfo:	15✔
348	# using an ordered set here in case of duplicates
349	track_alts = ordset(PATTERNS["track_alt"].findall(comments))	15✔
350
351	@lru_cache(maxsize=None)	15✔
352	def get_medium_total(medium: int) -> int:	15✔
353	starts = {1: "AB", 2: "CD", 3: "EF", 4: "GH", 5: "IJ"}[medium]	15✔
354	return len(re.findall(rf"^[{starts}]", "\n".join(track_alts), re.M))	15✔
355
356	medium = 1	15✔
357	medium_index = 1	15✔
358	if len(track_alts) == len(album.tracks):	15✔
359	for track, track_alt in zip(album.tracks, track_alts):	15✔
360	track.track_alt = track_alt	15✔
361	track.medium_index = medium_index	15✔
362	track.medium = medium	15✔
363	track.medium_total = get_medium_total(medium)	15✔
364	if track.medium_index == track.medium_total:	15✔
365	medium += 1	15✔
366	medium_index = 1	15✔
367	else:
368	medium_index += 1	15✔
369	return album	15✔

snejus / beetcamp / 8312886973

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous