8306877220

Committed 16 Mar 2024 09:38AM UTC coverage: 89.364%. Remained the same

Build # 8306877220

Build Type

push

github

Committed by

snejus

Commit Message

internal: normalize track delimiter in tracks.py

Run Details

484 of 552 branches covered (87.68%)

Branch coverage included in aggregate %.

10 of 10 new or added lines in 2 files covered. (100.0%)

10 existing lines in 2 files now uncovered.

978 of 1084 relevant lines covered (90.22%)

13.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.36

/beetsplug/bandcamp/tracks.py

"""Module with tracks parsing functionality."""

import itertools as it
import operator as op
import re
from collections import Counter
from dataclasses import dataclass
from functools import cached_property, reduce
from typing import Iterator, List, Optional, Set

from ordered_set import OrderedSet

from .helpers import CATNUM_PAT, Helpers, JSONDict, _remix_pat
from .track import Track

DELIMITER_PAT = re.compile(r" ([^\w&()+/[\] ]) ")
TITLE_IN_QUOTES = re.compile(r'^(.+[^ -])[ -]+"([^"]+)"$')
NUMBER_PREFIX = re.compile(r"(^|- )\d{2,}\W* ")


@dataclass
class Tracks:
    tracks: List[Track]

    def __iter__(self) -> Iterator[Track]:
        return iter(self.tracks)

    def __len__(self) -> int:
        return len(self.tracks)

    @staticmethod
    def split_quoted_titles(names: List[str]) -> List[str]:
        if len(names) > 1 and all(TITLE_IN_QUOTES.match(n) for n in names):
            return [TITLE_IN_QUOTES.sub(r"\1 - \2", n) for n in names]
        return names

    @staticmethod
    def remove_number_prefix(names: List[str]) -> List[str]:
        if len(names) > 1 and all(NUMBER_PREFIX.search(n) for n in names):
            return [NUMBER_PREFIX.sub(r"\1", n) for n in names]
        return names

    @staticmethod
    def find_common_track_delimiter(names: List[str]) -> str:
        """Return the track parts delimiter that is in effect in the current release.
        In some (rare) situations track parts are delimited by a pipe character
        or some UTF-8 equivalent of a dash.

        This checks every track for the first character (see the regex for exclusions)
        that splits it. The character that splits the most and at least half of
        the tracks is the character we need.

        If no such character is found, or if we have just one track, return a dash '-'.
        """

        def get_delim(string: str) -> str:
            m = DELIMITER_PAT.search(string)
            return m.group(1) if m else "-"

        delim, count = Counter(map(get_delim, names)).most_common(1).pop()
        return delim if (len(names) == 1 or count > len(names) / 2) else "-"

    @staticmethod
    def common_name_parts(tracks, names):
        # type: (List[JSONDict], List[str]) -> List[JSONDict]
        """Parse track names for parts that require knowledge of the other names.

        1. Split each track name into words
        2. Find the list of words that are common to all tracks
           a. check the *first* and the *last* word for the catalog number
              1. If found, remove it from every track name
           b. check whether tracks start the same way. This indicates an album with
              one unique root title and a couple of its remixes. This is especially
              relevant when the remix is not delimited appropriately.
        Return the catalog number and the new list of names.
        """
        names_tokens = map(str.split, names)
        common_words = reduce(op.and_, [OrderedSet(x) for x in names_tokens])
        if not common_words:
            return tracks

        matches = (CATNUM_PAT["anywhere"].search(common_words[i]) for i in [0, -1])
        try:
            cat, word = next(((m.group(1), m.string) for m in matches if m))
        except StopIteration:
            pass
        else:
            for track in tracks:
                track["catalognum"] = cat
                track["name"] = track["name"].replace(word, "").strip()

        joined = " ".join(common_words)
        if joined in names:  # it is one of the track names (root title)
            for track in tracks:
                leftover = track["name"].replace(joined, "").lstrip()
                # looking for a remix without brackets
                if re.fullmatch(_remix_pat, leftover, re.I):
                    track["name"] = f"{joined} ({leftover})"

        return tracks

    @classmethod
    def normalize_delimiter(cls, names: List[str]) -> List[str]:
        delim = cls.find_common_track_delimiter(names)
        return (
            names if delim == "-" else [n.replace(f" {delim} ", " - ") for n in names]
        )

    @classmethod
    def from_json(cls, meta: JSONDict) -> "Tracks":
        try:
            tracks = [{**t, **t["item"]} for t in meta["track"]["itemListElement"]]
        except (TypeError, KeyError):
            tracks = [meta]

        names = [i.get("name", "") for i in tracks]
        names = cls.split_quoted_titles(names)
        names = cls.remove_number_prefix(names)
        names = cls.normalize_delimiter(names)
        for track, name in zip(tracks, names):
            track["name"] = name

        tracks = cls.common_name_parts(tracks, names)
        return cls([Track.from_json(t, Helpers.get_label(meta)) for t in tracks])

    @cached_property
    def first(self) -> Track:
        return self.tracks[0]

    @cached_property
    def raw_names(self) -> List[str]:
        return [j.name for j in self.tracks]

    @cached_property
    def original_artists(self) -> List[str]:
        """Return all unique unsplit (original) main track artists."""
        return list(dict.fromkeys(j.artist for j in self.tracks))

    @property
    def artists(self) -> List[str]:
        """Return all unique split main track artists.

        "Artist1 x Artist2" -> ["Artist1", "Artist2"]
        """
        return list(dict.fromkeys(it.chain(*(j.artists for j in self.tracks))))

    @property
    def remixers(self) -> List[str]:
        """Return all remix artists."""
        return [
            t.remix.remixer
            for t in self.tracks
            if t.remix and not t.remix.by_other_artist
        ]

    @property
    def other_artists(self) -> Set[str]:
        """Return all unique remix and featuring artists."""
        ft = [j.ft for j in self.tracks if j.ft]
        return set(it.chain(self.remixers, ft))

    @cached_property
    def all_artists(self) -> Set[str]:
        """Return all unique (1) track, (2) remix, (3) featuring artists."""
        return self.other_artists | set(self.original_artists)

    @cached_property
    def artistitles(self) -> str:
        """Returned artists and titles joined into one long string."""
        return " ".join(it.chain(self.raw_names, self.all_artists)).lower()

    @cached_property
    def single_catalognum(self) -> Optional[str]:
        """Return catalognum if every track contains the same one."""
        cats = [t.catalognum for t in self if t.catalognum]
        if len(cats) == len(self) and len(set(cats)) == 1:
            return cats[0]
        return None

    @cached_property
    def albums_in_titles(self) -> Set[str]:
        return {t.album for t in self if t.album}

    def adjust_artists(self, albumartist: str) -> None:
        """Handle some track artist edge cases.

        These checks require knowledge of the entire release, therefore cannot be
        performed within the context of a single track.

        * When artist name is mistaken for the track_alt
        * When artist and title are delimited by '-' without spaces
        * When artist and title are delimited by a UTF-8 dash equivalent
        * Defaulting to the album artist
        """
        track_alts = {t.track_alt for t in self.tracks if t.track_alt}
        artists = [t.artist for t in self.tracks if t.artist]

        for t in [track for track in self.tracks if not track.artist]:
            if t.track_alt and len(track_alts) == 1:  # only one track_alt
                # the only track that parsed a track alt - it's most likely a mistake
                # one artist was confused for a track alt, like 'B2', - reverse this
                t.artist, t.track_alt = t.track_alt, None
            elif len(artists) == len(self) - 1:  # only 1 missing artist
                # if this is a remix and the parsed title is part of the albumartist or
                # is one of the track artists, we made a mistake parsing the remix:
                #  it is most probably the edge case where the `title_without_remix` is a
                #  legitimate artist and the track title is something like 'Hello Remix'
                if t.remix and (t.title_without_remix in albumartist):
                    t.artist, t.title = t.title_without_remix, t.remix.remix
                # this is the only artist that didn't get parsed - relax the rule
                # and try splitting with '-' without spaces
                split = t.title.split("-")
                if len(split) == 1:
                    # attempt to split by another ' ? ' where '?' may be some utf-8
                    # alternative of a dash
                    split = [s for s in DELIMITER_PAT.split(t.title) if len(s) > 1]
                if len(split) > 1:
                    t.artist, t.title = split
            if not t.artist:
                # default to the albumartist
                t.artist = albumartist

1	"""Module with tracks parsing functionality."""
2
3	import itertools as it	15✔
4	import operator as op	15✔
5	import re	15✔
6	from collections import Counter	15✔
7	from dataclasses import dataclass	15✔
8	from functools import cached_property, reduce	15✔
9	from typing import Iterator, List, Optional, Set	15✔
10
11	from ordered_set import OrderedSet	15✔
12
13	from .helpers import CATNUM_PAT, Helpers, JSONDict, _remix_pat	15✔
14	from .track import Track	15✔
15
16	DELIMITER_PAT = re.compile(r" ([^\w&()+/[\] ]) ")	15✔
17	TITLE_IN_QUOTES = re.compile(r'^(.+[^ -])[ -]+"([^"]+)"$')	15✔
18	NUMBER_PREFIX = re.compile(r"(^\|- )\d{2,}\W* ")	15✔
19
20
21	@dataclass	15✔
22	class Tracks:	15✔
23	tracks: List[Track]	15✔
24
25	def __iter__(self) -> Iterator[Track]:	15✔
26	return iter(self.tracks)	15✔
27
28	def __len__(self) -> int:	15✔
29	return len(self.tracks)	15✔
30
31	@staticmethod	15✔
32	def split_quoted_titles(names: List[str]) -> List[str]:	15✔
33	if len(names) > 1 and all(TITLE_IN_QUOTES.match(n) for n in names):	15!
UNCOV 34	return [TITLE_IN_QUOTES.sub(r"\1 - \2", n) for n in names]	×
35	return names	15✔
36
37	@staticmethod	15✔
38	def remove_number_prefix(names: List[str]) -> List[str]:	15✔
39	if len(names) > 1 and all(NUMBER_PREFIX.search(n) for n in names):	15✔
40	return [NUMBER_PREFIX.sub(r"\1", n) for n in names]	15✔
41	return names	15✔
42
43	@staticmethod	15✔
44	def find_common_track_delimiter(names: List[str]) -> str:	15✔
45	"""Return the track parts delimiter that is in effect in the current release.
46	In some (rare) situations track parts are delimited by a pipe character
47	or some UTF-8 equivalent of a dash.
48
49	This checks every track for the first character (see the regex for exclusions)
50	that splits it. The character that splits the most and at least half of
51	the tracks is the character we need.
52
53	If no such character is found, or if we have just one track, return a dash '-'.
54	"""
55
56	def get_delim(string: str) -> str:	15✔
57	m = DELIMITER_PAT.search(string)	15✔
58	return m.group(1) if m else "-"	15✔
59
60	delim, count = Counter(map(get_delim, names)).most_common(1).pop()	15✔
61	return delim if (len(names) == 1 or count > len(names) / 2) else "-"	15✔
62
63	@staticmethod	15✔
64	def common_name_parts(tracks, names):	15✔
65	# type: (List[JSONDict], List[str]) -> List[JSONDict]
66	"""Parse track names for parts that require knowledge of the other names.
67
68	1. Split each track name into words
69	2. Find the list of words that are common to all tracks
70	a. check the first and the last word for the catalog number
71	1. If found, remove it from every track name
72	b. check whether tracks start the same way. This indicates an album with
73	one unique root title and a couple of its remixes. This is especially
74	relevant when the remix is not delimited appropriately.
75	Return the catalog number and the new list of names.
76	"""
77	names_tokens = map(str.split, names)	15✔
78	common_words = reduce(op.and_, [OrderedSet(x) for x in names_tokens])	15✔
79	if not common_words:	15✔
80	return tracks	15✔
81
82	matches = (CATNUM_PAT["anywhere"].search(common_words[i]) for i in [0, -1])	15✔
83	try:	15✔
84	cat, word = next(((m.group(1), m.string) for m in matches if m))	15✔
85	except StopIteration:	15✔
86	pass	15✔
87	else:
88	for track in tracks:	15✔
89	track["catalognum"] = cat	15✔
90	track["name"] = track["name"].replace(word, "").strip()	15✔
91
92	joined = " ".join(common_words)	15✔
93	if joined in names: # it is one of the track names (root title)	15✔
94	for track in tracks:	15✔
95	leftover = track["name"].replace(joined, "").lstrip()	15✔
96	# looking for a remix without brackets
97	if re.fullmatch(_remix_pat, leftover, re.I):	15✔
98	track["name"] = f"{joined} ({leftover})"	15✔
99
100	return tracks	15✔
101
102	@classmethod	15✔
103	def normalize_delimiter(cls, names: List[str]) -> List[str]:	15✔
104	delim = cls.find_common_track_delimiter(names)	15✔
105	return (	15!
106	names if delim == "-" else [n.replace(f" {delim} ", " - ") for n in names]
107	)
108
109	@classmethod	15✔
110	def from_json(cls, meta: JSONDict) -> "Tracks":	15✔
111	try:	15✔
112	tracks = [{t, t["item"]} for t in meta["track"]["itemListElement"]]	15✔
113	except (TypeError, KeyError):	15✔
114	tracks = [meta]	15✔
115
116	names = [i.get("name", "") for i in tracks]	15✔
117	names = cls.split_quoted_titles(names)	15✔
118	names = cls.remove_number_prefix(names)	15✔
119	names = cls.normalize_delimiter(names)	15✔
120	for track, name in zip(tracks, names):	15✔
121	track["name"] = name	15✔
122
123	tracks = cls.common_name_parts(tracks, names)	15✔
124	return cls([Track.from_json(t, Helpers.get_label(meta)) for t in tracks])	15✔
125
126	@cached_property	15✔
127	def first(self) -> Track:	15✔
128	return self.tracks[0]	15✔
129
130	@cached_property	15✔
131	def raw_names(self) -> List[str]:	15✔
132	return [j.name for j in self.tracks]	15✔
133
134	@cached_property	15✔
135	def original_artists(self) -> List[str]:	15✔
136	"""Return all unique unsplit (original) main track artists."""
137	return list(dict.fromkeys(j.artist for j in self.tracks))	15✔
138
139	@property	15✔
140	def artists(self) -> List[str]:	15✔
141	"""Return all unique split main track artists.
142
143	"Artist1 x Artist2" -> ["Artist1", "Artist2"]
144	"""
145	return list(dict.fromkeys(it.chain(*(j.artists for j in self.tracks))))	15✔
146
147	@property	15✔
148	def remixers(self) -> List[str]:	15✔
149	"""Return all remix artists."""
150	return [	15✔
151	t.remix.remixer
152	for t in self.tracks
153	if t.remix and not t.remix.by_other_artist
154	]
155
156	@property	15✔
157	def other_artists(self) -> Set[str]:	15✔
158	"""Return all unique remix and featuring artists."""
159	ft = [j.ft for j in self.tracks if j.ft]	15✔
160	return set(it.chain(self.remixers, ft))	15✔
161
162	@cached_property	15✔
163	def all_artists(self) -> Set[str]:	15✔
164	"""Return all unique (1) track, (2) remix, (3) featuring artists."""
165	return self.other_artists \| set(self.original_artists)	15✔
166
167	@cached_property	15✔
168	def artistitles(self) -> str:	15✔
169	"""Returned artists and titles joined into one long string."""
170	return " ".join(it.chain(self.raw_names, self.all_artists)).lower()	15✔
171
172	@cached_property	15✔
173	def single_catalognum(self) -> Optional[str]:	15✔
174	"""Return catalognum if every track contains the same one."""
175	cats = [t.catalognum for t in self if t.catalognum]	15✔
176	if len(cats) == len(self) and len(set(cats)) == 1:	15✔
177	return cats[0]	15✔
178	return None	15✔
179
180	@cached_property	15✔
181	def albums_in_titles(self) -> Set[str]:	15✔
182	return {t.album for t in self if t.album}	15✔
183
184	def adjust_artists(self, albumartist: str) -> None:	15✔
185	"""Handle some track artist edge cases.
186
187	These checks require knowledge of the entire release, therefore cannot be
188	performed within the context of a single track.
189
190	* When artist name is mistaken for the track_alt
191	* When artist and title are delimited by '-' without spaces
192	* When artist and title are delimited by a UTF-8 dash equivalent
193	* Defaulting to the album artist
194	"""
195	track_alts = {t.track_alt for t in self.tracks if t.track_alt}	15✔
196	artists = [t.artist for t in self.tracks if t.artist]	15✔
197
198	for t in [track for track in self.tracks if not track.artist]:	15✔
199	if t.track_alt and len(track_alts) == 1: # only one track_alt	15!
200	# the only track that parsed a track alt - it's most likely a mistake
201	# one artist was confused for a track alt, like 'B2', - reverse this
UNCOV 202	t.artist, t.track_alt = t.track_alt, None	×
203	elif len(artists) == len(self) - 1: # only 1 missing artist	15!
204	# if this is a remix and the parsed title is part of the albumartist or
205	# is one of the track artists, we made a mistake parsing the remix:
206	# it is most probably the edge case where the `title_without_remix` is a
207	# legitimate artist and the track title is something like 'Hello Remix'
UNCOV 208	if t.remix and (t.title_without_remix in albumartist):	×
209	t.artist, t.title = t.title_without_remix, t.remix.remix	×
210	# this is the only artist that didn't get parsed - relax the rule
211	# and try splitting with '-' without spaces
UNCOV 212	split = t.title.split("-")	×
UNCOV 213	if len(split) == 1:	×
214	# attempt to split by another ' ? ' where '?' may be some utf-8
215	# alternative of a dash
216	split = [s for s in DELIMITER_PAT.split(t.title) if len(s) > 1]	×
UNCOV 217	if len(split) > 1:	×
UNCOV 218	t.artist, t.title = split	×
219	if not t.artist:	15!
220	# default to the albumartist
221	t.artist = albumartist	15✔

snejus / beetcamp / 8306877220

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous