13780152182

Committed 11 Mar 2025 04:08AM UTC coverage: 94.439% (-0.09%) from 94.524%

Build # 13780152182

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #102 from J-CPelletier/fix-windows-click

Set windows_expand_args to False

Run Details

1 of 2 new or added lines in 1 file covered. (50.0%)

985 of 1043 relevant lines covered (94.44%)

4.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.8

/webcomix/cli.py

#! python3

import click

from webcomix.comic import Comic
from webcomix.exceptions import CrawlerBlocked, NextLinkNotFound
from webcomix.search import discovery
from webcomix.supported_comics import supported_comics
from webcomix.docker import DockerManager


@click.group()
@click.version_option()
def cli():
    pass


@cli.command()
def comics():
    """
    Shows all predefined comics
    """
    comics_content = [
        "{}: {}".format(key, value["start_url"])
        for key, value in sorted(supported_comics.items())
    ]

    click.echo("\n".join(comics_content))


@cli.command()
@click.argument("name", type=click.Choice(list(supported_comics.keys())))
@click.option(
    "--cbz", is_flag=True, default=False, help="Outputs the comic as a cbz file"
)
@click.option(
    "--title", is_flag=True, default=False, help="Add title of comic in image names"
)
@click.option(
    "--verbose", "-v", is_flag=True, default=False, help="Add debugging output"
)
def download(name, cbz, title, verbose):
    """
    Downloads a predefined comic by name
    """
    if name in list(supported_comics.keys()):
        comic = Comic(**supported_comics[name], title=title, debug=verbose)
        download_webcomic(comic, cbz)


@cli.command()
@click.argument("name", type=click.STRING)
@click.option(
    "--start-url",
    "--start_url",
    prompt=True,
    type=click.STRING,
    help="URL of the comic's first page",
)
@click.option(
    "--end-url",
    "--end_url",
    prompt=False,
    type=click.STRING,
    help="URL of the comic's last page",
)
@click.option(
    "--start-page",
    "--start_page",
    type=click.INT,
    default=1,
    help="Number of comic's first page to be downloaded",
)
@click.option(
    "--cbz", default=False, is_flag=True, help="Outputs the comic as a cbz file"
)
@click.option(
    "--single-page",
    "--single_page",
    default=False,
    is_flag=True,
    help="Downloads from a single webpage",
)
@click.option(
    "--javascript",
    "--js",
    "-j",
    default=False,
    is_flag=True,
    help="Renders javascript in the page (slower)",
)
@click.option(
    "--title", is_flag=True, default=False, help="Add title of comic in image names"
)
@click.option(
    "--alt-text",
    "-a",
    default=None,
    type=click.STRING,
    help="Optional XPath to fetch an additionnal text while scraping",
)
@click.option(
    "--yes", "-y", default=False, is_flag=True, help="Skips the verification prompt"
)
@click.option(
    "--verbose", "-v", is_flag=True, default=False, help="Add debugging output"
)
def search(
    name,
    start_url,
    end_url,
    start_page,
    cbz,
    single_page,
    javascript,
    title,
    alt_text,
    yes,
    verbose,
):
    """
    Downloads a webcomic using a general XPath
    """
    with DockerManager(javascript):
        comic, validation = discovery(
            name,
            start_url,
            end_url=end_url,
            start_page=start_page,
            alt_text=alt_text,
            single_page=single_page,
            javascript=javascript,
            title=title,
            debug=verbose,
        )
        if comic is not None:
            print_verification(validation)
            click.echo("Verify that the links above are correct.")
            if yes or click.confirm("Are you sure you want to proceed?"):
                download_webcomic(comic, cbz)


@cli.command()
@click.argument("name", type=click.STRING)
@click.option(
    "--start-url",
    "--start_url",
    prompt=True,
    type=click.STRING,
    help="URL of the comic's first page to be downloaded",
)
@click.option(
    "--end-url",
    "--end_url",
    prompt=False,
    type=click.STRING,
    help="URL of the last comic's page to be downloaded",
)
@click.option(
    "--start-page",
    "--start_page",
    type=click.INT,
    default=1,
    help="Number of comic's first page to be downloaded",
)
@click.option(
    "--image-xpath",
    "--image_xpath",
    prompt=True,
    type=click.STRING,
    help="XPath expression giving the url to the image",
)
@click.option(
    "--next-page-xpath",
    "--next_page_xpath",
    prompt=True,
    type=click.STRING,
    help="XPath expression giving the url to the next page",
)
@click.option(
    "--block-xpath",
    "--block_xpath",
    "-x",
    type=click.STRING,
    multiple=True,
    help="XPath expressions that block the download of an image when any expression has elements",
)
@click.option(
    "--cbz", default=False, is_flag=True, help="Outputs the comic as a cbz file"
)
@click.option(
    "--single-page",
    "--single_page",
    "-s",
    default=False,
    is_flag=True,
    help="Downloads from a single webpage",
)
@click.option(
    "--javascript",
    "--js",
    "-j",
    default=False,
    is_flag=True,
    help="Renders javascript in the page (slower)",
)
@click.option(
    "--title", is_flag=True, default=False, help="Add title of comic in image names"
)
@click.option(
    "--alt-text",
    "-a",
    default=None,
    type=click.STRING,
    help="XPath to fetch an additionnal text while scraping",
)
@click.option(
    "--cookie",
    "-c",
    default=[],
    type=click.Tuple([str, str]),
    multiple=True,
    help="Value of cookie to add to the requests",
)
@click.option(
    "--yes", "-y", default=False, is_flag=True, help="Skips the verification prompt"
)
@click.option(
    "--verbose", "-v", is_flag=True, default=False, help="Add debugging output"
)
def custom(
    name,
    start_url,
    end_url,
    start_page,
    next_page_xpath,
    image_xpath,
    block_xpath,
    cbz,
    single_page,
    javascript,
    title,
    alt_text,
    cookie,
    yes,
    verbose,
):
    """
    Downloads a user-defined webcomic
    """
    with DockerManager(javascript):
        comic = Comic(
            name,
            start_url,
            image_xpath,
            next_page_xpath,
            end_url=end_url,
            block_selectors=block_xpath,
            start_page=start_page,
            alt_text=alt_text,
            single_page=single_page,
            javascript=javascript,
            title=title,
            cookies=cookie,
            debug=verbose,
        )
        try:
            validation = comic.verify_xpath()
        except NextLinkNotFound as exception:
            click.echo(
                "Could not find next link of: {} \nwith next page XPath expression: {}".format(
                    exception.failed_url, exception.next_page_xpath
                )
            )
            click.echo(
                "Have you tried testing your XPath expression with 'scrapy shell'?"
            )
            raise click.Abort()
        try:
            print_verification(validation)
        except CrawlerBlocked as exception:
            click.echo("{} could not be accessed with webcomix.".format(name))
            click.echo(
                "Chances are the website you're trying to download images from doesn't want to be scraped."
            )
            raise click.Abort()
        click.echo("Verify that the links above are correct.")
        if yes or click.confirm("Are you sure you want to proceed?"):
            download_webcomic(comic, cbz)


def main():
    cli(windows_expand_args=False)

def print_verification(validation):
    """
    Prints the verification given by the verify_xpath function
    """
    if validation is None:
        raise CrawlerBlocked()
    for item in sorted(validation, key=lambda x: x.get("page")):
        output = "Page {}:\nPage URL: {}\nImage URLs:\n{}\n".format(
            item.get("page"), item.get("url"), "\n".join(item.get("image_urls"))
        )
        if item.get("alt_text") is not None:
            output += "Alt text: {}\n".format(item.get("alt_text"))
        click.echo(output)


def download_webcomic(comic, cbz):
    comic.download()
    if cbz:
        comic.convert_to_cbz()

1	#! python3
2
3	import click	5✔
4
5	from webcomix.comic import Comic	5✔
6	from webcomix.exceptions import CrawlerBlocked, NextLinkNotFound	5✔
7	from webcomix.search import discovery	5✔
8	from webcomix.supported_comics import supported_comics	5✔
9	from webcomix.docker import DockerManager	5✔
10
11
12	@click.group()	5✔
13	@click.version_option()	5✔
14	def cli():	5✔
15	pass	×
16
17
18	@cli.command()	5✔
19	def comics():	5✔
20	"""
21	Shows all predefined comics
22	"""
23	comics_content = [	5✔
24	"{}: {}".format(key, value["start_url"])
25	for key, value in sorted(supported_comics.items())
26	]
27
28	click.echo("\n".join(comics_content))	5✔
29
30
31	@cli.command()	5✔
32	@click.argument("name", type=click.Choice(list(supported_comics.keys())))	5✔
33	@click.option(	5✔
34	"--cbz", is_flag=True, default=False, help="Outputs the comic as a cbz file"
35	)
36	@click.option(	5✔
37	"--title", is_flag=True, default=False, help="Add title of comic in image names"
38	)
39	@click.option(	5✔
40	"--verbose", "-v", is_flag=True, default=False, help="Add debugging output"
41	)
42	def download(name, cbz, title, verbose):	5✔
43	"""
44	Downloads a predefined comic by name
45	"""
46	if name in list(supported_comics.keys()):	5✔
47	comic = Comic(**supported_comics[name], title=title, debug=verbose)	5✔
48	download_webcomic(comic, cbz)	5✔
49
50
51	@cli.command()	5✔
52	@click.argument("name", type=click.STRING)	5✔
53	@click.option(	5✔
54	"--start-url",
55	"--start_url",
56	prompt=True,
57	type=click.STRING,
58	help="URL of the comic's first page",
59	)
60	@click.option(	5✔
61	"--end-url",
62	"--end_url",
63	prompt=False,
64	type=click.STRING,
65	help="URL of the comic's last page",
66	)
67	@click.option(	5✔
68	"--start-page",
69	"--start_page",
70	type=click.INT,
71	default=1,
72	help="Number of comic's first page to be downloaded",
73	)
74	@click.option(	5✔
75	"--cbz", default=False, is_flag=True, help="Outputs the comic as a cbz file"
76	)
77	@click.option(	5✔
78	"--single-page",
79	"--single_page",
80	default=False,
81	is_flag=True,
82	help="Downloads from a single webpage",
83	)
84	@click.option(	5✔
85	"--javascript",
86	"--js",
87	"-j",
88	default=False,
89	is_flag=True,
90	help="Renders javascript in the page (slower)",
91	)
92	@click.option(	5✔
93	"--title", is_flag=True, default=False, help="Add title of comic in image names"
94	)
95	@click.option(	5✔
96	"--alt-text",
97	"-a",
98	default=None,
99	type=click.STRING,
100	help="Optional XPath to fetch an additionnal text while scraping",
101	)
102	@click.option(	5✔
103	"--yes", "-y", default=False, is_flag=True, help="Skips the verification prompt"
104	)
105	@click.option(	5✔
106	"--verbose", "-v", is_flag=True, default=False, help="Add debugging output"
107	)
108	def search(	5✔
109	name,
110	start_url,
111	end_url,
112	start_page,
113	cbz,
114	single_page,
115	javascript,
116	title,
117	alt_text,
118	yes,
119	verbose,
120	):
121	"""
122	Downloads a webcomic using a general XPath
123	"""
124	with DockerManager(javascript):	5✔
125	comic, validation = discovery(	5✔
126	name,
127	start_url,
128	end_url=end_url,
129	start_page=start_page,
130	alt_text=alt_text,
131	single_page=single_page,
132	javascript=javascript,
133	title=title,
134	debug=verbose,
135	)
136	if comic is not None:	5✔
137	print_verification(validation)	5✔
138	click.echo("Verify that the links above are correct.")	5✔
139	if yes or click.confirm("Are you sure you want to proceed?"):	5✔
140	download_webcomic(comic, cbz)	5✔
141
142
143	@cli.command()	5✔
144	@click.argument("name", type=click.STRING)	5✔
145	@click.option(	5✔
146	"--start-url",
147	"--start_url",
148	prompt=True,
149	type=click.STRING,
150	help="URL of the comic's first page to be downloaded",
151	)
152	@click.option(	5✔
153	"--end-url",
154	"--end_url",
155	prompt=False,
156	type=click.STRING,
157	help="URL of the last comic's page to be downloaded",
158	)
159	@click.option(	5✔
160	"--start-page",
161	"--start_page",
162	type=click.INT,
163	default=1,
164	help="Number of comic's first page to be downloaded",
165	)
166	@click.option(	5✔
167	"--image-xpath",
168	"--image_xpath",
169	prompt=True,
170	type=click.STRING,
171	help="XPath expression giving the url to the image",
172	)
173	@click.option(	5✔
174	"--next-page-xpath",
175	"--next_page_xpath",
176	prompt=True,
177	type=click.STRING,
178	help="XPath expression giving the url to the next page",
179	)
180	@click.option(	5✔
181	"--block-xpath",
182	"--block_xpath",
183	"-x",
184	type=click.STRING,
185	multiple=True,
186	help="XPath expressions that block the download of an image when any expression has elements",
187	)
188	@click.option(	5✔
189	"--cbz", default=False, is_flag=True, help="Outputs the comic as a cbz file"
190	)
191	@click.option(	5✔
192	"--single-page",
193	"--single_page",
194	"-s",
195	default=False,
196	is_flag=True,
197	help="Downloads from a single webpage",
198	)
199	@click.option(	5✔
200	"--javascript",
201	"--js",
202	"-j",
203	default=False,
204	is_flag=True,
205	help="Renders javascript in the page (slower)",
206	)
207	@click.option(	5✔
208	"--title", is_flag=True, default=False, help="Add title of comic in image names"
209	)
210	@click.option(	5✔
211	"--alt-text",
212	"-a",
213	default=None,
214	type=click.STRING,
215	help="XPath to fetch an additionnal text while scraping",
216	)
217	@click.option(	5✔
218	"--cookie",
219	"-c",
220	default=[],
221	type=click.Tuple([str, str]),
222	multiple=True,
223	help="Value of cookie to add to the requests",
224	)
225	@click.option(	5✔
226	"--yes", "-y", default=False, is_flag=True, help="Skips the verification prompt"
227	)
228	@click.option(	5✔
229	"--verbose", "-v", is_flag=True, default=False, help="Add debugging output"
230	)
231	def custom(	5✔
232	name,
233	start_url,
234	end_url,
235	start_page,
236	next_page_xpath,
237	image_xpath,
238	block_xpath,
239	cbz,
240	single_page,
241	javascript,
242	title,
243	alt_text,
244	cookie,
245	yes,
246	verbose,
247	):
248	"""
249	Downloads a user-defined webcomic
250	"""
251	with DockerManager(javascript):	5✔
252	comic = Comic(	5✔
253	name,
254	start_url,
255	image_xpath,
256	next_page_xpath,
257	end_url=end_url,
258	block_selectors=block_xpath,
259	start_page=start_page,
260	alt_text=alt_text,
261	single_page=single_page,
262	javascript=javascript,
263	title=title,
264	cookies=cookie,
265	debug=verbose,
266	)
267	try:	5✔
268	validation = comic.verify_xpath()	5✔
269	except NextLinkNotFound as exception:	5✔
270	click.echo(	5✔
271	"Could not find next link of: {} \nwith next page XPath expression: {}".format(
272	exception.failed_url, exception.next_page_xpath
273	)
274	)
275	click.echo(	5✔
276	"Have you tried testing your XPath expression with 'scrapy shell'?"
277	)
278	raise click.Abort()	5✔
279	try:	5✔
280	print_verification(validation)	5✔
281	except CrawlerBlocked as exception:	5✔
282	click.echo("{} could not be accessed with webcomix.".format(name))	5✔
283	click.echo(	5✔
284	"Chances are the website you're trying to download images from doesn't want to be scraped."
285	)
286	raise click.Abort()	5✔
287	click.echo("Verify that the links above are correct.")	5✔
288	if yes or click.confirm("Are you sure you want to proceed?"):	5✔
289	download_webcomic(comic, cbz)	5✔
290
291
292	def main():	5✔
NEW 293	cli(windows_expand_args=False)	×
294
295	def print_verification(validation):	5✔
296	"""
297	Prints the verification given by the verify_xpath function
298	"""
299	if validation is None:	5✔
300	raise CrawlerBlocked()	5✔
301	for item in sorted(validation, key=lambda x: x.get("page")):	5✔
302	output = "Page {}:\nPage URL: {}\nImage URLs:\n{}\n".format(	5✔
303	item.get("page"), item.get("url"), "\n".join(item.get("image_urls"))
304	)
305	if item.get("alt_text") is not None:	5✔
306	output += "Alt text: {}\n".format(item.get("alt_text"))	5✔
307	click.echo(output)	5✔
308
309
310	def download_webcomic(comic, cbz):	5✔
311	comic.download()	5✔
312	if cbz:	5✔
313	comic.convert_to_cbz()	5✔

J-CPelletier / webcomix / 13780152182

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous