4845620836

Build # 4845620836

Build Type

github

Committed by GitHub

Commit Message

Merge fd7b05c7e into 39b2bbebd

Pull Request Pull Request #77: Fix dependency error in daily testing

Run Details

3112 of 3668 relevant lines covered (84.84%)

0.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.33

/pypots/cli/doc.py

"""
CLI tools to help the development team build PyPOTS.
"""

# Created by Wenjie Du <wenjay.du@gmail.com>
# License: GLP-v3

import os
import shutil
from argparse import Namespace

from tsdb.data_processing import _download_and_extract

from pypots.cli.base import BaseCommand
from pypots.utils.logging import logger

CLONED_LATEST_PYPOTS = "temp_pypots_latest"

# These files need to be updated while generating the new documentation
DOC_RST_FILES = [
    "pypots.rst",
    "pypots.imputation.rst",
    "pypots.classification.rst",
    "pypots.clustering.rst",
    "pypots.forecasting.rst",
    "pypots.data.rst",
    "pypots.utils.rst",
]

IMPORT_ERROR_MESSAGE = (
    "`pypots-cli doc` command is for PyPOTS developers to build documentation easily. "
    "Therefore, you need a complete PyPOTS development environment. However, you are missing some dependencies. "
    "Please refer to https://github.com/WenjieDu/PyPOTS/blob/main/environment-dev.yml for dependency details. "
)


def doc_command_factory(args: Namespace):
    return DocCommand(
        args.gene_rst,
        args.branch,
        args.gene_html,
        args.view_doc,
        args.port,
        args.cleanup,
    )


def purge_temp_files():
    logger.info(
        f"Directories _build and {CLONED_LATEST_PYPOTS} will be deleted if exist"
    )
    shutil.rmtree("docs/_build", ignore_errors=True)
    shutil.rmtree(CLONED_LATEST_PYPOTS, ignore_errors=True)


class DocCommand(BaseCommand):
    """CLI tools helping build PyPOTS documentation.

    Notes
    -----
    Only code pushed to GitHub can be used for documentation generation.
    During generation, PyPOTS repo on GitHub will firstly be cloned to local with the branch specified by
    argument `--branch`. The generated rst files will replace the old ones directly. To view the updated
    documentation, use option `--view_doc` to generate the doc into HTMLs and deploy it locally for view.
    After running `--view_doc`, the generated static files won't be deleted automatically. To purge them,
    run the command with option `--cleanup`.

    Examples
    --------
    $ pypots-cli doc --gene_rst [--branch main]
    $ pypots-cli doc --view_doc [--port 9075]
    $ pypots-cli doc --cleanup

    """

    @staticmethod
    def register_subcommand(parser):
        sub_parser = parser.add_parser(
            "doc",
            help="CLI tools helping build PyPOTS documentation",
            allow_abbrev=True,
        )

        sub_parser.add_argument(
            "--gene-rst",
            "--gene_rst",
            dest="gene_rst",
            action="store_true",
            help="Generate rst (reStructuredText) documentation according to the latest code on Github",
        )
        sub_parser.add_argument(
            "-b",
            "--branch",
            type=str,
            default="main",
            choices=["main", "dev"],
            help="Code on which branch will be used for documentation generating",
        )
        sub_parser.add_argument(
            "--gene-html",
            "--gene_html",
            dest="gene_html",
            action="store_true",
            help="Generate the sphinx documentation into static HTML files",
        )
        sub_parser.add_argument(
            "--view-doc",
            "--view_doc",
            dest="view_doc",
            action="store_true",
            help="Deploy the generated HTML documentation locally for view",
        )
        sub_parser.add_argument(
            "-p",
            "--port",
            type=int,
            default=9075,
            help="Use which port to deploy the web server for doc view",  # 9075 looks like "POTS", so use it as default
        )
        sub_parser.add_argument(
            "-c",
            "--cleanup",
            dest="cleanup",
            action="store_true",
            help="Delete all caches and static resources like HTML and CSS files ",
        )

        sub_parser.set_defaults(func=doc_command_factory)

    def __init__(
        self,
        gene_rst: bool,
        branch: str,
        gene_html: bool,
        view_doc: bool,
        port: int,
        cleanup: bool,
    ):
        self._gene_rst = gene_rst
        self._branch = branch
        self._gene_html = gene_html
        self._view_doc = view_doc
        self._port = port
        self._cleanup = cleanup

    def checkup(self):
        """Run some checks on the arguments to avoid error usages"""
        self.check_if_under_root_dir(strict=True)

        if self._cleanup:
            assert not self._gene_rst and not self._gene_html and not self._view_doc, (
                "Argument `--cleanup` should be used alone. "
                "Try `pypots-cli doc --cleanup`"
            )

    def run(self):
        """Execute the given command."""
        # run checks first
        self.checkup()

        try:
            if self._cleanup:
                logger.info("Purging static files...")
                purge_temp_files()
                logger.info("Purging finished successfully.")

            if self._gene_rst:
                if os.path.exists(CLONED_LATEST_PYPOTS):
                    logger.info(
                        f"Directory {CLONED_LATEST_PYPOTS} exists, deleting it..."
                    )
                    shutil.rmtree(CLONED_LATEST_PYPOTS, ignore_errors=True)

                # Download the latest code from GitHub
                logger.info(
                    f"Downloading PyPOTS with the latest code on branch '{self._branch}' "
                    f"from GitHub into {CLONED_LATEST_PYPOTS}..."
                )
                url = f"https://github.com/WenjieDu/PyPOTS/archive/refs/heads/{self._branch}.zip"
                _download_and_extract(url=url, saving_path=CLONED_LATEST_PYPOTS)

                code_dir = f"{CLONED_LATEST_PYPOTS}/PyPOTS-{self._branch}"
                files_to_move = os.listdir(code_dir)
                destination_dir = os.path.join(os.getcwd(), CLONED_LATEST_PYPOTS)
                for f_ in files_to_move:
                    shutil.move(os.path.join(code_dir, f_), destination_dir)
                # delete code in tests because we don't need its doc
                shutil.rmtree(
                    f"{CLONED_LATEST_PYPOTS}/pypots/tests", ignore_errors=True
                )

                # Generate the docs according to the cloned code
                logger.info("Generating rst files...")
                os.environ[
                    "SPHINX_APIDOC_OPTIONS"
                ] = "members,undoc-members,show-inheritance,inherited-members"
                self.execute_command(
                    f"sphinx-apidoc {CLONED_LATEST_PYPOTS} -o {CLONED_LATEST_PYPOTS}/rst"
                )

                # Only save the files we need.
                logger.info("Updating the old documentation...")
                for f_ in DOC_RST_FILES:
                    file_to_copy = f"{CLONED_LATEST_PYPOTS}/rst/{f_}"
                    shutil.copy(file_to_copy, "docs")

                # Delete the useless files.
                shutil.rmtree(f"{CLONED_LATEST_PYPOTS}", ignore_errors=True)

            if self._gene_html:
                logger.info("Generating static HTML files...")
                purge_temp_files()
                self.execute_command("cd docs && make html")

            if self._view_doc:
                assert os.path.exists(
                    "docs/_build/html"
                ), "docs/_build/html does not exists, please run `pypots-cli doc --gene_html` first"
                logger.info(f"Deploying HTML to http://127.0.0.1:{self._port}...")
                self.execute_command(
                    f"python -m http.server {self._port} -d docs/_build/html -b 127.0.0.1"
                )

        except ImportError:
            raise ImportError(IMPORT_ERROR_MESSAGE)
        except Exception as e:
            raise RuntimeError(e)

1	"""	1✔
2	CLI tools to help the development team build PyPOTS.
3	"""
4
5	# Created by Wenjie Du <wenjay.du@gmail.com>
6	# License: GLP-v3
7
8	import os	1✔
9	import shutil	1✔
10	from argparse import Namespace	1✔
11
12	from tsdb.data_processing import _download_and_extract	1✔
13
14	from pypots.cli.base import BaseCommand	1✔
15	from pypots.utils.logging import logger	1✔
16
17	CLONED_LATEST_PYPOTS = "temp_pypots_latest"	1✔
18
19	# These files need to be updated while generating the new documentation
20	DOC_RST_FILES = [	1✔
21	"pypots.rst",
22	"pypots.imputation.rst",
23	"pypots.classification.rst",
24	"pypots.clustering.rst",
25	"pypots.forecasting.rst",
26	"pypots.data.rst",
27	"pypots.utils.rst",
28	]
29
30	IMPORT_ERROR_MESSAGE = (	1✔
31	"`pypots-cli doc` command is for PyPOTS developers to build documentation easily. "
32	"Therefore, you need a complete PyPOTS development environment. However, you are missing some dependencies. "
33	"Please refer to https://github.com/WenjieDu/PyPOTS/blob/main/environment-dev.yml for dependency details. "
34	)
35
36
37	def doc_command_factory(args: Namespace):	1✔
38	return DocCommand(	1✔
39	args.gene_rst,
40	args.branch,
41	args.gene_html,
42	args.view_doc,
43	args.port,
44	args.cleanup,
45	)
46
47
48	def purge_temp_files():	1✔
49	logger.info(	1✔
50	f"Directories _build and {CLONED_LATEST_PYPOTS} will be deleted if exist"
51	)
52	shutil.rmtree("docs/_build", ignore_errors=True)	1✔
53	shutil.rmtree(CLONED_LATEST_PYPOTS, ignore_errors=True)	1✔
54
55
56	class DocCommand(BaseCommand):	1✔
57	"""CLI tools helping build PyPOTS documentation.	1✔
58
59	Notes
60	-----
61	Only code pushed to GitHub can be used for documentation generation.
62	During generation, PyPOTS repo on GitHub will firstly be cloned to local with the branch specified by
63	argument `--branch`. The generated rst files will replace the old ones directly. To view the updated
64	documentation, use option `--view_doc` to generate the doc into HTMLs and deploy it locally for view.
65	After running `--view_doc`, the generated static files won't be deleted automatically. To purge them,
66	run the command with option `--cleanup`.
67
68	Examples
69	--------
70	$ pypots-cli doc --gene_rst [--branch main]
71	$ pypots-cli doc --view_doc [--port 9075]
72	$ pypots-cli doc --cleanup
73
74	"""
75
76	@staticmethod	1✔
77	def register_subcommand(parser):
78	sub_parser = parser.add_parser(	×
79	"doc",
80	help="CLI tools helping build PyPOTS documentation",
81	allow_abbrev=True,
82	)
83
84	sub_parser.add_argument(	×
85	"--gene-rst",
86	"--gene_rst",
87	dest="gene_rst",
88	action="store_true",
89	help="Generate rst (reStructuredText) documentation according to the latest code on Github",
90	)
91	sub_parser.add_argument(	×
92	"-b",
93	"--branch",
94	type=str,
95	default="main",
96	choices=["main", "dev"],
97	help="Code on which branch will be used for documentation generating",
98	)
99	sub_parser.add_argument(	×
100	"--gene-html",
101	"--gene_html",
102	dest="gene_html",
103	action="store_true",
104	help="Generate the sphinx documentation into static HTML files",
105	)
106	sub_parser.add_argument(	×
107	"--view-doc",
108	"--view_doc",
109	dest="view_doc",
110	action="store_true",
111	help="Deploy the generated HTML documentation locally for view",
112	)
113	sub_parser.add_argument(	×
114	"-p",
115	"--port",
116	type=int,
117	default=9075,
118	help="Use which port to deploy the web server for doc view", # 9075 looks like "POTS", so use it as default
119	)
120	sub_parser.add_argument(	×
121	"-c",
122	"--cleanup",
123	dest="cleanup",
124	action="store_true",
125	help="Delete all caches and static resources like HTML and CSS files ",
126	)
127
128	sub_parser.set_defaults(func=doc_command_factory)	×
129
130	def __init__(	1✔
131	self,
132	gene_rst: bool,
133	branch: str,
134	gene_html: bool,
135	view_doc: bool,
136	port: int,
137	cleanup: bool,
138	):
139	self._gene_rst = gene_rst	1✔
140	self._branch = branch	1✔
141	self._gene_html = gene_html	1✔
142	self._view_doc = view_doc	1✔
143	self._port = port	1✔
144	self._cleanup = cleanup	1✔
145
146	def checkup(self):	1✔
147	"""Run some checks on the arguments to avoid error usages"""
148	self.check_if_under_root_dir(strict=True)	1✔
149
150	if self._cleanup:	1✔
151	assert not self._gene_rst and not self._gene_html and not self._view_doc, (	1✔
152	"Argument `--cleanup` should be used alone. "
153	"Try `pypots-cli doc --cleanup`"
154	)
155
156	def run(self):	1✔
157	"""Execute the given command."""
158	# run checks first
159	self.checkup()	1✔
160
161	try:	1✔
162	if self._cleanup:	1✔
163	logger.info("Purging static files...")	1✔
164	purge_temp_files()	1✔
165	logger.info("Purging finished successfully.")	1✔
166
167	if self._gene_rst:	1✔
168	if os.path.exists(CLONED_LATEST_PYPOTS):	1✔
169	logger.info(	×
170	f"Directory {CLONED_LATEST_PYPOTS} exists, deleting it..."
171	)
172	shutil.rmtree(CLONED_LATEST_PYPOTS, ignore_errors=True)	×
173
174	# Download the latest code from GitHub
175	logger.info(	1✔
176	f"Downloading PyPOTS with the latest code on branch '{self._branch}' "
177	f"from GitHub into {CLONED_LATEST_PYPOTS}..."
178	)
179	url = f"https://github.com/WenjieDu/PyPOTS/archive/refs/heads/{self._branch}.zip"	1✔
180	_download_and_extract(url=url, saving_path=CLONED_LATEST_PYPOTS)	1✔
181
182	code_dir = f"{CLONED_LATEST_PYPOTS}/PyPOTS-{self._branch}"	1✔
183	files_to_move = os.listdir(code_dir)	1✔
184	destination_dir = os.path.join(os.getcwd(), CLONED_LATEST_PYPOTS)	1✔
185	for f_ in files_to_move:	1✔
186	shutil.move(os.path.join(code_dir, f_), destination_dir)	1✔
187	# delete code in tests because we don't need its doc
188	shutil.rmtree(	1✔
189	f"{CLONED_LATEST_PYPOTS}/pypots/tests", ignore_errors=True
190	)
191
192	# Generate the docs according to the cloned code
193	logger.info("Generating rst files...")	1✔
194	os.environ[	1✔
195	"SPHINX_APIDOC_OPTIONS"
196	] = "members,undoc-members,show-inheritance,inherited-members"
197	self.execute_command(	1✔
198	f"sphinx-apidoc {CLONED_LATEST_PYPOTS} -o {CLONED_LATEST_PYPOTS}/rst"
199	)
200
201	# Only save the files we need.
202	logger.info("Updating the old documentation...")	1✔
203	for f_ in DOC_RST_FILES:	1✔
204	file_to_copy = f"{CLONED_LATEST_PYPOTS}/rst/{f_}"	1✔
205	shutil.copy(file_to_copy, "docs")	1✔
206
207	# Delete the useless files.
208	shutil.rmtree(f"{CLONED_LATEST_PYPOTS}", ignore_errors=True)	1✔
209
210	if self._gene_html:	1✔
211	logger.info("Generating static HTML files...")	1✔
212	purge_temp_files()	1✔
213	self.execute_command("cd docs && make html")	1✔
214
215	if self._view_doc:	1✔
216	assert os.path.exists(	1✔
217	"docs/_build/html"
218	), "docs/_build/html does not exists, please run `pypots-cli doc --gene_html` first"
219	logger.info(f"Deploying HTML to http://127.0.0.1:{self._port}...")	×
220	self.execute_command(	×
221	f"python -m http.server {self._port} -d docs/_build/html -b 127.0.0.1"
222	)
223
224	except ImportError:	1✔
225	raise ImportError(IMPORT_ERROR_MESSAGE)	×
226	except Exception as e:	1✔
227	raise RuntimeError(e)	1✔

WenjieDu / PyPOTS / 4845620836

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous