11959482446

Committed 21 Nov 2024 06:33PM UTC coverage: 57.75% (+0.1%) from 57.612%

Build # 11959482446

Build Type

Pull #652

github

Committed by

jakeichikawasalesforce

Commit Message

check windows 32 bit

Pull Request Pull Request #652: Add capability to deploying models remotely

Run Details

24 of 40 new or added lines in 4 files covered. (60.0%)

5 existing lines in 1 file now uncovered.

1427 of 2471 relevant lines covered (57.75%)

0.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.78

/tabpy/tabpy_server/common/endpoint_file_mgr.py

"""
This module provides functionality required for managing endpoint objects in
TabPy. It provides a way to download endpoint files from remote
and then properly cleanup local the endpoint files on update/remove of endpoint
objects.

The local temporary files for TabPy will by default located at
    /tmp/query_objects

"""
import logging
import os
import shutil
from re import compile as _compile


_name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$")


def _check_endpoint_name(name, logger=logging.getLogger(__name__)):
    """Checks that the endpoint name is valid by comparing it with an RE and
    checking that it is not reserved."""
    if not isinstance(name, str):
        msg = "Endpoint name must be a string"
        logger.log(logging.CRITICAL, msg)
        raise TypeError(msg)

    if name == "":
        msg = "Endpoint name cannot be empty"
        logger.log(logging.CRITICAL, msg)
        raise ValueError(msg)

    if not _name_checker.match(name):
        msg = (
            "Endpoint name can only contain: a-z, A-Z, 0-9,"
            " underscore, hyphens and spaces."
        )
        logger.log(logging.CRITICAL, msg)
        raise ValueError(msg)


def grab_files(directory):
    """
    Generator that returns all files in a directory.
    """
    if not os.path.isdir(directory):
        return
    else:
        for name in os.listdir(directory):
            full_path = os.path.join(directory, name)
            if os.path.isdir(full_path):
                for entry in grab_files(full_path):
                    yield entry
            elif os.path.isfile(full_path):
                yield full_path


def cleanup_endpoint_files(
    name, query_path, logger=logging.getLogger(__name__), retain_versions=None
):
    """
    Cleanup the disk space a certain endpiont uses.

    Parameters
    ----------
    name : str
        The endpoint name

    retain_version : int, optional
        If given, then all files for this endpoint are removed except the
        folder for the given version, otherwise, all files for that endpoint
        are removed.
    """
    _check_endpoint_name(name, logger=logger)
    local_dir = os.path.join(query_path, name)

    # nothing to clean, this is true for state file path where we load
    # Query Object directly from the state path instead of downloading
    # to temporary location
    if not os.path.exists(local_dir):
        return

    if not retain_versions:
        shutil.rmtree(local_dir)
    else:
        retain_folders = [
            os.path.join(local_dir, str(version)) for version in retain_versions
        ]
        logger.log(logging.INFO, f"Retain folders: {retain_folders}")

        for file_or_dir in os.listdir(local_dir):
            candidate_dir = os.path.join(local_dir, file_or_dir)
            if os.path.isdir(candidate_dir) and (candidate_dir not in retain_folders):
                shutil.rmtree(candidate_dir)

1	"""
2	This module provides functionality required for managing endpoint objects in
3	TabPy. It provides a way to download endpoint files from remote
4	and then properly cleanup local the endpoint files on update/remove of endpoint
5	objects.
6
7	The local temporary files for TabPy will by default located at
8	/tmp/query_objects
9
10	"""
11	import logging	1✔
12	import os	1✔
13	import shutil	1✔
14	from re import compile as _compile	1✔
15
16
17	_name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$")	1✔
18
19
20	def _check_endpoint_name(name, logger=logging.getLogger(__name__)):	1✔
21	"""Checks that the endpoint name is valid by comparing it with an RE and
22	checking that it is not reserved."""
23	if not isinstance(name, str):	1✔
24	msg = "Endpoint name must be a string"	1✔
25	logger.log(logging.CRITICAL, msg)	1✔
26	raise TypeError(msg)	1✔
27
28	if name == "":	1✔
29	msg = "Endpoint name cannot be empty"	1✔
30	logger.log(logging.CRITICAL, msg)	1✔
31	raise ValueError(msg)	1✔
32
33	if not _name_checker.match(name):	1✔
34	msg = (	1✔
35	"Endpoint name can only contain: a-z, A-Z, 0-9,"
36	" underscore, hyphens and spaces."
37	)
38	logger.log(logging.CRITICAL, msg)	1✔
39	raise ValueError(msg)	1✔
40
41
42	def grab_files(directory):	1✔
43	"""
44	Generator that returns all files in a directory.
45	"""
46	if not os.path.isdir(directory):	×
47	return	×
48	else:
49	for name in os.listdir(directory):	×
50	full_path = os.path.join(directory, name)	×
51	if os.path.isdir(full_path):	×
52	for entry in grab_files(full_path):	×
53	yield entry	×
54	elif os.path.isfile(full_path):	×
55	yield full_path	×
56
57
58	def cleanup_endpoint_files(	1✔
59	name, query_path, logger=logging.getLogger(__name__), retain_versions=None
60	):
61	"""
62	Cleanup the disk space a certain endpiont uses.
63
64	Parameters
65	----------
66	name : str
67	The endpoint name
68
69	retain_version : int, optional
70	If given, then all files for this endpoint are removed except the
71	folder for the given version, otherwise, all files for that endpoint
72	are removed.
73	"""
74	_check_endpoint_name(name, logger=logger)	×
75	local_dir = os.path.join(query_path, name)	×
76
77	# nothing to clean, this is true for state file path where we load
78	# Query Object directly from the state path instead of downloading
79	# to temporary location
80	if not os.path.exists(local_dir):	×
81	return	×
82
83	if not retain_versions:	×
84	shutil.rmtree(local_dir)	×
85	else:
86	retain_folders = [	×
87	os.path.join(local_dir, str(version)) for version in retain_versions
88	]
89	logger.log(logging.INFO, f"Retain folders: {retain_folders}")	×
90
91	for file_or_dir in os.listdir(local_dir):	×
92	candidate_dir = os.path.join(local_dir, file_or_dir)	×
93	if os.path.isdir(candidate_dir) and (candidate_dir not in retain_folders):	×
94	shutil.rmtree(candidate_dir)	×

tableau / TabPy / 11959482446

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous