Kintyre / ksconf / 307

pending completion

Build # 307

Build Type

push

travis-ci

Committed by

lowell80

Commit Message

<a href="https://github.com/Kintyre/ksconf/commit/<a class=hub.com/Kintyre/ksconf/commit/<a class="double-link" href="https://git"><a class=hub.com/Kintyre/ksconf/commit/20be65a55b86f18d8a6514348744c494198e9cc4">20be65a55<a href="https://github.com/Kintyre/ksconf/commit/20be65a55b86f18d8a6514348744c494198e9cc4">&lt;a href=&quot;https://github.com/Kintyre/ksconf/commit/20be65a55b86f18d8a6514348744c494198e9cc4&quot;&gt;&amp;quot;&amp;gt;Add &amp;amp;#39;rest-publish&amp;amp;#39; command

Supports:

- Pushing multiple stanzas to a specific config endpoint at once, in a fairly
  efficient process.
- Changes made on the server are show in the form of a diff.  (Eventually, this
  should be configurable, and a dry-run mode should be added.)
- Supports pushing to a live Splunk instance (via Splunkd REST API).  This
  should replace the use of &amp;amp;#39;rest-export&amp;amp;#39; for MOST use cases.
- Supports reading and applying metadata (ACLs, owner, shared, app) to imported
  objects.
- Any type of conf entry should be supported.

Limitations:

- NO UNIT TESTING!  (This will require some fancy mork work.)
- Doesn&amp;amp;#39;t support attribute-level metadata.
- No SSL validation setup yet.  This is primarily because of how Splunk&amp;#39;s
  Python SDK works.  For other Python HTTP libraries this is enabled by default.

Squashed commit of the following:

commit &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/Kintyre/ksconf/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/Kintyre/ksconf/commit/5ef130cfc1c526f4d1d026820f6f219038ac040b&quot;&gt;5ef130cfc&lt;/a&gt;&quot;&gt;5ef130cfc&lt;/a&gt;&lt;a href=&quot;https://github.com/Kintyre/ksconf/commit/20be65a55b86f18d8a6514348744c494198e9cc4&quot;&gt;
Author: Lowell Alleman &amp;lt;lowell@kintyre.co&amp;gt;
Date:   Mon Feb 25 18:39:54 2019 -0500

    Fixed bugs in rest-publish

    - Fixed issue where rest-publish would fail if &amp;#39;--conf&amp;#39; wasn&amp;#39;t explicitly set.
      Has something to do with encoding.  Splunk SDK is looking for a non-unicode
      value for conf-type.  Whatever.
    - Fixed corner-case thing where ACL has &#39;read&#39; or &#39;write&#39; is an empty list.

commit </a><a class="double-link" href="https://github.com/Kintyre/ksconf/commit/<a class="double-link" href="https://github.com/Kintyre/ksconf/commit/d0a3df2078f2c148214c7bd095fcca714bd355d8">d0a3df207</a>">d0a3df207</a><a href="https://github.com/Kintyre/ksconf/commit/20be65a55b86f18d8a6514348744c494198e9cc4">
Author: Lowell Alleman <lowell@kintyre.co>
Date:   Mon Feb 25 18:08:56 2019 -0500

    Reduce server roun... (continued)

Run Details

205 of 205 new or added lines in 4 files covered. (100.0%)

3436 of 3818 relevant lines covered (89.99%)

7.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.51

/ksconf/commands/__init__.py

from __future__ import absolute_import, unicode_literals

import argparse
import logging
import os
import sys
import textwrap

from io import open
from textwrap import dedent
from warnings import warn
from collections import namedtuple

from ksconf import KsconfPluginWarning
from ksconf.conf.parser import parse_conf, smart_write_conf, write_conf, ConfParserException, \
                               detect_by_bom
from ksconf.consts import SMART_CREATE
from ksconf.util import memoize, debug_traceback

__all__ = [
    "KsconfCmd",
    "ConfDirProxy",
    "ConfFileProxy",
    "ConfFileType",
    "dedent",
    "get_all_ksconf_cmds",
    "get_entrypoints",
    "add_splunkd_access_args",
    "add_splunkd_namespace",
]


class ConfDirProxy(object):
    def __init__(self, name, mode, parse_profile=None):
        self.name = name
        self._mode = mode
        self._parse_profile = parse_profile

    def get_file(self, relpath):
        path = os.path.join(self.name, relpath)
        return ConfFileProxy(path, self._mode, parse_profile=self._parse_profile, is_file=True)


class ConfFileProxy(object):
    def __init__(self, name, mode, stream=None, parse_profile=None, is_file=None):
        self.name = name
        self._mode = mode
        if is_file is not None:
            self._is_file = is_file
        elif stream:
            self._is_file = False
        elif self.writable() or os.path.isfile(name):
            self._is_file = True
        else:
            self._is_file = False
        self._stream = stream

        # Not sure if there's a good reason to keep a copy of the data locally?
        self._data = None
        self._parse_profile = parse_profile or {}

    def readable(self):
        return "r" in self._mode

    def writable(self):
        return "+" in self._mode or "w" in self._mode

    def is_file(self):
        # Is "seekable" a more appropriate distinction?  (match IOBase)
        return self._is_file

    def _type(self):    # pragma: no cover  (only used in exceptions)
        if self._is_file:
            return "file"
        else:
            return "stream"

    def close(self):
        if self._stream:
            if not self._stream.closed:
                try:
                    self._stream.close()
                finally:
                    del self._stream
        self._stream = None

    def reset(self):
        if self._data is not None:
            self._data = None
            if self.is_file():
                self.close()
            else:
                self.stream.seek(0)

    def set_parser_option(self, **kwargs):
        """ Setting a key to None will remove that setting. """
        profile = dict(self._parse_profile)
        for (k, v) in kwargs.items():
            if v is None:
                if k in profile:
                    del profile[k]
            else:
                cv = profile.get(k, None)
                if cv != v:
                    profile[k] = v
        if self._parse_profile != profile:
            self._parse_profile = profile
            self.reset()

    @property
    def stream(self):
        if self._stream is None:
            self._stream = open(self.name, self._mode)
        return self._stream

    @property
    def data(self):
        if self._data is None:
            self._data = self.load()
        return self._data

    def load(self, profile=None):
        if not self.readable():
            # Q: Should we mimic the exception caused by doing a read() on a write-only file object?
            raise ValueError("Unable to load() from {} with mode '{}'".format(self._type(),
                                                                              self._mode))
        parse_profile = dict(self._parse_profile)
        if profile:
            parse_profile.update(profile)
        data = parse_conf(self.stream, profile=parse_profile)
        return data

    def dump(self, data):
        if not self.writable():      # pragma: no cover
            raise ValueError("Unable to dump() to {} with mode '{}'".format(self._type(),
                                                                            self._mode))
        # Feels like the right thing to do????  OR self._data = data
        self._data = None
        # write vs smart write here ----
        if self._is_file:
            self.close()
            return smart_write_conf(self.name, data)
        else:
            write_conf(self._stream, data)
            return SMART_CREATE

    def unlink(self):
        # Eventually this could trigger some kind of backup or recovery mechanism
        self.close()
        return os.unlink(self.name)

    '''
    def backup(self, bkname=None):
        # One option:  Write this file directly to the git object store.  Just need to store some
        # kind of index to allow the users to pull it back.   (Sill, would need of fall-back
        # mechanism).  Git shouldn't be a hard-dependency
        raise NotImplementedError

    def checksum(self, hash="sha256"):
        raise NotImplementedError
    '''


class ConfFileType(object):
    """Factory for creating conf file object types;  returns a lazy-loader ConfFile proxy class

    Started from argparse.FileType() and then changed everything.   With our use case, it's often
    necessary to delay writing, or read before writing to a conf file (depending on weather or not
    --dry-run mode is enabled, for example.)

    Instances of FileType are typically passed as type= arguments to the
    ArgumentParser add_argument() method.


    :param mode: How the file is to be opened.  Accepts "r", "w", and "r+".
    :type mode: str
    :param action: Determine how much work should be handled during argument parsing vs handed off
                   to the caller.  Supports 'none', 'open', 'load'.  Full descriptions below.
    :type action: str
    :param parse_profile: parsing configuration settings passed along to the parser
    :param bool accept_dir: Should the CLI accept a directory of config files instead of an
                            individual file.  Defaults to `False`.

    **Values for action**

    ========    =============
    Action      Description
    ========    =============
    ``none``    No preparation or testing is done on the filename.
    ``open``    Ensure the file exists an can be opened.
    ``load``    Ensure the file can be opened and parsed successfully.
    ========    =============


    Once invoked, instances of this class will return a :class:`ConfFileProxy` object, or a
    :class:`ConfDirProxy` object if a directory is passed in via the CLI.
    """

    def __init__(self, mode='r', action="open", parse_profile=None, accept_dir=False):
        self._mode = mode
        self._action = action
        self._parse_profile = parse_profile or {}
        self._accept_dir = accept_dir

    def __call__(self, string):
        from argparse import ArgumentTypeError
        # the special argument "-" means sys.std{in,out}
        if string == '-':
            if 'r' in self._mode:
                cfp = ConfFileProxy("<stdin>", "r", stream=sys.stdin, is_file=False)
                if self._action == "load":
                    try:
                        d = cfp.data
                        del d
                    except ConfParserException as e:
                        raise ArgumentTypeError("failed to parse <stdin>: {}".format(e))
                return cfp
            elif 'w' in self._mode:
                return ConfFileProxy("<stdout>", "w", stream=sys.stdout, is_file=False)
            else:
                raise ValueError('argument "-" with mode {}'.format(self._mode))
        if self._accept_dir and os.path.isdir(string):
            return ConfDirProxy(string, self._mode, parse_profile=self._parse_profile)
        if self._action == "none":
            return ConfFileProxy(string, self._mode, parse_profile=self._parse_profile)
        else:
            try:
                # Another possible option is using:  seekable()  but that only works ONCE the stream is open
                if os.path.isfile(string):
                    encoding = detect_by_bom(string)
                    stream = open(string, self._mode, encoding=encoding)
                    cfp = ConfFileProxy(string, self._mode, stream=stream,
                                        parse_profile=self._parse_profile, is_file=True)
                else:
                    # Could be an explicit link to /dev/stdin; or /dev/fd/63; a bash <(cmd) input
                    # Assume UTF-8 here because that's the default encoding expected by Splunk
                    stream = open(string, self._mode, encoding="utf-8")
                    cfp = ConfFileProxy(string, self._mode, stream=stream,
                                        parse_profile=self._parse_profile, is_file=False)
                if self._action == "load":
                    # Force file to be parsed by accessing the 'data' property
                    d = cfp.data
                    del d
                return cfp
            except IOError as e:
                message = "can't open '%s': %s"
                debug_traceback()
                raise ArgumentTypeError(message % (string, e))
            except ConfParserException as e:
                debug_traceback()
                raise ArgumentTypeError("failed to parse '%s': %s" % (string, e))
            except TypeError as e:
                debug_traceback()
                raise ArgumentTypeError("Parser config error '%s': %s" % (string, e))

    def __repr__(self):     # pragma: no cover
        args = self._mode, self._action, self._parse_profile
        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
        return '%s(%s)' % (type(self).__name__, args_str)



class DescriptionFormatterNoReST(argparse.HelpFormatter):
    @staticmethod
    def strip_simple_rest(s):
        # No handling of embedded backticks for now...  let's keep this simple
        import re
        # Replace literals ``X`` with single quote version:  'X'
        s = re.sub(r'``([^`]*)``', r"'\1'", s)
        # Handle simple references and other named inline markups
        # Handle explicitly named entry first, Use "<title>"
        s = re.sub(r':[a-z][a-z-]{1,10}[a-z]:`[^`]*? <([^`>]*)>\s*`', r"\1", s)
        # Just keep the content of the ref name as-is.  (no "<title>")
        s = re.sub(r':[a-z][a-z-]{1,10}[a-z]:`([^`]*)`', r"'\1'", s)
        return s

    def _fill_text(self, text, width, indent):
        text = self.strip_simple_rest(text)
        text = self._whitespace_matcher.sub(' ', text).strip()
        return textwrap.fill(text, width, initial_indent=indent,
                                           subsequent_indent=indent)

    def _split_lines(self, text, width):
        text = self.strip_simple_rest(text)
        text = self._whitespace_matcher.sub(' ', text).strip()
        return textwrap.wrap(text, width)


class DescriptionHelpFormatterPreserveLayout(DescriptionFormatterNoReST):

    def _fill_text(self, text, width, indent):
        text = self.strip_simple_rest(text)
        # Looks like this one is ONLY used for the top-level description
        return ''.join([indent + line for line in text.splitlines(True)])


class KsconfCmd(object):
    """ Ksconf command specification base class. """
    help = None
    description = None
    format = "default"
    maturity = "alpha"

    def __init__(self, name):
        self.name = name.lower()
        # XXX:  Add logging support.  Find clean lines between logging and UI/console output.
        self.logger = logging.getLogger("ksconf.cmd.{}".format(self.name))
        self.stdin = sys.stdin
        self.stdout = sys.stdout
        self.stderr = sys.stderr

    @classmethod
    def _handle_imports(cls):
        """ Child classes can override this to provide a save place to import 3rd party modules.

        Any ImportErrors thrown here are handled as a special case.  Allow the user of external
        module without killing the entire suite due to one missing library. """
        pass

    '''
    def redirect_io(self, stdin=None, stdout=None, stderr=None):
        if stdin is not None:
            self.stdin = stdin
        if stdout is not None:
            self.stdout = stdout
        if stderr is not None:
            self.stderr = stderr

    def exit(self, exit_code):
        """ Allow overriding for unittesting or other high-level functionality, like an
        interactive interface. """
        sys.exit(exit_code)
    '''

    def add_parser(self, subparser):
        # Passing in the object return by 'ArgumentParser.add_subparsers()'
        kwargs = {
            "help" : self.help,
            "description" : self.description,
        }
        if self.format == "manual":
            kwargs["formatter_class"] = DescriptionHelpFormatterPreserveLayout
        else:
            kwargs["formatter_class"] = DescriptionFormatterNoReST
        self.parser = subparser.add_parser(self.name, **kwargs)
        self.parser.set_defaults(funct=self.launch)
        self.register_args(self.parser)

    def register_args(self, parser):        # pragma: no cover
        # type: (ArgumentParser) -> None
        """ This function in passed the """
        raise NotImplementedError

    def launch(self, args):
        """ Handle flow control between pre_run() / run() / post_run() """
        # If this fails, exception is passed up, no handling errors/logging done here.
        self.pre_run(args)

        exc = None
        try:
            return_code = self.run(args)
        except:     # pragma: no cover
            exc = sys.exc_info()
            raise
        finally:
            # No matter what, post_run is called.
            self.post_run(args, exc)
        return return_code

    def pre_run(self, args):
        """ Pre-run hook.  Any exceptions here prevent run() from being called. """
        pass

    def run(self, args):    # pragma: no cover
        """ Actual works happens here.  Return code should be an EXIT_CODE_* from consts. """
        raise NotImplementedError

    def post_run(self, args, exec_info=None):
        """ Any custom clean up work that needs done.  Always called if run() was.  Presence of
        exc_info indicates failure. """
        pass


def add_splunkd_access_args(parser):
    # type: (argparse.ArgumentParser) -> argparse.ArgumentParser
    parser.add_argument("--url", default="https://localhost:8089",
                        help="URL of Splunkd.  Default:  %(default)s")
    parser.add_argument("--user", default="admin",
                        help="Login username Splunkd.  Default:  %(default)s")
    parser.add_argument("--pass", dest="password", default="changeme",
                        help="Login password Splunkd.  Default:  %(default)s")
    parser.add_argument("-k", "--insecure", action="store_true", default=False,
                        help="Disable SSL cert validation.")
    return parser


def add_splunkd_namespace(parser):
    # type: (argparse.ArgumentParser) -> argparse.ArgumentParser
    parser.add_argument("--app", default="$SPLUNK_APP",
                        help="Set the namespace (app name) for the endpoint")
    parser.add_argument("--owner", default="nobody",
                        help="Set the user who owns the content.  "
                             "The default of 'nobody' works well for app-level sharing.")
    parser.add_argument("--sharing", default="global", choices=["user", "app", "global"],
                        help="Set the sharing mode.")
    return parser


def _get_entrypoints_lib(group, name=None):
    import entrypoints

    # Monkey patch some attributes for better API compatibility
    entrypoints.EntryPoint.dist = property(lambda self: self.distro)

    if name:
        return entrypoints.get_single(group, name)
    else:
        from collections import OrderedDict
        # Copied from 'get_group_named()' except that it preserves order
        result = OrderedDict()
        for ep in entrypoints.get_group_all(group):
            if ep.name not in result:
                result[ep.name] = ep
        return result


""" Disabling this.   Because the DistributionNotFound isn't thrown until the entrypoint.load()
function is called outside of our control.   Going with 'entrypoints' module or local fallback,
giving up on the built-in utility...

def _get_pkgresources_lib(group, name=None):
    # Part of setuptools (widely used but quite slow); It's presence can't be assumed
    if name: raise NotImplementedError
    import pkg_resources
    d = {}
    try:
        for entrypoint in pkg_resources.iter_entry_points(group):
            d[entrypoint.name] = entrypoint
    except pkg_resources.DistributionNotFound:
        # Not really the same thing, but for our purposes; it's close enough, and we can't catch
        # this directly as we can't guarantee pkg_resources is available in the first place...
        raise ImportError
    return d
"""


def _get_fallback(group, name=None):
    from ksconf.setup_entrypoints import get_entrypoints_fallback
    entrypoints = get_entrypoints_fallback(group)
    if name is None:
        return entrypoints
    else:
        return entrypoints[name]

# Removed _get_pkgresources_lib as middle option
__get_entity_resolvers = [ _get_entrypoints_lib, _get_fallback ]

if "ksconf_cmd" in os.environ.get("KSCONF_DISABLE_PLUGINS", ""):    # pragma: no cover
    # Only use the fallback built in mechanism.  This is helpful when unittesting and building docs
    # as we don't want to accidentally document/test code from other packages.
    __get_entity_resolvers = [ _get_fallback ]


# This caching is *mostly* beneficial for unittest CLI testing
@memoize
def get_entrypoints(group, name=None):

    for resolver in list(__get_entity_resolvers):
        results = None
        try:
            results = resolver(group, name=name)
        except ImportError as e:    # pragma: no cover
            __get_entity_resolvers.remove(resolver)
        if results:
            return results
        # Otherwise try next technique ...


KsconfCmdEntryPoint = namedtuple("KsconfCmdEntryPoint", ["name", "entry", "cmd_cls", "error"])


def get_all_ksconf_cmds(on_error="warn"):
    for (name, entry) in get_entrypoints("ksconf_cmd").items():
        try:
            cmd_cls = entry.load()
        except (ImportError, NameError, SyntaxError) as e:
            if on_error == "warn":
                warn("Unable to load entrypoint for {}.  Disabling."
                     "Base exception {}.".format(name, e), KsconfPluginWarning)
            elif on_error == "return":
                error = "Internal error:  {}".format(e)
                yield KsconfCmdEntryPoint(name, entry, None, error)
            else:
                raise e
            continue
        if not issubclass(cmd_cls, KsconfCmd):
            msg = "Issue loading class for entrypoint:  Disabling." \
                  "{!r} is not derived from KsconfCmd.  ".format(entry)
            if on_error == "warn":
                warn(msg, KsconfPluginWarning)
            elif on_error == "return":
                yield KsconfCmdEntryPoint(name, entry, cmd_cls, msg)
            else:
                raise RuntimeError(msg)
            continue
        try:
            cmd_cls._handle_imports()
        except ImportError as e:
            if hasattr(e, "name"):         # PY3
                module = e.name
            else:
                module = e
            if on_error == "warn":
                warn("Unable to load external modules for {}.  Disabling.  "
                     "{}.".format(name, module), KsconfPluginWarning)
            elif on_error == "return":
                error = "Missing 3rd party module:  {}".format(module)
                yield KsconfCmdEntryPoint(name, entry, cmd_cls, error)
            else:
                raise e
            continue
        yield KsconfCmdEntryPoint(name, entry, cmd_cls, None)

1	from __future__ import absolute_import, unicode_literals	8✔
2
3	import argparse	8✔
4	import logging	8✔
5	import os	8✔
6	import sys	8✔
7	import textwrap	8✔
8
9	from io import open	8✔
10	from textwrap import dedent	8✔
11	from warnings import warn	8✔
12	from collections import namedtuple	8✔
13
14	from ksconf import KsconfPluginWarning	8✔
15	from ksconf.conf.parser import parse_conf, smart_write_conf, write_conf, ConfParserException, \	8✔
16	detect_by_bom
17	from ksconf.consts import SMART_CREATE	8✔
18	from ksconf.util import memoize, debug_traceback	8✔
19
20	__all__ = [	8✔
21	"KsconfCmd",
22	"ConfDirProxy",
23	"ConfFileProxy",
24	"ConfFileType",
25	"dedent",
26	"get_all_ksconf_cmds",
27	"get_entrypoints",
28	"add_splunkd_access_args",
29	"add_splunkd_namespace",
30	]
31
32
33	class ConfDirProxy(object):	8✔
34	def __init__(self, name, mode, parse_profile=None):	8✔
35	self.name = name	8✔
36	self._mode = mode	8✔
37	self._parse_profile = parse_profile	8✔
38
39	def get_file(self, relpath):	8✔
40	path = os.path.join(self.name, relpath)	8✔
41	return ConfFileProxy(path, self._mode, parse_profile=self._parse_profile, is_file=True)	8✔
42
43
44	class ConfFileProxy(object):	8✔
45	def __init__(self, name, mode, stream=None, parse_profile=None, is_file=None):	8✔
46	self.name = name	8✔
47	self._mode = mode	8✔
48	if is_file is not None:	8✔
49	self._is_file = is_file	8✔
50	elif stream:	8✔
51	self._is_file = False	8✔
52	elif self.writable() or os.path.isfile(name):	8✔
53	self._is_file = True	8✔
54	else:
55	self._is_file = False	×
56	self._stream = stream	8✔
57
58	# Not sure if there's a good reason to keep a copy of the data locally?
59	self._data = None	8✔
60	self._parse_profile = parse_profile or {}	8✔
61
62	def readable(self):	8✔
63	return "r" in self._mode	8✔
64
65	def writable(self):	8✔
66	return "+" in self._mode or "w" in self._mode	8✔
67
68	def is_file(self):	8✔
69	# Is "seekable" a more appropriate distinction? (match IOBase)
70	return self._is_file	8✔
71
72	def _type(self): # pragma: no cover (only used in exceptions)
73	if self._is_file:
74	return "file"
75	else:
76	return "stream"
77
78	def close(self):	8✔
79	if self._stream:	8✔
80	if not self._stream.closed:	8✔
81	try:	8✔
82	self._stream.close()	8✔
83	finally:
84	del self._stream	8✔
85	self._stream = None	8✔
86
87	def reset(self):	8✔
88	if self._data is not None:	8✔
89	self._data = None	×
90	if self.is_file():	×
91	self.close()	×
92	else:
93	self.stream.seek(0)	×
94
95	def set_parser_option(self, **kwargs):	8✔
96	""" Setting a key to None will remove that setting. """
97	profile = dict(self._parse_profile)	8✔
98	for (k, v) in kwargs.items():	8✔
99	if v is None:	8✔
100	if k in profile:	×
101	del profile[k]	×
102	else:
103	cv = profile.get(k, None)	8✔
104	if cv != v:	8✔
105	profile[k] = v	8✔
106	if self._parse_profile != profile:	8✔
107	self._parse_profile = profile	8✔
108	self.reset()	8✔
109
110	@property	8✔
111	def stream(self):
112	if self._stream is None:	8✔
113	self._stream = open(self.name, self._mode)	8✔
114	return self._stream	8✔
115
116	@property	8✔
117	def data(self):
118	if self._data is None:	8✔
119	self._data = self.load()	8✔
120	return self._data	8✔
121
122	def load(self, profile=None):	8✔
123	if not self.readable():	8✔
124	# Q: Should we mimic the exception caused by doing a read() on a write-only file object?
125	raise ValueError("Unable to load() from {} with mode '{}'".format(self._type(),	×
126	self._mode))
127	parse_profile = dict(self._parse_profile)	8✔
128	if profile:	8✔
129	parse_profile.update(profile)	×
130	data = parse_conf(self.stream, profile=parse_profile)	8✔
131	return data	8✔
132
133	def dump(self, data):	8✔
134	if not self.writable(): # pragma: no cover
135	raise ValueError("Unable to dump() to {} with mode '{}'".format(self._type(),
136	self._mode))
137	# Feels like the right thing to do???? OR self._data = data
138	self._data = None	8✔
139	# write vs smart write here ----
140	if self._is_file:	8✔
141	self.close()	8✔
142	return smart_write_conf(self.name, data)	8✔
143	else:
144	write_conf(self._stream, data)	8✔
145	return SMART_CREATE	8✔
146
147	def unlink(self):	8✔
148	# Eventually this could trigger some kind of backup or recovery mechanism
149	self.close()	8✔
150	return os.unlink(self.name)	8✔
151
152	'''
153	def backup(self, bkname=None):
154	# One option: Write this file directly to the git object store. Just need to store some
155	# kind of index to allow the users to pull it back. (Sill, would need of fall-back
156	# mechanism). Git shouldn't be a hard-dependency
157	raise NotImplementedError
158
159	def checksum(self, hash="sha256"):
160	raise NotImplementedError
161	'''
162
163
164	class ConfFileType(object):	8✔
165	"""Factory for creating conf file object types; returns a lazy-loader ConfFile proxy class
166
167	Started from argparse.FileType() and then changed everything. With our use case, it's often
168	necessary to delay writing, or read before writing to a conf file (depending on weather or not
169	--dry-run mode is enabled, for example.)
170
171	Instances of FileType are typically passed as type= arguments to the
172	ArgumentParser add_argument() method.
173
174
175	:param mode: How the file is to be opened. Accepts "r", "w", and "r+".
176	:type mode: str
177	:param action: Determine how much work should be handled during argument parsing vs handed off
178	to the caller. Supports 'none', 'open', 'load'. Full descriptions below.
179	:type action: str
180	:param parse_profile: parsing configuration settings passed along to the parser
181	:param bool accept_dir: Should the CLI accept a directory of config files instead of an
182	individual file. Defaults to `False`.
183
184	Values for action
185
186	======== =============
187	Action Description
188	======== =============
189	``none`` No preparation or testing is done on the filename.
190	``open`` Ensure the file exists an can be opened.
191	``load`` Ensure the file can be opened and parsed successfully.
192	======== =============
193
194
195	Once invoked, instances of this class will return a :class:`ConfFileProxy` object, or a
196	:class:`ConfDirProxy` object if a directory is passed in via the CLI.
197	"""
198
199	def __init__(self, mode='r', action="open", parse_profile=None, accept_dir=False):	8✔
200	self._mode = mode	8✔
201	self._action = action	8✔
202	self._parse_profile = parse_profile or {}	8✔
203	self._accept_dir = accept_dir	8✔
204
205	def __call__(self, string):	8✔
206	from argparse import ArgumentTypeError	8✔
207	# the special argument "-" means sys.std{in,out}
208	if string == '-':	8✔
209	if 'r' in self._mode:	8✔
210	cfp = ConfFileProxy("<stdin>", "r", stream=sys.stdin, is_file=False)	8✔
211	if self._action == "load":	8✔
212	try:	8✔
213	d = cfp.data	8✔
214	del d	×
215	except ConfParserException as e:	8✔
216	raise ArgumentTypeError("failed to parse <stdin>: {}".format(e))	8✔
217	return cfp	8✔
218	elif 'w' in self._mode:	×
219	return ConfFileProxy("<stdout>", "w", stream=sys.stdout, is_file=False)	×
220	else:
221	raise ValueError('argument "-" with mode {}'.format(self._mode))	×
222	if self._accept_dir and os.path.isdir(string):	8✔
223	return ConfDirProxy(string, self._mode, parse_profile=self._parse_profile)	8✔
224	if self._action == "none":	8✔
225	return ConfFileProxy(string, self._mode, parse_profile=self._parse_profile)	8✔
226	else:
227	try:	8✔
228	# Another possible option is using: seekable() but that only works ONCE the stream is open
229	if os.path.isfile(string):	8✔
230	encoding = detect_by_bom(string)	8✔
231	stream = open(string, self._mode, encoding=encoding)	8✔
232	cfp = ConfFileProxy(string, self._mode, stream=stream,	8✔
233	parse_profile=self._parse_profile, is_file=True)
234	else:
235	# Could be an explicit link to /dev/stdin; or /dev/fd/63; a bash <(cmd) input
236	# Assume UTF-8 here because that's the default encoding expected by Splunk
237	stream = open(string, self._mode, encoding="utf-8")	8✔
238	cfp = ConfFileProxy(string, self._mode, stream=stream,	6✔
239	parse_profile=self._parse_profile, is_file=False)
240	if self._action == "load":	8✔
241	# Force file to be parsed by accessing the 'data' property
242	d = cfp.data	8✔
243	del d	8✔
244	return cfp	8✔
245	except IOError as e:	8✔
246	message = "can't open '%s': %s"	8✔
247	debug_traceback()	8✔
248	raise ArgumentTypeError(message % (string, e))	3✔
249	except ConfParserException as e:	8✔
250	debug_traceback()	8✔
251	raise ArgumentTypeError("failed to parse '%s': %s" % (string, e))	3✔
252	except TypeError as e:	×
253	debug_traceback()	×
254	raise ArgumentTypeError("Parser config error '%s': %s" % (string, e))	×
255
256	def __repr__(self): # pragma: no cover
257	args = self._mode, self._action, self._parse_profile
258	args_str = ', '.join(repr(arg) for arg in args if arg != -1)
259	return '%s(%s)' % (type(self).__name__, args_str)
260
261
262
263	class DescriptionFormatterNoReST(argparse.HelpFormatter):	8✔
264	@staticmethod	8✔
265	def strip_simple_rest(s):
266	# No handling of embedded backticks for now... let's keep this simple
267	import re	8✔
268	# Replace literals ``X`` with single quote version: 'X'
269	s = re.sub(r'``([^`]*)``', r"'\1'", s)	8✔
270	# Handle simple references and other named inline markups
271	# Handle explicitly named entry first, Use "<title>"
272	s = re.sub(r':[a-z][a-z-]{1,10}[a-z]:`[^`]? <([^`>])>\s*`', r"\1", s)	8✔
273	# Just keep the content of the ref name as-is. (no "<title>")
274	s = re.sub(r':[a-z][a-z-]{1,10}[a-z]:`([^`]*)`', r"'\1'", s)	8✔
275	return s	8✔
276
277	def _fill_text(self, text, width, indent):	8✔
278	text = self.strip_simple_rest(text)	×
279	text = self._whitespace_matcher.sub(' ', text).strip()	×
280	return textwrap.fill(text, width, initial_indent=indent,	×
281	subsequent_indent=indent)
282
283	def _split_lines(self, text, width):	8✔
284	text = self.strip_simple_rest(text)	8✔
285	text = self._whitespace_matcher.sub(' ', text).strip()	8✔
286	return textwrap.wrap(text, width)	8✔
287
288
289	class DescriptionHelpFormatterPreserveLayout(DescriptionFormatterNoReST):	8✔
290
291	def _fill_text(self, text, width, indent):	8✔
292	text = self.strip_simple_rest(text)	8✔
293	# Looks like this one is ONLY used for the top-level description
294	return ''.join([indent + line for line in text.splitlines(True)])	8✔
295
296
297	class KsconfCmd(object):	8✔
298	""" Ksconf command specification base class. """
299	help = None	8✔
300	description = None	8✔
301	format = "default"	8✔
302	maturity = "alpha"	8✔
303
304	def __init__(self, name):	8✔
305	self.name = name.lower()	8✔
306	# XXX: Add logging support. Find clean lines between logging and UI/console output.
307	self.logger = logging.getLogger("ksconf.cmd.{}".format(self.name))	8✔
308	self.stdin = sys.stdin	8✔
309	self.stdout = sys.stdout	8✔
310	self.stderr = sys.stderr	8✔
311
312	@classmethod	8✔
313	def _handle_imports(cls):
314	""" Child classes can override this to provide a save place to import 3rd party modules.
315
316	Any ImportErrors thrown here are handled as a special case. Allow the user of external
317	module without killing the entire suite due to one missing library. """
318	pass	8✔
319
320	'''
321	def redirect_io(self, stdin=None, stdout=None, stderr=None):
322	if stdin is not None:
323	self.stdin = stdin
324	if stdout is not None:
325	self.stdout = stdout
326	if stderr is not None:
327	self.stderr = stderr
328
329	def exit(self, exit_code):
330	""" Allow overriding for unittesting or other high-level functionality, like an
331	interactive interface. """
332	sys.exit(exit_code)
333	'''
334
335	def add_parser(self, subparser):	8✔
336	# Passing in the object return by 'ArgumentParser.add_subparsers()'
337	kwargs = {	8✔
338	"help" : self.help,
339	"description" : self.description,
340	}
341	if self.format == "manual":	8✔
342	kwargs["formatter_class"] = DescriptionHelpFormatterPreserveLayout	8✔
343	else:
344	kwargs["formatter_class"] = DescriptionFormatterNoReST	8✔
345	self.parser = subparser.add_parser(self.name, **kwargs)	8✔
346	self.parser.set_defaults(funct=self.launch)	8✔
347	self.register_args(self.parser)	8✔
348
349	def register_args(self, parser): # pragma: no cover
350	# type: (ArgumentParser) -> None
351	""" This function in passed the """
352	raise NotImplementedError
353
354	def launch(self, args):	8✔
355	""" Handle flow control between pre_run() / run() / post_run() """
356	# If this fails, exception is passed up, no handling errors/logging done here.
357	self.pre_run(args)	8✔
358
359	exc = None	8✔
360	try:	8✔
361	return_code = self.run(args)	8✔
362	except: # pragma: no cover
363	exc = sys.exc_info()
364	raise
365	finally:
366	# No matter what, post_run is called.
367	self.post_run(args, exc)	8✔
368	return return_code	8✔
369
370	def pre_run(self, args):	8✔
371	""" Pre-run hook. Any exceptions here prevent run() from being called. """
372	pass	8✔
373
374	def run(self, args): # pragma: no cover
375	""" Actual works happens here. Return code should be an EXIT_CODE_* from consts. """
376	raise NotImplementedError
377
378	def post_run(self, args, exec_info=None):	8✔
379	""" Any custom clean up work that needs done. Always called if run() was. Presence of
380	exc_info indicates failure. """
381	pass	8✔
382
383
384	def add_splunkd_access_args(parser):	8✔
385	# type: (argparse.ArgumentParser) -> argparse.ArgumentParser
386	parser.add_argument("--url", default="https://localhost:8089",	8✔
387	help="URL of Splunkd. Default: %(default)s")
388	parser.add_argument("--user", default="admin",	8✔
389	help="Login username Splunkd. Default: %(default)s")
390	parser.add_argument("--pass", dest="password", default="changeme",	8✔
391	help="Login password Splunkd. Default: %(default)s")
392	parser.add_argument("-k", "--insecure", action="store_true", default=False,	8✔
393	help="Disable SSL cert validation.")
394	return parser	8✔
395
396
397	def add_splunkd_namespace(parser):	8✔
398	# type: (argparse.ArgumentParser) -> argparse.ArgumentParser
399	parser.add_argument("--app", default="$SPLUNK_APP",	8✔
400	help="Set the namespace (app name) for the endpoint")
401	parser.add_argument("--owner", default="nobody",	8✔
402	help="Set the user who owns the content. "
403	"The default of 'nobody' works well for app-level sharing.")
404	parser.add_argument("--sharing", default="global", choices=["user", "app", "global"],	8✔
405	help="Set the sharing mode.")
406	return parser	8✔
407
408
409	def _get_entrypoints_lib(group, name=None):	8✔
410	import entrypoints	8✔
411
412	# Monkey patch some attributes for better API compatibility
413	entrypoints.EntryPoint.dist = property(lambda self: self.distro)	8✔
414
415	if name:	8✔
416	return entrypoints.get_single(group, name)	8✔
417	else:
418	from collections import OrderedDict	8✔
419	# Copied from 'get_group_named()' except that it preserves order
420	result = OrderedDict()	8✔
421	for ep in entrypoints.get_group_all(group):	8✔
422	if ep.name not in result:	8✔
423	result[ep.name] = ep	8✔
424	return result	8✔
425
426
427	""" Disabling this. Because the DistributionNotFound isn't thrown until the entrypoint.load()
428	function is called outside of our control. Going with 'entrypoints' module or local fallback,
429	giving up on the built-in utility...
430
431	def _get_pkgresources_lib(group, name=None):
432	# Part of setuptools (widely used but quite slow); It's presence can't be assumed
433	if name: raise NotImplementedError
434	import pkg_resources
435	d = {}
436	try:
437	for entrypoint in pkg_resources.iter_entry_points(group):
438	d[entrypoint.name] = entrypoint
439	except pkg_resources.DistributionNotFound:
440	# Not really the same thing, but for our purposes; it's close enough, and we can't catch
441	# this directly as we can't guarantee pkg_resources is available in the first place...
442	raise ImportError
443	return d
444	"""
445
446
447	def _get_fallback(group, name=None):	8✔
448	from ksconf.setup_entrypoints import get_entrypoints_fallback	8✔
449	entrypoints = get_entrypoints_fallback(group)	8✔
450	if name is None:	8✔
451	return entrypoints	8✔
452	else:
453	return entrypoints[name]	×
454
455	# Removed _get_pkgresources_lib as middle option
456	__get_entity_resolvers = [ _get_entrypoints_lib, _get_fallback ]	8✔
457
458	if "ksconf_cmd" in os.environ.get("KSCONF_DISABLE_PLUGINS", ""): # pragma: no cover
459	# Only use the fallback built in mechanism. This is helpful when unittesting and building docs
460	# as we don't want to accidentally document/test code from other packages.
461	__get_entity_resolvers = [ _get_fallback ]
462
463
464	# This caching is mostly beneficial for unittest CLI testing
465	@memoize	8✔
466	def get_entrypoints(group, name=None):	8✔
467
468	for resolver in list(__get_entity_resolvers):	8✔
469	results = None	8✔
470	try:	8✔
471	results = resolver(group, name=name)	8✔
472	except ImportError as e: # pragma: no cover
473	__get_entity_resolvers.remove(resolver)
474	if results:	8✔
475	return results	8✔
476	# Otherwise try next technique ...
477
478
479	KsconfCmdEntryPoint = namedtuple("KsconfCmdEntryPoint", ["name", "entry", "cmd_cls", "error"])	8✔
480
481
482	def get_all_ksconf_cmds(on_error="warn"):	8✔
483	for (name, entry) in get_entrypoints("ksconf_cmd").items():	8✔
484	try:	8✔
485	cmd_cls = entry.load()	8✔
486	except (ImportError, NameError, SyntaxError) as e:	×
487	if on_error == "warn":	×
488	warn("Unable to load entrypoint for {}. Disabling."	×
489	"Base exception {}.".format(name, e), KsconfPluginWarning)
490	elif on_error == "return":	×
491	error = "Internal error: {}".format(e)	×
492	yield KsconfCmdEntryPoint(name, entry, None, error)	×
493	else:
494	raise e	×
495	continue	×
496	if not issubclass(cmd_cls, KsconfCmd):	8✔
497	msg = "Issue loading class for entrypoint: Disabling." \	×
498	"{!r} is not derived from KsconfCmd. ".format(entry)
499	if on_error == "warn":	×
500	warn(msg, KsconfPluginWarning)	×
501	elif on_error == "return":	×
502	yield KsconfCmdEntryPoint(name, entry, cmd_cls, msg)	×
503	else:
504	raise RuntimeError(msg)	×
505	continue	×
506	try:	8✔
507	cmd_cls._handle_imports()	8✔
508	except ImportError as e:	×
509	if hasattr(e, "name"): # PY3	×
510	module = e.name	×
511	else:
512	module = e	×
513	if on_error == "warn":	×
514	warn("Unable to load external modules for {}. Disabling. "	×
515	"{}.".format(name, module), KsconfPluginWarning)
516	elif on_error == "return":	×
517	error = "Missing 3rd party module: {}".format(module)	×
518	yield KsconfCmdEntryPoint(name, entry, cmd_cls, error)	×
519	else:
520	raise e	×
521	continue	×
522	yield KsconfCmdEntryPoint(name, entry, cmd_cls, None)	8✔

Kintyre / ksconf / 307

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous