Kintyre / ksconf / 203

pending completion

Build # 203

Build Type

push

travis-ci

Committed by

lowell80

Commit Message

<a href="https://github.com/Kintyre/ksconf/commit/<a class=hub.com/Kintyre/ksconf/commit/<a class="double-link" href="https://git"><a class=hub.com/Kintyre/ksconf/commit/7c5068788232d3cce60bbfc856c953bee8ab8777">7c5068788<a href="https://github.com/Kintyre/ksconf/commit/7c5068788232d3cce60bbfc856c953bee8ab8777">&lt;a href=&quot;https://github.com/Kintyre/ksconf/commit/7c5068788232d3cce60bbfc856c953bee8ab8777">&quot;&gt;Merge pull request #32 (squashed)

commit &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/Kintyre/ksconf/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/Kintyre/ksconf/commit/a8806774837e11c5be81579beaf89ad0102c3f68&quot;&gt;a88067748&lt;/a&gt;&quot;&gt;a88067748&lt;/a&gt;&lt;a href=&quot;https://github.com/Kintyre/ksconf/commit/7c5068788232d3cce60bbfc856c953bee8ab8777">
Author: Lowell Alleman &lt;lowell@kintyre.co&gt;
Date:   Thu Jan 3 18:28:07 2019 -0500

    Add CLI unittests for rest-export

commit </a><a class="double-link" href="https://github.com/Kintyre/ksconf/commit/<a class="double-link" href="https://github.com/Kintyre/ksconf/commit/57e385ecc18520bb60e492d7cf894fb91c10c91a">57e385ecc</a>">57e385ecc</a><a href="https://github.com/Kintyre/ksconf/commit/7c5068788232d3cce60bbfc856c953bee8ab8777">
Author: Lowell Alleman &lt;lowell@kintyre.co&gt;
Date:   Thu Dec 13 15:32:39 2018 -0500

    New &#39;rest-export&#39; command

    Added new &#39;rest-export&#39; command that takes .conf files and turns them into a
    shell script of curl calls for importing configs into Splunk Cloud or other
    UI-only accessible instances.

    Missing unit tests.
    Ready for wider test audience.

Run Details

113 of 113 new or added lines in 2 files covered. (100.0%)

2215 of 2446 relevant lines covered (90.56%)

7.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.65

/ksconf/commands/restexport.py

# -*- coding: utf-8 -*-
"""
SUBCOMMAND:  ksconf rest-export --output=script.sh <CONF>

Usage example:

    ksconf rest-export --output=apply_props.sh /opt/splunk/etc/app/Splunk_TA_aws/local/props.conf


NOTE:

    If we add support for Windows CURL, then we'll need to also support proper quoting for the '%'
    character.   This can be done with '%^', wonky, I know...

"""
from __future__ import absolute_import, unicode_literals

import sys
import os

from argparse import FileType
from six.moves.urllib.parse import quote

from ksconf.commands import KsconfCmd, dedent, ConfFileType
from ksconf.conf.parser import PARSECONF_LOOSE, GLOBAL_STANZA
from ksconf.consts import EXIT_CODE_SUCCESS
from ksconf.util.completers import conf_files_completer
from collections import OrderedDict



class CurlCommand(object):
    def __init__(self):
        self.url = None
        self.pre_args = [ "-k" ]
        self.post_args = []
        self.headers = OrderedDict()
        self.data = OrderedDict()

    @classmethod
    def quote(cls, s):
        if "$" in s:
            s = '"{}"'.format(s)
        elif " " in s or "$" in s:
            s = "'{}'".format(s)
        return s

    def get_command(self):
        cmd = ["curl"]

        args = []
        if self.headers:
            for header in self.headers:
                value = self.headers[header]
                args.append("-H")
                args.append("{}: {}".format(header, value))
        if self.data:
            for key in self.data:
                value = self.data[key]
                args.append("-d")
                args.append("{}={}".format(quote(key), quote(value)))


        if self.pre_args:
            cmd.append(" ".join(self.pre_args))
        cmd.append(self.url)
        args = [ self.quote(arg) for arg in args ]
        cmd.extend(args)
        if self.post_args:
            cmd.append(" ".join(self.post_args))
        return " ".join(cmd)



class RestExportCmd(KsconfCmd):
    help = "Export .conf settings as a curl script to apply to a Splunk instance later (via REST)"
    description = dedent("""\
    Build an executable script of the stanzas in a configuration file that can be later applied to
    a running Splunk instance via the Splunkd REST endpoint.

    This can be helpful when pushing complex props & transforms to an instance where you only have
    UI access and can't directly publish an app.

    WARNING:  This command is indented for manual admin workflows.  It's quite possible that shell
    escaping bugs exist that may allow full shell access if you put this into an automated workflow.
    Evalute the risks, review the code, and run as a least-privilege user, and be responsible.

    For now the assumption is that 'curl' command will be used.  (Patches to support the Power Shell
    Invoke-WebRequest cmdlet would be greatly welcomed!)

    ksconf rest-export --output=apply_props.sh etc/app/Splunk_TA_aws/local/props.conf
    """)
    format = "manual"

    def register_args(self, parser):
        parser.add_argument("conf", metavar="FILE", nargs="+",
                            type=ConfFileType("r", "load", parse_profile=PARSECONF_LOOSE),
                            help="Configuration file(s) to export settings from."
                            ).completer = conf_files_completer
        parser.add_argument("--output", "-t", metavar="FILE",
                            type=FileType("w"), default=sys.stdout,
                            help="Save the shell script output to this file.  "
                                 "If not provided, the output is written to standard output.")
        '''
        parser.add_argument("--syntax", choices=["curl", "powershell"],  # curl-windows?
                            default="curl",
                            help="Pick the output syntax mode.  "
                                 "Currently only 'curl' is supported.")
        '''
        parser.add_argument("-u", "--update", action="store_true", default=False,
                            help="Assume that the REST entities already exist.  "
                                 "By default output assumes stanzas are being created.  "
                                 "(This is an unfortunate quark of the configs REST API)")
        parser.add_argument("--url", default="https://localhost:8089",
                            help="URL of Splunkd.  Default:  %(default)s")
        parser.add_argument("--app", default="$SPLUNK_APP",
                            help="Set the namespace (app name) for the endpoint")
        parser.add_argument("--user", default="nobody",
                            help="Set the user associated.  Typically the default of 'nobody' is "
                                 "ideal if you want to share the configurations at the app-level.")

    @staticmethod
    def build_rest_url(base, user, app, conf):
        # XXX: Quote user & app; however for now we're still allowing the user to pass though an
        #  environmental variable as-is and quoting would break that.   Need to make a decision,
        # for now this is not likely to be a big issue given app and user name restrictions.
        url = "{}/servicesNS/{}/{}/configs/conf-{}".format(base, user, app, conf)
        return url

    def run(self, args):
        ''' Snapshot multiple configuration files into a single json snapshot. '''
        """

        Some inspiration in the form of CURL commands...

        [single_quote_kv]
        REGEX = ([^=\s]+)='([^']+)'
        FORMAT = $1::$2
        MV_ADD = 0

        CREATE NEW:

        curl -k https://SPLUNK:8089/servicesNS/nobody/my_app/configs/conf-transforms \
         -H "Authorization: Splunk $SPLUNKDAUTH" -X POST \
         -d name=single_quote_kv \
         -d REGEX="(%5B%5E%3D%5Cs%5D%2B)%3D%27(%5B%5E%27%5D%2B)%27" \
         -d FORMAT='$1::$2'

        UPDATE EXISTING:  (note the change in URL/name attribute)

        curl -k https://SPLUNK:8089/servicesNS/nobody/my_app/configs/conf-transforms/single_quote_kv \
         -H "Authorization: Splunk $SPLUNKDAUTH" -X POST \
         -d REGEX="(%5B%5E%3D%5Cs%5D%2B)%3D%27(%5B%5E%27%5D%2B)%27" \
         -d FORMAT='$1::$2' \
         -d MV_ADD=0
        """
        #  XXX:  Someday make multiline output that looks pretty...  someday

        stream = args.output

        if True:
            # Make this preamble optional
            stream.write("## Example of creating a local SPLUNKDAUTH token\n")
            stream.write("export SPLUNKDAUTH=$("
                         "curl -ks {}/services/auth/login -d username=admin -d password=changeme "
                         "| grep sessionKey "
                         r"| sed -re 's/\s*<sessionKey>(.*)<.sessionKey>/\1/')".format(args.url))
            stream.write("\n\n\n")

        for conf_proxy in args.conf:
            conf = conf_proxy.data
            conf_type = os.path.basename(conf_proxy.name).replace(".conf", "")

            stream.write("# CURL REST commands for {}\n".format(conf_proxy.name))

            for stanza_name, stanza_data in conf.items():
                cc = CurlCommand()
                cc.url = self.build_rest_url(args.url, args.user, args.app, conf_type)

                if stanza_name is GLOBAL_STANZA:
                    # XXX:  Research proper handling of default/global stanazas..
                    # As-is, curl returns an HTTP error, but yet the new entry is added to the
                    # conf file.  So I suppose we could ignore the exit code?!    ¯\_(ツ)_/¯
                    stream.write("### WARN:  Writing to the default stanza may not work as "
                                 "expected.  Or it may work, but be reported as a failure.  "
                                 "Patches welcome!\n")
                    cc.url += "/default"
                elif args.update:
                    cc.url += "/" + quote(stanza_name, "")   # Must quote '/'s too.
                else:
                    cc.data["name"] = stanza_name

                # Add individual keys
                for (key, value) in stanza_data.items():
                    cc.data[key] = value

                cc.headers["Authorization"] = "Splunk $SPLUNKDAUTH"

                stream.write(cc.get_command())
                stream.write("\n")
            stream.write("\n")
        stream.write("\n")

        return EXIT_CODE_SUCCESS

1	# -- coding: utf-8 --
2	"""	8✔
3	SUBCOMMAND: ksconf rest-export --output=script.sh <CONF>
4
5	Usage example:
6
7	ksconf rest-export --output=apply_props.sh /opt/splunk/etc/app/Splunk_TA_aws/local/props.conf
8
9
10	NOTE:
11
12	If we add support for Windows CURL, then we'll need to also support proper quoting for the '%'
13	character. This can be done with '%^', wonky, I know...
14
15	"""
16	from __future__ import absolute_import, unicode_literals	8✔
17
18	import sys	8✔
19	import os	8✔
20
21	from argparse import FileType	8✔
22	from six.moves.urllib.parse import quote	8✔
23
24	from ksconf.commands import KsconfCmd, dedent, ConfFileType	8✔
25	from ksconf.conf.parser import PARSECONF_LOOSE, GLOBAL_STANZA	8✔
26	from ksconf.consts import EXIT_CODE_SUCCESS	8✔
27	from ksconf.util.completers import conf_files_completer	8✔
28	from collections import OrderedDict	8✔
29
30
31
32	class CurlCommand(object):	8✔
33	def __init__(self):	8✔
34	self.url = None	8✔
35	self.pre_args = [ "-k" ]	8✔
36	self.post_args = []	8✔
37	self.headers = OrderedDict()	8✔
38	self.data = OrderedDict()	8✔
39
40	@classmethod	8✔
41	def quote(cls, s):
42	if "$" in s:	8✔
43	s = '"{}"'.format(s)	8✔
44	elif " " in s or "$" in s:	8✔
45	s = "'{}'".format(s)	×
46	return s	8✔
47
48	def get_command(self):	8✔
49	cmd = ["curl"]	8✔
50
51	args = []	8✔
52	if self.headers:	8✔
53	for header in self.headers:	8✔
54	value = self.headers[header]	8✔
55	args.append("-H")	8✔
56	args.append("{}: {}".format(header, value))	8✔
57	if self.data:	8✔
58	for key in self.data:	8✔
59	value = self.data[key]	8✔
60	args.append("-d")	8✔
61	args.append("{}={}".format(quote(key), quote(value)))	8✔
62
63
64	if self.pre_args:	8✔
65	cmd.append(" ".join(self.pre_args))	8✔
66	cmd.append(self.url)	8✔
67	args = [ self.quote(arg) for arg in args ]	8✔
68	cmd.extend(args)	8✔
69	if self.post_args:	8✔
70	cmd.append(" ".join(self.post_args))	×
71	return " ".join(cmd)	8✔
72
73
74
75	class RestExportCmd(KsconfCmd):	8✔
76	help = "Export .conf settings as a curl script to apply to a Splunk instance later (via REST)"	8✔
77	description = dedent("""\	8✔
78	Build an executable script of the stanzas in a configuration file that can be later applied to
79	a running Splunk instance via the Splunkd REST endpoint.
80
81	This can be helpful when pushing complex props & transforms to an instance where you only have
82	UI access and can't directly publish an app.
83
84	WARNING: This command is indented for manual admin workflows. It's quite possible that shell
85	escaping bugs exist that may allow full shell access if you put this into an automated workflow.
86	Evalute the risks, review the code, and run as a least-privilege user, and be responsible.
87
88	For now the assumption is that 'curl' command will be used. (Patches to support the Power Shell
89	Invoke-WebRequest cmdlet would be greatly welcomed!)
90
91	ksconf rest-export --output=apply_props.sh etc/app/Splunk_TA_aws/local/props.conf
92	""")
93	format = "manual"	8✔
94
95	def register_args(self, parser):	8✔
96	parser.add_argument("conf", metavar="FILE", nargs="+",	8✔
97	type=ConfFileType("r", "load", parse_profile=PARSECONF_LOOSE),
98	help="Configuration file(s) to export settings from."
99	).completer = conf_files_completer
100	parser.add_argument("--output", "-t", metavar="FILE",	8✔
101	type=FileType("w"), default=sys.stdout,
102	help="Save the shell script output to this file. "
103	"If not provided, the output is written to standard output.")
104	'''
105	parser.add_argument("--syntax", choices=["curl", "powershell"], # curl-windows?
106	default="curl",
107	help="Pick the output syntax mode. "
108	"Currently only 'curl' is supported.")
109	'''
110	parser.add_argument("-u", "--update", action="store_true", default=False,	8✔
111	help="Assume that the REST entities already exist. "
112	"By default output assumes stanzas are being created. "
113	"(This is an unfortunate quark of the configs REST API)")
114	parser.add_argument("--url", default="https://localhost:8089",	8✔
115	help="URL of Splunkd. Default: %(default)s")
116	parser.add_argument("--app", default="$SPLUNK_APP",	8✔
117	help="Set the namespace (app name) for the endpoint")
118	parser.add_argument("--user", default="nobody",	8✔
119	help="Set the user associated. Typically the default of 'nobody' is "
120	"ideal if you want to share the configurations at the app-level.")
121
122	@staticmethod	8✔
123	def build_rest_url(base, user, app, conf):
124	# XXX: Quote user & app; however for now we're still allowing the user to pass though an
125	# environmental variable as-is and quoting would break that. Need to make a decision,
126	# for now this is not likely to be a big issue given app and user name restrictions.
127	url = "{}/servicesNS/{}/{}/configs/conf-{}".format(base, user, app, conf)	8✔
128	return url	8✔
129
130	def run(self, args):	8✔
131	''' Snapshot multiple configuration files into a single json snapshot. '''
132	"""
133
134	Some inspiration in the form of CURL commands...
135
136	[single_quote_kv]
137	REGEX = ([^=\s]+)='([^']+)'
138	FORMAT = $1::$2
139	MV_ADD = 0
140
141	CREATE NEW:
142
143	curl -k https://SPLUNK:8089/servicesNS/nobody/my_app/configs/conf-transforms \
144	-H "Authorization: Splunk $SPLUNKDAUTH" -X POST \
145	-d name=single_quote_kv \
146	-d REGEX="(%5B%5E%3D%5Cs%5D%2B)%3D%27(%5B%5E%27%5D%2B)%27" \
147	-d FORMAT='$1::$2'
148
149	UPDATE EXISTING: (note the change in URL/name attribute)
150
151	curl -k https://SPLUNK:8089/servicesNS/nobody/my_app/configs/conf-transforms/single_quote_kv \
152	-H "Authorization: Splunk $SPLUNKDAUTH" -X POST \
153	-d REGEX="(%5B%5E%3D%5Cs%5D%2B)%3D%27(%5B%5E%27%5D%2B)%27" \
154	-d FORMAT='$1::$2' \
155	-d MV_ADD=0
156	"""
157	# XXX: Someday make multiline output that looks pretty... someday
158
159	stream = args.output	8✔
160
161	if True:	3✔
162	# Make this preamble optional
163	stream.write("## Example of creating a local SPLUNKDAUTH token\n")	8✔
164	stream.write("export SPLUNKDAUTH=$("	8✔
165	"curl -ks {}/services/auth/login -d username=admin -d password=changeme "
166	"\| grep sessionKey "
167	r"\| sed -re 's/\s<sessionKey>(.)<.sessionKey>/\1/')".format(args.url))
168	stream.write("\n\n\n")	8✔
169
170	for conf_proxy in args.conf:	8✔
171	conf = conf_proxy.data	8✔
172	conf_type = os.path.basename(conf_proxy.name).replace(".conf", "")	8✔
173
174	stream.write("# CURL REST commands for {}\n".format(conf_proxy.name))	8✔
175
176	for stanza_name, stanza_data in conf.items():	8✔
177	cc = CurlCommand()	8✔
178	cc.url = self.build_rest_url(args.url, args.user, args.app, conf_type)	8✔
179
180	if stanza_name is GLOBAL_STANZA:	8✔
181	# XXX: Research proper handling of default/global stanazas..
182	# As-is, curl returns an HTTP error, but yet the new entry is added to the
183	# conf file. So I suppose we could ignore the exit code?! ¯\_(ツ)_/¯
184	stream.write("### WARN: Writing to the default stanza may not work as "	8✔
185	"expected. Or it may work, but be reported as a failure. "
186	"Patches welcome!\n")
187	cc.url += "/default"	8✔
188	elif args.update:	8✔
189	cc.url += "/" + quote(stanza_name, "") # Must quote '/'s too.	8✔
190	else:
191	cc.data["name"] = stanza_name	8✔
192
193	# Add individual keys
194	for (key, value) in stanza_data.items():	8✔
195	cc.data[key] = value	8✔
196
197	cc.headers["Authorization"] = "Splunk $SPLUNKDAUTH"	8✔
198
199	stream.write(cc.get_command())	8✔
200	stream.write("\n")	8✔
201	stream.write("\n")	8✔
202	stream.write("\n")	8✔
203
204	return EXIT_CODE_SUCCESS	8✔

Kintyre / ksconf / 203

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous