6650421557

Committed 26 Oct 2023 06:39AM UTC coverage: 60.541% (-2.1%) from 62.617%

Build # 6650421557

Build Type

Pull #35

github-actions

Committed by

web-flow

Commit Message

Merge 6c2cfcfd0 into 0ebcd057d

Pull Request Pull Request #35: validate log_bog_logs inputs againts schema

Run Details

32 of 32 new or added lines in 1 file covered. (100.0%)

537 of 887 relevant lines covered (60.54%)

0.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

37.5

/telemetry/cli.py

"""Console script for telemetry."""
import sys
import click
import datetime
import telemetry
import os


def validate(field, value, schema):
    """Validate a field and value data type
    againts a given es schema
    """
    properties = schema["mappings"]["properties"]
    type_maps = {
        "keyword" : "str",
        "text" : "str",
        "boolean" : "bool",
        "integer" : "int",
        "date" : "datetime"
    }
    if not field in properties.keys():
        raise Exception(f"Schema validator: {field} not supported")

    # Validate data type
    expected_type = type_maps[properties[field]["type"]]
    actual_type = type(value).__name__

    try:
        if not actual_type == expected_type:
            raise Exception(f"Schema validator: {field} expects {expected_type} type, {actual_type} is given")
    except Exception:
        if expected_type == "bool":
            if value.lower() in ["false", "true"]:
                value = bool(value)
        if expected_type == "int":
            value = int(value)

@click.group()
def cli():
    pass


@click.command()
@click.option(
    "--tdir",
    default="/test/logs/unprocessed",
    help="Path to directory of unprocessed test logs",
)
@click.option("--server", default=None, help="Address of mongo server")
@click.option("--username", default=None, help="Username for mongo server")
@click.option("--password", default=None, help="Password for mongo server")
@click.option("--dbname", default=None, help="Target collection for mongo server")
@click.option("--board", default=None, help="Name of target board")
def prod_logs_upload(tdir, server, username, password, dbname, board):
    """Upload unprocessed test logs to mongo for synchrona."""
    sync = telemetry.prod.BoardLog(server, username, password, dbname, board)
    sync.default_unprocessed_log_dir = tdir
    sync.default_processed_log_dir = os.path.join(tdir, "processed")
    if not os.path.isdir(sync.default_processed_log_dir):
        os.mkdir(sync.default_processed_log_dir)
    sync() # go go


@click.command()
@click.option("--server", default="picard", help="Address of Elasticsearch server")
@click.option(
    "--filename",
    default="resource_utilization.csv",
    help="Full path to resource utilization csv file generated by HDL builds",
)
def log_hdl_resources_from_csv(server, filename):
    tel = telemetry.ingest(server=server)
    tel.log_hdl_resources_from_csv(filename)


@click.command()
@click.option("--server", default="picard", help="Address of Elasticsearch server")
@click.argument("in_args", nargs=-1)
def log_artifacts(server, in_args):
    entry = {
        "url": "NA",
        "server": "NA",
        "job": "NA",
        "job_no": 0,
        "job_date": None,
        "job_build_parameters": "NA",
        "file_name": "NA",
        "target_board": "NA",
        "artifact_info_type": "NA",
        "payload_raw": "NA",
        "payload_ts": "NA",
        "payload": "NA",
        "payload_param": "NA"
    }
    if len(in_args) == 0:
        click.echo("Must have non-zero arguments for database entry")
        sys.exit(1)
    if int(len(in_args) / 2) != len(in_args) / 2:
        click.echo(
            "ERROR: Number of inputs arguments must be even\n"
            + "       and in the form of: entry1<space>value1<space>entry2<space>value2"
        )
        sys.exit(1)
    for i in range(0, len(in_args), 2):
        if in_args[i] in entry:
            if in_args[i + 1].lower() == "true":
                entry[in_args[i]] = True
            elif in_args[i + 1].lower() == "false":
                entry[in_args[i]] = False
            else:
                entry[in_args[i]] = in_args[i + 1]
        else:
            click.echo("ERROR: " + in_args[i] + " not a valid entry")
            sys.exit(1)
    tel = telemetry.ingest(server=server)
    tel.log_artifacts(**entry)

@click.command()
@click.option("--jenkins-server", required=True, help="Address of Jenkins server")
@click.option("--jenkins-username", required=False, help="Username with access on the Jenkins server")
@click.option("--jenkins-password", required=False, help="Password for Jenkins username")
@click.option("--es-server", required=True, help="Address of Elasticsearch server")
@click.option("--job-name", default="HW_tests/HW_test_multiconfig", help="Name of Jenkins job")
@click.option("--job", multiple=True, help="Job(s)/build(s) to process")
def grab_and_log_artifacts(
        jenkins_server,
        jenkins_username,
        jenkins_password,
        es_server,
        job_name,
        job
    ):
    if not len(job) > 0:
        click.echo("Atleast 1 Job/Build (--job) is needed.")
        sys.exit(1)
    g = telemetry.gargantua(
        jenkins_server,
        jenkins_username,
        jenkins_password,
        es_server,
        job_name,
        job
    )
    g.log_artifacts()

@click.command()
@click.option("--server", default="picard", help="Address of Elasticsearch server")
@click.argument("in_args", nargs=-1)
def log_boot_logs(server, in_args):
    
    tel = telemetry.ingest(server=server)
    schema = tel.db.import_schema(tel._get_schema("boot_tests.json"))
    entry = dict()
    for k,v in schema["mappings"]["properties"].items():
        if k == "source_adjacency_matrix":
            continue
        if v["type"] in ["txt","keyword"]:
            entry.update({k:"NA"})
        elif v["type"] in ["integer"]:
            entry.update({k: 0})
        elif v["type"] in ["boolean"]:
            entry.update({k: False})
        elif k == "jenkins_job_date":
            entry.update({k:datetime.datetime.now()})
        else:
            entry.update({k: None})

    if len(in_args) == 0:
        click.echo("Must have non-zero arguments for database entry")
        sys.exit(1)
    if int(len(in_args) / 2) != len(in_args) / 2:
        click.echo(
            "ERROR: Number of inputs arguments must be even\n"
            + "       and in the form of: entry1<space>value1<space>entry2<space>value2"
        )
        sys.exit(1)

    for i in range(0, len(in_args), 2):
        if in_args[i] in entry:
            validate(in_args[i], in_args[i+1],schema)
            if in_args[i + 1].lower() == "true":
                entry[in_args[i]] = True
            elif in_args[i + 1].lower() == "false":
                entry[in_args[i]] = False
            else:
                entry[in_args[i]] = in_args[i + 1]
        else:
            click.echo("ERROR: " + in_args[i] + " not a valid entry")
            sys.exit(1)

    tel = telemetry.ingest(server=server)
    tel.log_boot_tests(**entry)


@click.command()
def main(args=None):
    """Console script for telemetry."""
    click.echo("Replace this message by putting your code into " "telemetry.cli.main")
    click.echo("See click documentation at https://click.palletsprojects.com/")
    return 0


cli.add_command(prod_logs_upload)
cli.add_command(log_boot_logs)
cli.add_command(log_hdl_resources_from_csv)
cli.add_command(log_artifacts)
cli.add_command(grab_and_log_artifacts)
cli.add_command(main)

if __name__ == "__main__":
    sys.exit(cli())  # pragma: no cover

1	"""Console script for telemetry."""
2	import sys	1✔
3	import click	1✔
4	import datetime	1✔
5	import telemetry	1✔
6	import os	1✔
7
8
9	def validate(field, value, schema):	1✔
10	"""Validate a field and value data type
11	againts a given es schema
12	"""
13	properties = schema["mappings"]["properties"]	×
14	type_maps = {	×
15	"keyword" : "str",
16	"text" : "str",
17	"boolean" : "bool",
18	"integer" : "int",
19	"date" : "datetime"
20	}
21	if not field in properties.keys():	×
22	raise Exception(f"Schema validator: {field} not supported")	×
23
24	# Validate data type
25	expected_type = type_maps[properties[field]["type"]]	×
26	actual_type = type(value).__name__	×
27
28	try:	×
29	if not actual_type == expected_type:	×
30	raise Exception(f"Schema validator: {field} expects {expected_type} type, {actual_type} is given")	×
31	except Exception:	×
32	if expected_type == "bool":	×
33	if value.lower() in ["false", "true"]:	×
34	value = bool(value)	×
35	if expected_type == "int":	×
36	value = int(value)	×
37
38	@click.group()	1✔
39	def cli():	1✔
40	pass	×
41
42
43	@click.command()	1✔
44	@click.option(	1✔
45	"--tdir",
46	default="/test/logs/unprocessed",
47	help="Path to directory of unprocessed test logs",
48	)
49	@click.option("--server", default=None, help="Address of mongo server")	1✔
50	@click.option("--username", default=None, help="Username for mongo server")	1✔
51	@click.option("--password", default=None, help="Password for mongo server")	1✔
52	@click.option("--dbname", default=None, help="Target collection for mongo server")	1✔
53	@click.option("--board", default=None, help="Name of target board")	1✔
54	def prod_logs_upload(tdir, server, username, password, dbname, board):	1✔
55	"""Upload unprocessed test logs to mongo for synchrona."""
56	sync = telemetry.prod.BoardLog(server, username, password, dbname, board)	×
57	sync.default_unprocessed_log_dir = tdir	×
58	sync.default_processed_log_dir = os.path.join(tdir, "processed")	×
59	if not os.path.isdir(sync.default_processed_log_dir):	×
60	os.mkdir(sync.default_processed_log_dir)	×
61	sync() # go go	×
62
63
64	@click.command()	1✔
65	@click.option("--server", default="picard", help="Address of Elasticsearch server")	1✔
66	@click.option(	1✔
67	"--filename",
68	default="resource_utilization.csv",
69	help="Full path to resource utilization csv file generated by HDL builds",
70	)
71	def log_hdl_resources_from_csv(server, filename):	1✔
72	tel = telemetry.ingest(server=server)	×
73	tel.log_hdl_resources_from_csv(filename)	×
74
75
76	@click.command()	1✔
77	@click.option("--server", default="picard", help="Address of Elasticsearch server")	1✔
78	@click.argument("in_args", nargs=-1)	1✔
79	def log_artifacts(server, in_args):	1✔
80	entry = {	×
81	"url": "NA",
82	"server": "NA",
83	"job": "NA",
84	"job_no": 0,
85	"job_date": None,
86	"job_build_parameters": "NA",
87	"file_name": "NA",
88	"target_board": "NA",
89	"artifact_info_type": "NA",
90	"payload_raw": "NA",
91	"payload_ts": "NA",
92	"payload": "NA",
93	"payload_param": "NA"
94	}
95	if len(in_args) == 0:	×
96	click.echo("Must have non-zero arguments for database entry")	×
97	sys.exit(1)	×
98	if int(len(in_args) / 2) != len(in_args) / 2:	×
99	click.echo(	×
100	"ERROR: Number of inputs arguments must be even\n"
101	+ " and in the form of: entry1<space>value1<space>entry2<space>value2"
102	)
103	sys.exit(1)	×
104	for i in range(0, len(in_args), 2):	×
105	if in_args[i] in entry:	×
106	if in_args[i + 1].lower() == "true":	×
107	entry[in_args[i]] = True	×
108	elif in_args[i + 1].lower() == "false":	×
109	entry[in_args[i]] = False	×
110	else:
111	entry[in_args[i]] = in_args[i + 1]	×
112	else:
113	click.echo("ERROR: " + in_args[i] + " not a valid entry")	×
114	sys.exit(1)	×
115	tel = telemetry.ingest(server=server)	×
116	tel.log_artifacts(**entry)	×
117
118	@click.command()	1✔
119	@click.option("--jenkins-server", required=True, help="Address of Jenkins server")	1✔
120	@click.option("--jenkins-username", required=False, help="Username with access on the Jenkins server")	1✔
121	@click.option("--jenkins-password", required=False, help="Password for Jenkins username")	1✔
122	@click.option("--es-server", required=True, help="Address of Elasticsearch server")	1✔
123	@click.option("--job-name", default="HW_tests/HW_test_multiconfig", help="Name of Jenkins job")	1✔
124	@click.option("--job", multiple=True, help="Job(s)/build(s) to process")	1✔
125	def grab_and_log_artifacts(	1✔
126	jenkins_server,
127	jenkins_username,
128	jenkins_password,
129	es_server,
130	job_name,
131	job
132	):
133	if not len(job) > 0:	×
134	click.echo("Atleast 1 Job/Build (--job) is needed.")	×
135	sys.exit(1)	×
136	g = telemetry.gargantua(	×
137	jenkins_server,
138	jenkins_username,
139	jenkins_password,
140	es_server,
141	job_name,
142	job
143	)
144	g.log_artifacts()	×
145
146	@click.command()	1✔
147	@click.option("--server", default="picard", help="Address of Elasticsearch server")	1✔
148	@click.argument("in_args", nargs=-1)	1✔
149	def log_boot_logs(server, in_args):	1✔
150
151	tel = telemetry.ingest(server=server)	×
152	schema = tel.db.import_schema(tel._get_schema("boot_tests.json"))	×
153	entry = dict()	×
154	for k,v in schema["mappings"]["properties"].items():	×
155	if k == "source_adjacency_matrix":	×
156	continue	×
157	if v["type"] in ["txt","keyword"]:	×
158	entry.update({k:"NA"})	×
159	elif v["type"] in ["integer"]:	×
160	entry.update({k: 0})	×
161	elif v["type"] in ["boolean"]:	×
162	entry.update({k: False})	×
163	elif k == "jenkins_job_date":	×
164	entry.update({k:datetime.datetime.now()})	×
165	else:
166	entry.update({k: None})	×
167
168	if len(in_args) == 0:	×
169	click.echo("Must have non-zero arguments for database entry")	×
170	sys.exit(1)	×
171	if int(len(in_args) / 2) != len(in_args) / 2:	×
172	click.echo(	×
173	"ERROR: Number of inputs arguments must be even\n"
174	+ " and in the form of: entry1<space>value1<space>entry2<space>value2"
175	)
176	sys.exit(1)	×
177
178	for i in range(0, len(in_args), 2):	×
179	if in_args[i] in entry:	×
180	validate(in_args[i], in_args[i+1],schema)	×
181	if in_args[i + 1].lower() == "true":	×
182	entry[in_args[i]] = True	×
183	elif in_args[i + 1].lower() == "false":	×
184	entry[in_args[i]] = False	×
185	else:
186	entry[in_args[i]] = in_args[i + 1]	×
187	else:
188	click.echo("ERROR: " + in_args[i] + " not a valid entry")	×
189	sys.exit(1)	×
190
191	tel = telemetry.ingest(server=server)	×
192	tel.log_boot_tests(**entry)	×
193
194
195	@click.command()	1✔
196	def main(args=None):	1✔
197	"""Console script for telemetry."""
198	click.echo("Replace this message by putting your code into " "telemetry.cli.main")	1✔
199	click.echo("See click documentation at https://click.palletsprojects.com/")	1✔
200	return 0	1✔
201
202
203	cli.add_command(prod_logs_upload)	1✔
204	cli.add_command(log_boot_logs)	1✔
205	cli.add_command(log_hdl_resources_from_csv)	1✔
206	cli.add_command(log_artifacts)	1✔
207	cli.add_command(grab_and_log_artifacts)	1✔
208	cli.add_command(main)	1✔
209
210	if __name__ == "__main__":	1✔
211	sys.exit(cli()) # pragma: no cover

tfcollins / telemetry / 6650421557

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous