1239

pending completion

Build # 1239

Build Type

push

travis-ci

Committed by

JohanLorenzo

Commit Message

20.0.0

Run Details

2331 of 2331 relevant lines covered (100.0%)

2.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0

/scriptworker/client.py

#!/usr/bin/env python
"""Scripts running in scriptworker will use functions in this file.

This module should be largely standalone.  This should only depend on
scriptworker.exceptions and scriptworker.constants, or other standalone
modules, to avoid circular imports.

Attributes:
    log (logging.Logger): the log object for the module

"""
import aiohttp
import asyncio
import jsonschema
import logging
import os
import sys
from urllib.parse import unquote

from scriptworker.constants import STATUSES
from scriptworker.context import Context
from scriptworker.exceptions import ScriptWorkerException, ScriptWorkerTaskException, TaskVerificationError
from scriptworker.utils import load_json_or_yaml, match_url_regex

log = logging.getLogger(__name__)


def get_task(config):
    """Read the task.json from work_dir.

    Args:
        config (dict): the running config, to find work_dir.

    Returns:
        dict: the contents of task.json

    Raises:
        ScriptWorkerTaskException: on error.

    """
    path = os.path.join(config['work_dir'], "task.json")
    message = "Can't read task from {}!\n%(exc)s".format(path)
    contents = load_json_or_yaml(path, is_path=True, message=message)
    return contents


def validate_json_schema(data, schema, name="task"):
    """Given data and a jsonschema, let's validate it.

    This happens for tasks and chain of trust artifacts.

    Args:
        data (dict): the json to validate.
        schema (dict): the jsonschema to validate against.
        name (str, optional): the name of the json, for exception messages.
            Defaults to "task".

    Raises:
        ScriptWorkerTaskException: on failure

    """
    try:
        jsonschema.validate(data, schema)
    except jsonschema.exceptions.ValidationError as exc:
        raise ScriptWorkerTaskException(
            "Can't validate {} schema!\n{}".format(name, str(exc)),
            exit_code=STATUSES['malformed-payload']
        )


def validate_task_schema(context, schema_key='schema_file'):
    """Validate the task definition.

    Args:
        context (scriptworker.context.Context): the scriptworker context. It must contain a task and
            the config pointing to the schema file
        schema_key: the key in `context.config` where the path to the schema file is. Key can contain
            dots (e.g.: 'schema_files.file_a'), in which case

    Raises:
        TaskVerificationError: if the task doesn't match the schema

    """
    schema_path = context.config
    schema_keys = schema_key.split('.')
    for key in schema_keys:
        schema_path = schema_path[key]

    task_schema = load_json_or_yaml(schema_path, is_path=True)
    log.debug('Task is validated against this schema: {}'.format(task_schema))

    try:
        validate_json_schema(context.task, task_schema)
    except ScriptWorkerTaskException as e:
        raise TaskVerificationError('Cannot validate task against schema. Task: {}.'.format(context.task)) from e


def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):
    """Ensure a URL fits in given scheme, netloc, and path restrictions.

    If we fail any checks, raise a ScriptWorkerTaskException with
    ``malformed-payload``.

    Args:
        valid_artifact_rules (tuple): the tests to run, with ``schemas``, ``netlocs``,
            and ``path_regexes``.
        valid_artifact_task_ids (list): the list of valid task IDs to download from.
        url (str): the url of the artifact.

    Returns:
        str: the ``filepath`` of the path regex.

    Raises:
        ScriptWorkerTaskException: on failure to validate.

    """
    def callback(match):
        path_info = match.groupdict()
        # make sure we're pointing at a valid task ID
        if 'taskId' in path_info and \
                path_info['taskId'] not in valid_artifact_task_ids:
            return
        if 'filepath' not in path_info:
            return
        return path_info['filepath']

    filepath = match_url_regex(valid_artifact_rules, url, callback)
    if filepath is None:
        raise ScriptWorkerTaskException(
            "Can't validate url {}".format(url),
            exit_code=STATUSES['malformed-payload']
        )
    return unquote(filepath).lstrip('/')


def sync_main(async_main, config_path=None, default_config=None,
              should_validate_task=True, loop_function=asyncio.get_event_loop):
    """Entry point for scripts using scriptworker.

    This function sets up the basic needs for a script to run. More specifically:
        * it creates the scriptworker context and initializes it with the provided config
        * the path to the config file is either taken from `config_path` or from `sys.argv[1]`.
        * it verifies `sys.argv` doesn't have more arguments than the config path.
        * it creates the asyncio event loop so that `async_main` can run

    Args:
        async_main (function): The function to call once everything is set up
        config_path (str, optional): The path to the file to load the config from.
            Loads from ``sys.argv[1]`` if ``None``. Defaults to None.
        default_config (dict, optional): the default config to use for ``_init_context``.
            defaults to None.
        should_validate_task (bool, optional): whether we should validate the task
            schema. Defaults to True.
        loop_function (function, optional): the function to call to get the
            event loop; here for testing purposes. Defaults to
            ``asyncio.get_event_loop``.

    """
    context = _init_context(config_path, default_config)
    _init_logging(context)
    if should_validate_task:
        validate_task_schema(context)
    loop = loop_function()
    loop.run_until_complete(_handle_asyncio_loop(async_main, context))


def _init_context(config_path=None, default_config=None):
    context = Context()

    # This prevents *script from pasting the whole context.
    context.write_json = lambda *args: None

    if config_path is None:
        if len(sys.argv) != 2:
            _usage()
        config_path = sys.argv[1]

    context.config = {} if default_config is None else default_config
    context.config.update(load_json_or_yaml(config_path, is_path=True))

    context.task = get_task(context.config)

    return context


def _usage():
    print('Usage: {} CONFIG_FILE'.format(sys.argv[0]), file=sys.stderr)
    sys.exit(1)


def _init_logging(context):
    logging.basicConfig(
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        level=logging.DEBUG if context.config.get('verbose') else logging.INFO
    )
    logging.getLogger('taskcluster').setLevel(logging.WARNING)


async def _handle_asyncio_loop(async_main, context):
    async with aiohttp.ClientSession() as session:
        context.session = session
        try:
            await async_main(context)
        except ScriptWorkerException as exc:
            log.exception("Failed to run async_main")
            sys.exit(exc.exit_code)

1	#!/usr/bin/env python
2	"""Scripts running in scriptworker will use functions in this file.	2✔
3
4	This module should be largely standalone. This should only depend on
5	scriptworker.exceptions and scriptworker.constants, or other standalone
6	modules, to avoid circular imports.
7
8	Attributes:
9	log (logging.Logger): the log object for the module
10
11	"""
12	import aiohttp	2✔
13	import asyncio	2✔
14	import jsonschema	2✔
15	import logging	2✔
16	import os	2✔
17	import sys	2✔
18	from urllib.parse import unquote	2✔
19
20	from scriptworker.constants import STATUSES	2✔
21	from scriptworker.context import Context	2✔
22	from scriptworker.exceptions import ScriptWorkerException, ScriptWorkerTaskException, TaskVerificationError	2✔
23	from scriptworker.utils import load_json_or_yaml, match_url_regex	2✔
24
25	log = logging.getLogger(__name__)	2✔
26
27
28	def get_task(config):	2✔
29	"""Read the task.json from work_dir.
30
31	Args:
32	config (dict): the running config, to find work_dir.
33
34	Returns:
35	dict: the contents of task.json
36
37	Raises:
38	ScriptWorkerTaskException: on error.
39
40	"""
41	path = os.path.join(config['work_dir'], "task.json")	2✔
42	message = "Can't read task from {}!\n%(exc)s".format(path)	2✔
43	contents = load_json_or_yaml(path, is_path=True, message=message)	2✔
44	return contents	2✔
45
46
47	def validate_json_schema(data, schema, name="task"):	2✔
48	"""Given data and a jsonschema, let's validate it.
49
50	This happens for tasks and chain of trust artifacts.
51
52	Args:
53	data (dict): the json to validate.
54	schema (dict): the jsonschema to validate against.
55	name (str, optional): the name of the json, for exception messages.
56	Defaults to "task".
57
58	Raises:
59	ScriptWorkerTaskException: on failure
60
61	"""
62	try:	2✔
63	jsonschema.validate(data, schema)	2✔
64	except jsonschema.exceptions.ValidationError as exc:	2✔
65	raise ScriptWorkerTaskException(	2✔
66	"Can't validate {} schema!\n{}".format(name, str(exc)),
67	exit_code=STATUSES['malformed-payload']
68	)
69
70
71	def validate_task_schema(context, schema_key='schema_file'):	2✔
72	"""Validate the task definition.
73
74	Args:
75	context (scriptworker.context.Context): the scriptworker context. It must contain a task and
76	the config pointing to the schema file
77	schema_key: the key in `context.config` where the path to the schema file is. Key can contain
78	dots (e.g.: 'schema_files.file_a'), in which case
79
80	Raises:
81	TaskVerificationError: if the task doesn't match the schema
82
83	"""
84	schema_path = context.config	2✔
85	schema_keys = schema_key.split('.')	2✔
86	for key in schema_keys:	2✔
87	schema_path = schema_path[key]	2✔
88
89	task_schema = load_json_or_yaml(schema_path, is_path=True)	2✔
90	log.debug('Task is validated against this schema: {}'.format(task_schema))	2✔
91
92	try:	2✔
93	validate_json_schema(context.task, task_schema)	2✔
94	except ScriptWorkerTaskException as e:	2✔
95	raise TaskVerificationError('Cannot validate task against schema. Task: {}.'.format(context.task)) from e	2✔
96
97
98	def validate_artifact_url(valid_artifact_rules, valid_artifact_task_ids, url):	2✔
99	"""Ensure a URL fits in given scheme, netloc, and path restrictions.
100
101	If we fail any checks, raise a ScriptWorkerTaskException with
102	``malformed-payload``.
103
104	Args:
105	valid_artifact_rules (tuple): the tests to run, with ``schemas``, ``netlocs``,
106	and ``path_regexes``.
107	valid_artifact_task_ids (list): the list of valid task IDs to download from.
108	url (str): the url of the artifact.
109
110	Returns:
111	str: the ``filepath`` of the path regex.
112
113	Raises:
114	ScriptWorkerTaskException: on failure to validate.
115
116	"""
117	def callback(match):	2✔
118	path_info = match.groupdict()	2✔
119	# make sure we're pointing at a valid task ID
120	if 'taskId' in path_info and \	2✔
121	path_info['taskId'] not in valid_artifact_task_ids:
122	return	2✔
123	if 'filepath' not in path_info:	2✔
124	return	2✔
125	return path_info['filepath']	2✔
126
127	filepath = match_url_regex(valid_artifact_rules, url, callback)	2✔
128	if filepath is None:	2✔
129	raise ScriptWorkerTaskException(	2✔
130	"Can't validate url {}".format(url),
131	exit_code=STATUSES['malformed-payload']
132	)
133	return unquote(filepath).lstrip('/')	2✔
134
135
136	def sync_main(async_main, config_path=None, default_config=None,	2✔
137	should_validate_task=True, loop_function=asyncio.get_event_loop):
138	"""Entry point for scripts using scriptworker.
139
140	This function sets up the basic needs for a script to run. More specifically:
141	* it creates the scriptworker context and initializes it with the provided config
142	* the path to the config file is either taken from `config_path` or from `sys.argv[1]`.
143	* it verifies `sys.argv` doesn't have more arguments than the config path.
144	* it creates the asyncio event loop so that `async_main` can run
145
146	Args:
147	async_main (function): The function to call once everything is set up
148	config_path (str, optional): The path to the file to load the config from.
149	Loads from ``sys.argv[1]`` if ``None``. Defaults to None.
150	default_config (dict, optional): the default config to use for ``_init_context``.
151	defaults to None.
152	should_validate_task (bool, optional): whether we should validate the task
153	schema. Defaults to True.
154	loop_function (function, optional): the function to call to get the
155	event loop; here for testing purposes. Defaults to
156	``asyncio.get_event_loop``.
157
158	"""
159	context = _init_context(config_path, default_config)	2✔
160	_init_logging(context)	2✔
161	if should_validate_task:	2✔
162	validate_task_schema(context)	2✔
163	loop = loop_function()	2✔
164	loop.run_until_complete(_handle_asyncio_loop(async_main, context))	2✔
165
166
167	def _init_context(config_path=None, default_config=None):	2✔
168	context = Context()	2✔
169
170	# This prevents *script from pasting the whole context.
171	context.write_json = lambda *args: None	2✔
172
173	if config_path is None:	2✔
174	if len(sys.argv) != 2:	2✔
175	_usage()	2✔
176	config_path = sys.argv[1]	2✔
177
178	context.config = {} if default_config is None else default_config	2✔
179	context.config.update(load_json_or_yaml(config_path, is_path=True))	2✔
180
181	context.task = get_task(context.config)	2✔
182
183	return context	2✔
184
185
186	def _usage():	2✔
187	print('Usage: {} CONFIG_FILE'.format(sys.argv[0]), file=sys.stderr)	2✔
188	sys.exit(1)	2✔
189
190
191	def _init_logging(context):	2✔
192	logging.basicConfig(	2✔
193	format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
194	level=logging.DEBUG if context.config.get('verbose') else logging.INFO
195	)
196	logging.getLogger('taskcluster').setLevel(logging.WARNING)	2✔
197
198
199	async def _handle_asyncio_loop(async_main, context):	2✔
200	async with aiohttp.ClientSession() as session:	2✔
201	context.session = session	2✔
202	try:	2✔
203	await async_main(context)	2✔
204	except ScriptWorkerException as exc:	2✔
205	log.exception("Failed to run async_main")	2✔
206	sys.exit(exc.exit_code)	2✔

mozilla-releng / scriptworker / 1239

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous