3649277334

Build Type

push

github

Committed by Pedro Paulo

Commit Message

Makes extensible metrics in remotecv

Run Details

71 of 117 branches covered (60.68%)

Branch coverage included in aggregate %.

52 of 93 new or added lines in 13 files covered. (55.91%)

56 existing lines in 4 files now uncovered.

330 of 541 relevant lines covered (61.0%)

3.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.83

/remotecv/worker.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

# remote cv service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com timehome@corp.globo.com

import argparse
import logging
import sys
from http.server import HTTPServer
from importlib import import_module
from threading import Thread

from remotecv.error_handler import ErrorHandler
from remotecv.healthcheck import HealthCheckHandler
from remotecv.importer import Importer
from remotecv.utils import config, redis_client, SINGLE_NODE, SENTINEL, context


def start_pyres_worker():
    from remotecv.unique_queue import (  # NOQA pylint: disable=import-outside-toplevel
        UniqueWorker,
    )

    redis = redis_client()

    def after_fork(_):
        config.error_handler.install_handler()

    worker = UniqueWorker(
        queues=["Detect"], server=redis, timeout=config.timeout, after_fork=after_fork
    )
    worker.work()


def start_celery_worker():
    from remotecv.celery_tasks import (  # NOQA pylint: disable=import-outside-toplevel
        CeleryTasks,
    )

    celery_tasks = CeleryTasks(
        config.key_id,
        config.key_secret,
        config.region,
        config.timeout,
        config.polling_interval,
    )
    celery_tasks.run_commands(config.extra_args, log_level=config.log_level)


def start_http_server():
    def serve_forever(httpd):
        with httpd:
            httpd.serve_forever()

    httpd = HTTPServer(("", config.server_port), HealthCheckHandler)

    thread = Thread(target=serve_forever, args=(httpd,))
    thread.setDaemon(True)
    thread.start()


def import_modules():
    Metrics = Importer.import_class("Metrics", config.metrics)
    context.metrics = Metrics(config)
    context.metrics.initialize()


def main(params=None):
    if params is None:
        params = sys.argv[1:]
    parser = argparse.ArgumentParser(description="Runs RemoteCV.")

    conn_group = parser.add_argument_group("Worker Backend")
    conn_group.add_argument(
        "-b",
        "--backend",
        default="pyres",
        choices=["pyres", "celery"],
        help="Worker backend",
    )

    conn_group = parser.add_argument_group("Pyres Connection Arguments")
    conn_group.add_argument("--host", default="localhost", help="Redis host")
    conn_group.add_argument("--port", default=6379, type=int, help="Redis port")
    conn_group.add_argument("--database", default=0, type=int, help="Redis database")
    conn_group.add_argument("--password", default=None, help="Redis password")
    conn_group.add_argument(
        "--redis-mode",
        default=SINGLE_NODE,
        choices=[SINGLE_NODE, SENTINEL],
        help="Redis mode",
    )
    conn_group.add_argument(
        "--sentinel-instances",
        default="localhost:26376",
        help="Redis Sentinel instances e.g. 'localhost:26376,localhost:26377'",
    )
    conn_group.add_argument(
        "--sentinel-password", default=None, help="Redis Sentinel password"
    )
    conn_group.add_argument(
        "--master-instance", default=None, help="Redis Sentinel master instance"
    )
    conn_group.add_argument(
        "--master-password", default=None, help="Redis Sentinel master password"
    )
    conn_group.add_argument(
        "--master-database",
        default=0,
        type=int,
        help="Redis Sentinel master database",
    )
    conn_group.add_argument(
        "--socket-timeout",
        default=10.0,
        type=float,
        help="Redis Sentinel socket timeout",
    )

    conn_group = parser.add_argument_group("Celery/SQS Connection Arguments")
    conn_group.add_argument("--region", default="us-east-1", help="AWS SQS Region")
    conn_group.add_argument("--key_id", default="", help="AWS access key id")
    conn_group.add_argument("--key_secret", default="", help="AWS access key secret")
    conn_group.add_argument(
        "--polling_interval", default=20, help="AWS polling interval"
    )

    other_group = parser.add_argument_group("Other arguments")
    other_group.add_argument(
        "--server-port", default=8080, type=int, help="Server http port"
    )
    other_group.add_argument(
        "--with-healthcheck",
        default=False,
        action='store_true',
        help="Start an healthchecker http endpoint",
    )
    other_group.add_argument("-l", "--level", default="debug", help="Logging level")
    other_group.add_argument(
        "-o", "--loader", default="remotecv.http_loader", help="Loader used"
    )
    other_group.add_argument(
        "-s", "--store", default="remotecv.result_store.redis_store", help="Loader used"
    )
    other_group.add_argument(
        "-t",
        "--timeout",
        default=None,
        type=int,
        help="Timeout in seconds for image detection",
    )
    other_group.add_argument(
        "--sentry_url", default=None, help="URL used to send errors to sentry"
    )
    other_group.add_argument(
        "--metrics",
        default="remotecv.metrics.logger_metrics",
        help="Metrics client, should be the full name of a python module",
    )

    memcache_store_group = parser.add_argument_group("Memcache store arguments")
    memcache_store_group.add_argument(
        "--memcache_hosts",
        default="localhost:11211",
        help="Comma separated list of memcache hosts",
    )

    parser.add_argument("args", nargs=argparse.REMAINDER)

    arguments = parser.parse_args(params)
    logging.basicConfig(level=getattr(logging, arguments.level.upper()))

    config.backend = arguments.backend
    config.redis_host = arguments.host
    config.redis_port = arguments.port
    config.redis_database = arguments.database
    config.redis_password = arguments.password
    config.redis_mode = arguments.redis_mode
    config.redis_sentinel_instances = arguments.sentinel_instances
    config.redis_sentinel_password = arguments.sentinel_password
    config.redis_sentinel_socket_timeout = arguments.socket_timeout
    config.redis_sentinel_master_instance = arguments.master_instance
    config.redis_sentinel_master_password = arguments.master_password
    config.redis_sentinel_master_database = arguments.master_database

    config.region = arguments.region
    config.key_id = arguments.key_id
    config.key_secret = arguments.key_secret
    config.polling_interval = arguments.polling_interval

    config.timeout = arguments.timeout
    config.server_port = arguments.server_port
    config.log_level = arguments.level.upper()
    config.loader = import_module(arguments.loader)
    config.store = import_module(arguments.store)
    config.metrics = arguments.metrics

    config.memcache_hosts = arguments.memcache_hosts

    config.extra_args = sys.argv[:1] + arguments.args

    config.error_handler = ErrorHandler(arguments.sentry_url)

    import_modules()

    if arguments.with_healthcheck:
        start_http_server()

    if arguments.backend == "pyres":
        start_pyres_worker()
    elif arguments.backend == "celery":
        start_celery_worker()


if __name__ == "__main__":
    main()

1	#!/usr/bin/python
2	# -- coding: utf-8 --
3
4	# remote cv service
5	# https://github.com/globocom/thumbor/wiki
6
7	# Licensed under the MIT license:
8	# http://www.opensource.org/licenses/mit-license
9	# Copyright (c) 2011 globo.com timehome@corp.globo.com
10
11	import argparse	5✔
12	import logging	5✔
13	import sys	5✔
14	from http.server import HTTPServer	5✔
15	from importlib import import_module	5✔
16	from threading import Thread	5✔
17
18	from remotecv.error_handler import ErrorHandler	5✔
19	from remotecv.healthcheck import HealthCheckHandler	5✔
20	from remotecv.importer import Importer	5✔
21	from remotecv.utils import config, redis_client, SINGLE_NODE, SENTINEL, context	5✔
22
23
24	def start_pyres_worker():	5✔
UNCOV 25	from remotecv.unique_queue import ( # NOQA pylint: disable=import-outside-toplevel	×
26	UniqueWorker,
27	)
28
UNCOV 29	redis = redis_client()	×
30
UNCOV 31	def after_fork(_):	×
32	config.error_handler.install_handler()	×
33
UNCOV 34	worker = UniqueWorker(	×
35	queues=["Detect"], server=redis, timeout=config.timeout, after_fork=after_fork
36	)
UNCOV 37	worker.work()	×
38
39
40	def start_celery_worker():	5✔
UNCOV 41	from remotecv.celery_tasks import ( # NOQA pylint: disable=import-outside-toplevel	×
42	CeleryTasks,
43	)
44
UNCOV 45	celery_tasks = CeleryTasks(	×
46	config.key_id,
47	config.key_secret,
48	config.region,
49	config.timeout,
50	config.polling_interval,
51	)
UNCOV 52	celery_tasks.run_commands(config.extra_args, log_level=config.log_level)	×
53
54
55	def start_http_server():	5✔
56	def serve_forever(httpd):	5✔
57	with httpd:	5✔
58	httpd.serve_forever()	5✔
59
60	httpd = HTTPServer(("", config.server_port), HealthCheckHandler)	5✔
61
62	thread = Thread(target=serve_forever, args=(httpd,))	5✔
63	thread.setDaemon(True)	5✔
64	thread.start()	5✔
65
66
67	def import_modules():	5✔
NEW 68	Metrics = Importer.import_class("Metrics", config.metrics)	×
NEW 69	context.metrics = Metrics(config)	×
NEW 70	context.metrics.initialize()	×
71
72
73	def main(params=None):	5✔
UNCOV 74	if params is None:	×
75	params = sys.argv[1:]	×
76	parser = argparse.ArgumentParser(description="Runs RemoteCV.")	×
77
UNCOV 78	conn_group = parser.add_argument_group("Worker Backend")	×
UNCOV 79	conn_group.add_argument(	×
80	"-b",
81	"--backend",
82	default="pyres",
83	choices=["pyres", "celery"],
84	help="Worker backend",
85	)
86
UNCOV 87	conn_group = parser.add_argument_group("Pyres Connection Arguments")	×
UNCOV 88	conn_group.add_argument("--host", default="localhost", help="Redis host")	×
UNCOV 89	conn_group.add_argument("--port", default=6379, type=int, help="Redis port")	×
UNCOV 90	conn_group.add_argument("--database", default=0, type=int, help="Redis database")	×
UNCOV 91	conn_group.add_argument("--password", default=None, help="Redis password")	×
UNCOV 92	conn_group.add_argument(	×
93	"--redis-mode",
94	default=SINGLE_NODE,
95	choices=[SINGLE_NODE, SENTINEL],
96	help="Redis mode",
97	)
98	conn_group.add_argument(	×
99	"--sentinel-instances",
100	default="localhost:26376",
101	help="Redis Sentinel instances e.g. 'localhost:26376,localhost:26377'",
102	)
UNCOV 103	conn_group.add_argument(	×
104	"--sentinel-password", default=None, help="Redis Sentinel password"
105	)
UNCOV 106	conn_group.add_argument(	×
107	"--master-instance", default=None, help="Redis Sentinel master instance"
108	)
UNCOV 109	conn_group.add_argument(	×
110	"--master-password", default=None, help="Redis Sentinel master password"
111	)
UNCOV 112	conn_group.add_argument(	×
113	"--master-database",
114	default=0,
115	type=int,
116	help="Redis Sentinel master database",
117	)
UNCOV 118	conn_group.add_argument(	×
119	"--socket-timeout",
120	default=10.0,
121	type=float,
122	help="Redis Sentinel socket timeout",
123	)
124
125	conn_group = parser.add_argument_group("Celery/SQS Connection Arguments")	×
UNCOV 126	conn_group.add_argument("--region", default="us-east-1", help="AWS SQS Region")	×
UNCOV 127	conn_group.add_argument("--key_id", default="", help="AWS access key id")	×
UNCOV 128	conn_group.add_argument("--key_secret", default="", help="AWS access key secret")	×
UNCOV 129	conn_group.add_argument(	×
130	"--polling_interval", default=20, help="AWS polling interval"
131	)
132
133	other_group = parser.add_argument_group("Other arguments")	×
134	other_group.add_argument(	×
135	"--server-port", default=8080, type=int, help="Server http port"
136	)
UNCOV 137	other_group.add_argument(	×
138	"--with-healthcheck",
139	default=False,
140	action='store_true',
141	help="Start an healthchecker http endpoint",
142	)
UNCOV 143	other_group.add_argument("-l", "--level", default="debug", help="Logging level")	×
144	other_group.add_argument(	×
145	"-o", "--loader", default="remotecv.http_loader", help="Loader used"
146	)
UNCOV 147	other_group.add_argument(	×
148	"-s", "--store", default="remotecv.result_store.redis_store", help="Loader used"
149	)
150	other_group.add_argument(	×
151	"-t",
152	"--timeout",
153	default=None,
154	type=int,
155	help="Timeout in seconds for image detection",
156	)
157	other_group.add_argument(	×
158	"--sentry_url", default=None, help="URL used to send errors to sentry"
159	)
NEW 160	other_group.add_argument(	×
161	"--metrics",
162	default="remotecv.metrics.logger_metrics",
163	help="Metrics client, should be the full name of a python module",
164	)
165
UNCOV 166	memcache_store_group = parser.add_argument_group("Memcache store arguments")	×
UNCOV 167	memcache_store_group.add_argument(	×
168	"--memcache_hosts",
169	default="localhost:11211",
170	help="Comma separated list of memcache hosts",
171	)
172
UNCOV 173	parser.add_argument("args", nargs=argparse.REMAINDER)	×
174
UNCOV 175	arguments = parser.parse_args(params)	×
UNCOV 176	logging.basicConfig(level=getattr(logging, arguments.level.upper()))	×
177
178	config.backend = arguments.backend	×
179	config.redis_host = arguments.host	×
UNCOV 180	config.redis_port = arguments.port	×
UNCOV 181	config.redis_database = arguments.database	×
UNCOV 182	config.redis_password = arguments.password	×
UNCOV 183	config.redis_mode = arguments.redis_mode	×
UNCOV 184	config.redis_sentinel_instances = arguments.sentinel_instances	×
185	config.redis_sentinel_password = arguments.sentinel_password	×
UNCOV 186	config.redis_sentinel_socket_timeout = arguments.socket_timeout	×
187	config.redis_sentinel_master_instance = arguments.master_instance	×
188	config.redis_sentinel_master_password = arguments.master_password	×
UNCOV 189	config.redis_sentinel_master_database = arguments.master_database	×
190
191	config.region = arguments.region	×
192	config.key_id = arguments.key_id	×
193	config.key_secret = arguments.key_secret	×
194	config.polling_interval = arguments.polling_interval	×
195
196	config.timeout = arguments.timeout	×
197	config.server_port = arguments.server_port	×
198	config.log_level = arguments.level.upper()	×
199	config.loader = import_module(arguments.loader)	×
200	config.store = import_module(arguments.store)	×
NEW 201	config.metrics = arguments.metrics	×
202
UNCOV 203	config.memcache_hosts = arguments.memcache_hosts	×
204
205	config.extra_args = sys.argv[:1] + arguments.args	×
206
207	config.error_handler = ErrorHandler(arguments.sentry_url)	×
208
NEW 209	import_modules()	×
210
211	if arguments.with_healthcheck:	×
212	start_http_server()	×
213
214	if arguments.backend == "pyres":	×
215	start_pyres_worker()	×
216	elif arguments.backend == "celery":	×
UNCOV 217	start_celery_worker()	×
218
219
220	if __name__ == "__main__":	5✔
221	main()

thumbor / remotecv / 3649277334

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous