ionelmc / django-redisboard / 91

pending completion

Build # 91

Build Type

push

travis-ci

Committed by

ionelmc

Commit Message

solve issue #35 and #36

mark_safe some server column html
convert str key request parameter in bytes

Run Details

82 of 139 branches covered (58.99%)

7 of 7 new or added lines in 2 files covered. (100.0%)

397 of 521 relevant lines covered (76.2%)

10.86 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.54

/src/redisboard/views.py

from logging import getLogger

from django.conf import settings
from django.core.paginator import Paginator
from django.http import HttpResponseNotFound
from django.shortcuts import render
from django.utils.functional import curry
from redis.exceptions import ResponseError

from .utils import PY3
from .utils import LazySlicingIterable

try:
    from django.utils.datastructures import SortedDict as OrderedDict
except ImportError:
    from django.utils.datastructures import OrderedDict
    


logger = getLogger(__name__)

REDISBOARD_ITEMS_PER_PAGE = getattr(settings, 'REDISBOARD_ITEMS_PER_PAGE', 100)


def safeint(value):
    try:
        return int(value)
    except ValueError:
        return value


def _fixup_pair(pair):
    a, b = pair
    return a, safeint(b)


LENGTH_GETTERS = {
    b'list': lambda conn, key: conn.llen(key),
    b'string': lambda conn, key: conn.strlen(key),
    b'set': lambda conn, key: conn.scard(key),
    b'zset': lambda conn, key: conn.zcount(key, '-inf', '+inf'),
    b'hash': lambda conn, key: conn.hlen(key),
}


def _get_key_info(conn, key):
    try:
        obj_type = conn.type(key)
        pipe = conn.pipeline()
        try:
            pipe.object('REFCOUNT', key)
            pipe.object('ENCODING', key)
            pipe.object('IDLETIME', key)
            LENGTH_GETTERS[obj_type](pipe, key)
            pipe.ttl(key)

            refcount, encoding, idletime, obj_length, obj_ttl = pipe.execute()
        except ResponseError as exc:
            logger.exception("Failed to get object info for key %r: %s", key, exc)
            return {
                'type': obj_type,
                'name': key,
                'length': "n/a",
                'error': str(exc),
                'ttl': "n/a",
                'refcount': "n/a",
                'encoding': "n/a",
                'idletime': "n/a",
            }
        return {
            'type': obj_type,
            'name': key,
            'length': obj_length,
            'ttl': obj_ttl,
            'refcount': refcount,
            'encoding': encoding,
            'idletime': idletime,
        }
    except ResponseError as exc:
        logger.exception("Failed to get details for key %r: %s", key, exc)
        return {
            'type': "n/a",
            'length': "n/a",
            'name': key,
            'error': str(exc),
            'ttl': "n/a",
            'refcount': "n/a",
            'encoding': "n/a",
            'idletime': "n/a",
        }

VALUE_GETTERS = {
    b'list': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.lrange(key, start, end))],
    b'string': lambda conn, key, *args: [('string', conn.get(key))],
    b'set': lambda conn, key, *args: list(enumerate(conn.smembers(key))),
    b'zset': lambda conn, key, start=0, end=-1: [(pos + start, val)
                                                 for (pos, val) in enumerate(conn.zrange(key, start, end))],
    b'hash': lambda conn, key, *args: conn.hgetall(key).items(),
    b'n/a': lambda conn, key, *args: (),
}


def _get_key_details(conn, db, key, page):
    conn.execute_command('SELECT', db)
    details = _get_key_info(conn, key)
    details['db'] = db
    if details['type'] in ('list', 'zset'):
        details['data'] = Paginator(
            LazySlicingIterable(
                lambda: details['length'],
                curry(VALUE_GETTERS[details['type']], conn, key)
            ),
            REDISBOARD_ITEMS_PER_PAGE
        ).page(page)
    else:
        details['data'] = VALUE_GETTERS[details['type']](conn, key)

    return details

def _raw_get_db_summary(server, db):
    server.connection.execute_command('SELECT', db)
    pipe = server.connection.pipeline()

    pipe.dbsize()
    for i in range(server.sampling_threshold):
        pipe.randomkey()

    results = pipe.execute()
    size = results.pop(0)
    keys = sorted(set(results))

    pipe = server.connection.pipeline()
    for key in keys:
        pipe.execute_command('DEBUG', 'OBJECT', key)
        pipe.ttl(key)

    total_memory = 0
    volatile_memory = 0
    persistent_memory = 0
    total_keys = 0
    volatile_keys = 0
    persistent_keys = 0
    results = pipe.execute()
    for key, details, ttl in zip(keys, results[::2], results[1::2]):
        if not isinstance(details, dict):
            details = dict(_fixup_pair(i.split(b':'))
                           for i in details.split() if b':' in i)

        length = details[b'serializedlength'] + len(key)

        if ttl:
            persistent_memory += length
            persistent_keys += 1
        else:
            volatile_memory += length
            volatile_keys += 1
        total_memory += length
        total_keys += 1

    if total_keys:
        total_memory = (total_memory / total_keys) * size
    else:
        total_memory = 0

    if persistent_keys:
        persistent_memory = (persistent_memory / persistent_keys) * size
    else:
        persistent_memory = 0

    if volatile_keys:
        volatile_memory = (volatile_memory / volatile_keys) * size
    else:
        volatile_memory = 0
    return dict(
        size=size,
        total_memory=total_memory,
        volatile_memory=volatile_memory,
        persistent_memory=persistent_memory,
    )

def _get_db_summary(server, db):
    try:
        return _raw_get_db_summary(server, db)
    except ResponseError as exc:
        logger.exception("Failed to get summary for db %r: %s", db, exc)
        return dict(
            size=0,
            total_memory=0,
            volatile_memory=0,
            persistent_memory=0,
        )


def _get_db_details(server, db):
    conn = server.connection
    conn.execute_command('SELECT', db)
    size = conn.dbsize()

    key_details = {}
    if size > server.sampling_threshold:
        sampling = True
        pipe = conn.pipeline()
        for _ in (range if PY3 else xrange)(server.sampling_size):  # flake8=noqa
            pipe.randomkey()

        for key in set(pipe.execute()):
            key_details[key] = _get_key_info(conn, key)

    else:
        sampling = False
        for key in conn.keys():
            key_details[key] = _get_key_info(conn, key)

    return dict(
        keys=key_details,
        sampling=sampling,
    )


def inspect(request, server):
    stats = server.stats
    conn = server.connection
    database_details = OrderedDict()
    key_details = None

    if stats['status'] == 'UP':
        if 'key' in request.GET:
            key = request.GET['key']
            # request key parameter is of type str like: "b'asgi::group:users'"
            # change it by bytes type: b''asgi::group:users'
            if isinstance(key, str) and key.startswith("b'") and key.endswith("'"):
                key =str.encode(key[2:-1])
            db = request.GET.get('db', 0)
            page = request.GET.get('page', 1)
            key_details = _get_key_details(conn, db, key, page)
        else:
            databases = sorted(name[2:] for name in conn.info()
                               if name.startswith('db'))
            total_size = 0
            for db in databases:
                database_details[db] = summary = _get_db_summary(server, db)
                total_size += summary['size']
            if total_size < server.sampling_threshold:
                for db in databases:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
            elif 'db' in request.GET:
                db = request.GET['db']
                if db in database_details:
                    database_details[db].update(
                        _get_db_details(server, db),
                        active=True,
                    )
                else:
                    return HttpResponseNotFound("Unknown database.")

    return render(request, "redisboard/inspect.html", {
        'databases': database_details,
        'key_details': key_details,
        'original': server,
        'stats': stats,
        'app_label': 'redisboard',
    })

1	from logging import getLogger	15✔
2
3	from django.conf import settings	15✔
4	from django.core.paginator import Paginator	15✔
5	from django.http import HttpResponseNotFound	15✔
6	from django.shortcuts import render	15✔
7	from django.utils.functional import curry	15✔
8	from redis.exceptions import ResponseError	15✔
9
10	from .utils import PY3	15✔
11	from .utils import LazySlicingIterable	15✔
12
13	try:	15✔
14	from django.utils.datastructures import SortedDict as OrderedDict	15✔
15	except ImportError:	15✔
16	from django.utils.datastructures import OrderedDict	15✔
17
18
19
20	logger = getLogger(__name__)	15✔
21
22	REDISBOARD_ITEMS_PER_PAGE = getattr(settings, 'REDISBOARD_ITEMS_PER_PAGE', 100)	15✔
23
24
25	def safeint(value):	15✔
26	try:	15✔
27	return int(value)	15✔
28	except ValueError:	15✔
29	return value	15✔
30
31
32	def _fixup_pair(pair):	15✔
33	a, b = pair	15✔
34	return a, safeint(b)	15✔
35
36
37	LENGTH_GETTERS = {	15✔
38	b'list': lambda conn, key: conn.llen(key),
39	b'string': lambda conn, key: conn.strlen(key),
40	b'set': lambda conn, key: conn.scard(key),
41	b'zset': lambda conn, key: conn.zcount(key, '-inf', '+inf'),
42	b'hash': lambda conn, key: conn.hlen(key),
43	}
44
45
46	def _get_key_info(conn, key):	15✔
47	try:	15✔
48	obj_type = conn.type(key)	15✔
49	pipe = conn.pipeline()	15✔
50	try:	15✔
51	pipe.object('REFCOUNT', key)	15✔
52	pipe.object('ENCODING', key)	15✔
53	pipe.object('IDLETIME', key)	15✔
54	LENGTH_GETTERS[obj_type](pipe, key)	15✔
55	pipe.ttl(key)	15✔
56
57	refcount, encoding, idletime, obj_length, obj_ttl = pipe.execute()	15✔
58	except ResponseError as exc:	×
59	logger.exception("Failed to get object info for key %r: %s", key, exc)	×
60	return {	×
61	'type': obj_type,
62	'name': key,
63	'length': "n/a",
64	'error': str(exc),
65	'ttl': "n/a",
66	'refcount': "n/a",
67	'encoding': "n/a",
68	'idletime': "n/a",
69	}
70	return {	15✔
71	'type': obj_type,
72	'name': key,
73	'length': obj_length,
74	'ttl': obj_ttl,
75	'refcount': refcount,
76	'encoding': encoding,
77	'idletime': idletime,
78	}
79	except ResponseError as exc:	×
80	logger.exception("Failed to get details for key %r: %s", key, exc)	×
81	return {	×
82	'type': "n/a",
83	'length': "n/a",
84	'name': key,
85	'error': str(exc),
86	'ttl': "n/a",
87	'refcount': "n/a",
88	'encoding': "n/a",
89	'idletime': "n/a",
90	}
91
92	VALUE_GETTERS = {	15!
93	b'list': lambda conn, key, start=0, end=-1: [(pos + start, val)
94	for (pos, val) in enumerate(conn.lrange(key, start, end))],
95	b'string': lambda conn, key, *args: [('string', conn.get(key))],
96	b'set': lambda conn, key, *args: list(enumerate(conn.smembers(key))),
97	b'zset': lambda conn, key, start=0, end=-1: [(pos + start, val)
98	for (pos, val) in enumerate(conn.zrange(key, start, end))],
99	b'hash': lambda conn, key, *args: conn.hgetall(key).items(),
100	b'n/a': lambda conn, key, *args: (),
101	}
102
103
104	def _get_key_details(conn, db, key, page):	15✔
105	conn.execute_command('SELECT', db)	15✔
106	details = _get_key_info(conn, key)	15✔
107	details['db'] = db	15✔
108	if details['type'] in ('list', 'zset'):	15✔
109	details['data'] = Paginator(	2✔
110	LazySlicingIterable(
111	lambda: details['length'],
112	curry(VALUE_GETTERS[details['type']], conn, key)
113	),
114	REDISBOARD_ITEMS_PER_PAGE
115	).page(page)
116	else:
117	details['data'] = VALUE_GETTERS[details['type']](conn, key)	15✔
118
119	return details	15✔
120
121	def _raw_get_db_summary(server, db):	15✔
122	server.connection.execute_command('SELECT', db)	15✔
123	pipe = server.connection.pipeline()	15✔
124
125	pipe.dbsize()	15✔
126	for i in range(server.sampling_threshold):	15✔
127	pipe.randomkey()	15✔
128
129	results = pipe.execute()	15✔
130	size = results.pop(0)	15✔
131	keys = sorted(set(results))	15✔
132
133	pipe = server.connection.pipeline()	15✔
134	for key in keys:	15✔
135	pipe.execute_command('DEBUG', 'OBJECT', key)	15✔
136	pipe.ttl(key)	15✔
137
138	total_memory = 0	15✔
139	volatile_memory = 0	15✔
140	persistent_memory = 0	15✔
141	total_keys = 0	15✔
142	volatile_keys = 0	15✔
143	persistent_keys = 0	15✔
144	results = pipe.execute()	15✔
145	for key, details, ttl in zip(keys, results[::2], results[1::2]):	15✔
146	if not isinstance(details, dict):	15!
147	details = dict(_fixup_pair(i.split(b':'))	15✔
148	for i in details.split() if b':' in i)
149
150	length = details[b'serializedlength'] + len(key)	15✔
151
152	if ttl:	15!
153	persistent_memory += length	×
154	persistent_keys += 1	×
155	else:
156	volatile_memory += length	15✔
157	volatile_keys += 1	15✔
158	total_memory += length	15✔
159	total_keys += 1	15✔
160
161	if total_keys:	15!
162	total_memory = (total_memory / total_keys) * size	15✔
163	else:
164	total_memory = 0	×
165
166	if persistent_keys:	15!
167	persistent_memory = (persistent_memory / persistent_keys) * size	×
168	else:
169	persistent_memory = 0	15✔
170
171	if volatile_keys:	15!
172	volatile_memory = (volatile_memory / volatile_keys) * size	15✔
173	else:
174	volatile_memory = 0	×
175	return dict(	15✔
176	size=size,
177	total_memory=total_memory,
178	volatile_memory=volatile_memory,
179	persistent_memory=persistent_memory,
180	)
181
182	def _get_db_summary(server, db):	15✔
183	try:	15✔
184	return _raw_get_db_summary(server, db)	15✔
185	except ResponseError as exc:	×
186	logger.exception("Failed to get summary for db %r: %s", db, exc)	×
187	return dict(	×
188	size=0,
189	total_memory=0,
190	volatile_memory=0,
191	persistent_memory=0,
192	)
193
194
195	def _get_db_details(server, db):	15✔
196	conn = server.connection	15✔
197	conn.execute_command('SELECT', db)	15✔
198	size = conn.dbsize()	15✔
199
200	key_details = {}	15✔
201	if size > server.sampling_threshold:	15!
202	sampling = True	×
203	pipe = conn.pipeline()	×
204	for _ in (range if PY3 else xrange)(server.sampling_size): # flake8=noqa	×
205	pipe.randomkey()	×
206
207	for key in set(pipe.execute()):	×
208	key_details[key] = _get_key_info(conn, key)	×
209
210	else:
211	sampling = False	15✔
212	for key in conn.keys():	15✔
213	key_details[key] = _get_key_info(conn, key)	15✔
214
215	return dict(	15✔
216	keys=key_details,
217	sampling=sampling,
218	)
219
220
221	def inspect(request, server):	15✔
222	stats = server.stats	15✔
223	conn = server.connection	15✔
224	database_details = OrderedDict()	15✔
225	key_details = None	15✔
226
227	if stats['status'] == 'UP':	15!
228	if 'key' in request.GET:	15✔
229	key = request.GET['key']	15✔
230	# request key parameter is of type str like: "b'asgi::group:users'"
231	# change it by bytes type: b''asgi::group:users'
232	if isinstance(key, str) and key.startswith("b'") and key.endswith("'"):	15!
233	key =str.encode(key[2:-1])	×
234	db = request.GET.get('db', 0)	15✔
235	page = request.GET.get('page', 1)	15✔
236	key_details = _get_key_details(conn, db, key, page)	15✔
237	else:
238	databases = sorted(name[2:] for name in conn.info()	15✔
239	if name.startswith('db'))
240	total_size = 0	15✔
241	for db in databases:	15✔
242	database_details[db] = summary = _get_db_summary(server, db)	15✔
243	total_size += summary['size']	15✔
244	if total_size < server.sampling_threshold:	15!
245	for db in databases:	15✔
246	database_details[db].update(	15✔
247	_get_db_details(server, db),
248	active=True,
249	)
250	elif 'db' in request.GET:	×
251	db = request.GET['db']	×
252	if db in database_details:	×
253	database_details[db].update(	×
254	_get_db_details(server, db),
255	active=True,
256	)
257	else:
258	return HttpResponseNotFound("Unknown database.")	×
259
260	return render(request, "redisboard/inspect.html", {	15✔
261	'databases': database_details,
262	'key_details': key_details,
263	'original': server,
264	'stats': stats,
265	'app_label': 'redisboard',
266	})

ionelmc / django-redisboard / 91

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous