BertrandBordage / django-cachalot / 471

pending completion

Build # 471

Build Type

Pull #94

travis-ci

Committed by

web-flow

Commit Message

Fix some client returns invalid get_many response

Some client's get_many method, e.g pymemcache,
returns like `{'key': False}`.

This cause following TypeError.

```
timestamp, result = data.pop(cache_key)
TypeError: 'bool' object is not iterable
```

This commit check `result` is valid and if invalid value were returned,
ignore and return query result.

Pull Request Pull Request #94: Fix some client returns invalid get_many response

Run Details

29 of 29 new or added lines in 3 files covered. (100.0%)

2996 of 3010 relevant lines covered (99.53%)

42.09 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.07

/cachalot/utils.py

# coding: utf-8

from __future__ import unicode_literals
import datetime
from decimal import Decimal
from hashlib import sha1
from time import time
from uuid import UUID

from django import VERSION as django_version
from django.db import connections
from django.db.models import QuerySet
from django.db.models.sql import Query
from django.db.models.sql.where import (
    ExtraWhere, SubqueryConstraint, WhereNode)
from django.utils.six import text_type, binary_type, PY2

from .settings import ITERABLES, cachalot_settings
from .transaction import AtomicCache


class UncachableQuery(Exception):
    pass


class IsRawQuery(Exception):
    pass


CACHABLE_PARAM_TYPES = {
    bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
    datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
}

if PY2:
    CACHABLE_PARAM_TYPES.add(long)

UNCACHABLE_FUNCS = set()
if django_version[:2] >= (1, 9):
    from django.db.models.functions import Now
    from django.contrib.postgres.functions import TransactionNow
    UNCACHABLE_FUNCS.update((Now, TransactionNow))

try:
    from psycopg2 import Binary
    from psycopg2.extras import (
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
except ImportError:
    pass
else:
    CACHABLE_PARAM_TYPES.update((
        Binary,
        NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json))
    if django_version[:2] >= (1, 11):
        from django.contrib.postgres.fields.jsonb import JsonAdapter
        CACHABLE_PARAM_TYPES.add(JsonAdapter)


def check_parameter_types(params):
    for p in params:
        cl = p.__class__
        if cl not in CACHABLE_PARAM_TYPES:
            if cl in ITERABLES:
                check_parameter_types(p)
            elif cl is dict:
                check_parameter_types(p.items())
            else:
                raise UncachableQuery


def get_query_cache_key(compiler):
    """
    Generates a cache key from a SQLCompiler.

    This cache key is specific to the SQL query and its context
    (which database is used).  The same query in the same context
    (= the same database) must generate the same cache key.

    :arg compiler: A SQLCompiler that will generate the SQL query
    :type compiler: django.db.models.sql.compiler.SQLCompiler
    :return: A cache key
    :rtype: int
    """
    sql, params = compiler.as_sql()
    check_parameter_types(params)
    cache_key = '%s:%s:%s' % (compiler.using, sql,
                              [text_type(p) for p in params])
    return sha1(cache_key.encode('utf-8')).hexdigest()


def get_table_cache_key(db_alias, table):
    """
    Generates a cache key from a SQL table.

    :arg db_alias: Alias of the used database
    :type db_alias: str or unicode
    :arg table: Name of the SQL table
    :type table: str or unicode
    :return: A cache key
    :rtype: int
    """
    cache_key = '%s:%s' % (db_alias, table)
    return sha1(cache_key.encode('utf-8')).hexdigest()


def _get_tables_from_sql(connection, lowercased_sql):
    return {t for t in connection.introspection.django_table_names()
            if t in lowercased_sql}


def _find_subqueries(children):
    for child in children:
        child_class = child.__class__
        if child_class is WhereNode:
            for grand_child in _find_subqueries(child.children):
                yield grand_child
        # TODO: Remove this condition when we drop Django 1.8 support.
        elif child_class is SubqueryConstraint:
            query_object = child.query_object
            yield (query_object if query_object.__class__ is Query
                   else query_object.query)
        elif child_class is ExtraWhere:
            raise IsRawQuery
        else:
            rhs = getattr(child, 'rhs', None)
            rhs_class = rhs.__class__
            if rhs_class is Query:
                yield rhs
            elif rhs_class is QuerySet:
                yield rhs.query
            elif rhs_class in UNCACHABLE_FUNCS:
                raise UncachableQuery


def is_cachable(table):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and table not in whitelist:
        return False
    return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES


def are_all_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    if whitelist and not tables.issubset(whitelist):
        return False
    return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)


def filter_cachable(tables):
    whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES
    tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)
    if whitelist:
        return tables.intersection(whitelist)
    return tables


def _get_tables(db_alias, query):
    if query.select_for_update or (
            not cachalot_settings.CACHALOT_CACHE_RANDOM
            and '?' in query.order_by):
        raise UncachableQuery

    try:
        if query.extra_select or getattr(query, 'subquery', False):
            raise IsRawQuery
        tables = set(query.table_map)
        tables.add(query.get_meta().db_table)
        for subquery in _find_subqueries(query.where.children):
            tables.update(_get_tables(db_alias, subquery))
    except IsRawQuery:
        sql = query.get_compiler(db_alias).as_sql()[0].lower()
        tables = _get_tables_from_sql(connections[db_alias], sql)

    if not are_all_cachable(tables):
        raise UncachableQuery
    return tables


def _get_table_cache_keys(compiler):
    db_alias = compiler.using
    get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
    return [get_table_cache_key(db_alias, t)
            for t in _get_tables(db_alias, compiler.query)]


def _invalidate_tables(cache, db_alias, tables):
    tables = filter_cachable(set(tables))
    if not tables:
        return
    now = time()
    get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN
    cache.set_many(
        {get_table_cache_key(db_alias, t): now for t in tables},
        cachalot_settings.CACHALOT_TIMEOUT)

    if isinstance(cache, AtomicCache):
        cache.to_be_invalidated.update(tables)

1	# coding: utf-8
2
3	from __future__ import unicode_literals	45✔
4	import datetime	45✔
5	from decimal import Decimal	45✔
6	from hashlib import sha1	45✔
7	from time import time	45✔
8	from uuid import UUID	45✔
9
10	from django import VERSION as django_version	45✔
11	from django.db import connections	45✔
12	from django.db.models import QuerySet	45✔
13	from django.db.models.sql import Query	45✔
14	from django.db.models.sql.where import (	45✔
15	ExtraWhere, SubqueryConstraint, WhereNode)
16	from django.utils.six import text_type, binary_type, PY2	45✔
17
18	from .settings import ITERABLES, cachalot_settings	45✔
19	from .transaction import AtomicCache	45✔
20
21
22	class UncachableQuery(Exception):	45✔
23	pass	45✔
24
25
26	class IsRawQuery(Exception):	45✔
27	pass	45✔
28
29
30	CACHABLE_PARAM_TYPES = {	45✔
31	bool, int, float, Decimal, bytearray, binary_type, text_type, type(None),
32	datetime.date, datetime.time, datetime.datetime, datetime.timedelta, UUID,
33	}
34
35	if PY2:	45✔
36	CACHABLE_PARAM_TYPES.add(long)	×
37
38	UNCACHABLE_FUNCS = set()	45✔
39	if django_version[:2] >= (1, 9):	45✔
40	from django.db.models.functions import Now	45✔
41	from django.contrib.postgres.functions import TransactionNow	45✔
42	UNCACHABLE_FUNCS.update((Now, TransactionNow))	45✔
43
44	try:	45✔
45	from psycopg2 import Binary	45✔
46	from psycopg2.extras import (	45✔
47	NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json)
48	except ImportError:	×
49	pass	×
50	else:
51	CACHABLE_PARAM_TYPES.update((	45✔
52	Binary,
53	NumericRange, DateRange, DateTimeRange, DateTimeTZRange, Inet, Json))
54	if django_version[:2] >= (1, 11):	45✔
55	from django.contrib.postgres.fields.jsonb import JsonAdapter	30 all except TOXENV=py3.5-django1.10-sqlite3-filebased, TOXENV=py3.5-django1.10-postgresql-redis, TOXENV=py3.5-django1.10-postgresql-memcached, TOXENV=py3.5-django1.10-postgresql-locmem, TOXENV=py3.5-django1.10-sqlite3-memcached, TOXENV=py3.5-django1.10-postgresql-pylibmc, TOXENV=py3.5-django1.10-mysql-pylibmc, TOXENV=py3.5-django1.10-sqlite3-pylibmc, TOXENV=py3.5-django1.10-mysql-locmem, TOXENV=py3.5-django1.10-mysql-filebased, TOXENV=py3.5-django1.10-sqlite3-locmem, TOXENV=py3.5-django1.10-sqlite3-redis, TOXENV=py3.5-django1.10-mysql-redis, TOXENV=py3.5-django1.10-mysql-memcached, and TOXENV=py3.5-django1.10-postgresql-filebased ✔
56	CACHABLE_PARAM_TYPES.add(JsonAdapter)	30 all except TOXENV=py3.5-django1.10-sqlite3-filebased, TOXENV=py3.5-django1.10-postgresql-redis, TOXENV=py3.5-django1.10-postgresql-memcached, TOXENV=py3.5-django1.10-postgresql-locmem, TOXENV=py3.5-django1.10-sqlite3-memcached, TOXENV=py3.5-django1.10-postgresql-pylibmc, TOXENV=py3.5-django1.10-mysql-pylibmc, TOXENV=py3.5-django1.10-sqlite3-pylibmc, TOXENV=py3.5-django1.10-mysql-locmem, TOXENV=py3.5-django1.10-mysql-filebased, TOXENV=py3.5-django1.10-sqlite3-locmem, TOXENV=py3.5-django1.10-sqlite3-redis, TOXENV=py3.5-django1.10-mysql-redis, TOXENV=py3.5-django1.10-mysql-memcached, and TOXENV=py3.5-django1.10-postgresql-filebased ✔
57
58
59	def check_parameter_types(params):	45✔
60	for p in params:	45✔
61	cl = p.__class__	45✔
62	if cl not in CACHABLE_PARAM_TYPES:	45✔
63	if cl in ITERABLES:	30 all except TOXENV=py3.5-django1.10-mysql-pylibmc, TOXENV=py3.5-django1.10-mysql-locmem, TOXENV=py3.5-django1.10-mysql-filebased, TOXENV=py3.5-django1.11-mysql-pylibmc, TOXENV=py3.5-django1.11-mysql-redis, TOXENV=py3.5-django1.11-mysql-memcached, TOXENV=py3.6-django1.11-mysql-pylibmc, TOXENV=py3.6-django1.11-mysql-locmem, TOXENV=py3.6-django1.11-mysql-filebased, TOXENV=py3.6-django1.11-mysql-memcached, TOXENV=py3.5-django1.10-mysql-redis, TOXENV=py3.5-django1.10-mysql-memcached, TOXENV=py3.5-django1.11-mysql-locmem, TOXENV=py3.5-django1.11-mysql-filebased, and TOXENV=py3.6-django1.11-mysql-redis ✔
64	check_parameter_types(p)	15 only TOXENV=py3.5-django1.10-postgresql-redis, TOXENV=py3.5-django1.10-postgresql-memcached, TOXENV=py3.5-django1.10-postgresql-locmem, TOXENV=py3.5-django1.10-postgresql-pylibmc, TOXENV=py3.5-django1.11-postgresql-redis, TOXENV=py3.5-django1.11-postgresql-memcached, TOXENV=py3.5-django1.11-postgresql-locmem, TOXENV=py3.6-django1.11-postgresql-redis, TOXENV=py3.6-django1.11-postgresql-memcached, TOXENV=py3.6-django1.11-postgresql-pylibmc, TOXENV=py3.6-django1.11-postgresql-locmem, TOXENV=py3.5-django1.11-postgresql-pylibmc, TOXENV=py3.5-django1.10-postgresql-filebased, TOXENV=py3.5-django1.11-postgresql-filebased, and TOXENV=py3.6-django1.11-postgresql-filebased ✔
65	elif cl is dict:	30 all except TOXENV=py3.5-django1.10-mysql-pylibmc, TOXENV=py3.5-django1.10-mysql-locmem, TOXENV=py3.5-django1.10-mysql-filebased, TOXENV=py3.5-django1.11-mysql-pylibmc, TOXENV=py3.5-django1.11-mysql-redis, TOXENV=py3.5-django1.11-mysql-memcached, TOXENV=py3.6-django1.11-mysql-pylibmc, TOXENV=py3.6-django1.11-mysql-locmem, TOXENV=py3.6-django1.11-mysql-filebased, TOXENV=py3.6-django1.11-mysql-memcached, TOXENV=py3.5-django1.10-mysql-redis, TOXENV=py3.5-django1.10-mysql-memcached, TOXENV=py3.5-django1.11-mysql-locmem, TOXENV=py3.5-django1.11-mysql-filebased, and TOXENV=py3.6-django1.11-mysql-redis ✔
66	check_parameter_types(p.items())	15 only TOXENV=py3.5-django1.10-postgresql-redis, TOXENV=py3.5-django1.10-postgresql-memcached, TOXENV=py3.5-django1.10-postgresql-locmem, TOXENV=py3.5-django1.10-postgresql-pylibmc, TOXENV=py3.5-django1.11-postgresql-redis, TOXENV=py3.5-django1.11-postgresql-memcached, TOXENV=py3.5-django1.11-postgresql-locmem, TOXENV=py3.6-django1.11-postgresql-redis, TOXENV=py3.6-django1.11-postgresql-memcached, TOXENV=py3.6-django1.11-postgresql-pylibmc, TOXENV=py3.6-django1.11-postgresql-locmem, TOXENV=py3.5-django1.11-postgresql-pylibmc, TOXENV=py3.5-django1.10-postgresql-filebased, TOXENV=py3.5-django1.11-postgresql-filebased, and TOXENV=py3.6-django1.11-postgresql-filebased ✔
67	else:
68	raise UncachableQuery	15 only TOXENV=py3.5-django1.10-sqlite3-filebased, TOXENV=py3.5-django1.10-sqlite3-memcached, TOXENV=py3.5-django1.10-sqlite3-pylibmc, TOXENV=py3.5-django1.11-sqlite3-memcached, TOXENV=py3.5-django1.11-sqlite3-redis, TOXENV=py3.5-django1.11-sqlite3-pylibmc, TOXENV=py3.5-django1.11-sqlite3-filebased, TOXENV=py3.5-django1.11-sqlite3-locmem, TOXENV=py3.6-django1.11-sqlite3-pylibmc, TOXENV=py3.6-django1.11-sqlite3-locmem, TOXENV=py3.6-django1.11-sqlite3-redis, TOXENV=py3.5-django1.10-sqlite3-locmem, TOXENV=py3.5-django1.10-sqlite3-redis, TOXENV=py3.6-django1.11-sqlite3-memcached, and TOXENV=py3.6-django1.11-sqlite3-filebased ✔
69
70
71	def get_query_cache_key(compiler):	45✔
72	"""
73	Generates a cache key from a SQLCompiler.
74
75	This cache key is specific to the SQL query and its context
76	(which database is used). The same query in the same context
77	(= the same database) must generate the same cache key.
78
79	:arg compiler: A SQLCompiler that will generate the SQL query
80	:type compiler: django.db.models.sql.compiler.SQLCompiler
81	:return: A cache key
82	:rtype: int
83	"""
84	sql, params = compiler.as_sql()	45✔
85	check_parameter_types(params)	45✔
86	cache_key = '%s:%s:%s' % (compiler.using, sql,	45✔
87	[text_type(p) for p in params])
88	return sha1(cache_key.encode('utf-8')).hexdigest()	45✔
89
90
91	def get_table_cache_key(db_alias, table):	45✔
92	"""
93	Generates a cache key from a SQL table.
94
95	:arg db_alias: Alias of the used database
96	:type db_alias: str or unicode
97	:arg table: Name of the SQL table
98	:type table: str or unicode
99	:return: A cache key
100	:rtype: int
101	"""
102	cache_key = '%s:%s' % (db_alias, table)	45✔
103	return sha1(cache_key.encode('utf-8')).hexdigest()	45✔
104
105
106	def _get_tables_from_sql(connection, lowercased_sql):	45✔
107	return {t for t in connection.introspection.django_table_names()	45✔
108	if t in lowercased_sql}
109
110
111	def _find_subqueries(children):	45✔
112	for child in children:	45✔
113	child_class = child.__class__	45✔
114	if child_class is WhereNode:	45✔
115	for grand_child in _find_subqueries(child.children):	45✔
116	yield grand_child	45✔
117	# TODO: Remove this condition when we drop Django 1.8 support.
118	elif child_class is SubqueryConstraint:	45✔
119	query_object = child.query_object	×
120	yield (query_object if query_object.__class__ is Query	×
121	else query_object.query)
122	elif child_class is ExtraWhere:	45✔
123	raise IsRawQuery	45✔
124	else:
125	rhs = getattr(child, 'rhs', None)	45✔
126	rhs_class = rhs.__class__	45✔
127	if rhs_class is Query:	45✔
128	yield rhs	45✔
129	elif rhs_class is QuerySet:	45✔
130	yield rhs.query	15 only TOXENV=py3.5-django1.10-sqlite3-filebased, TOXENV=py3.5-django1.10-postgresql-redis, TOXENV=py3.5-django1.10-postgresql-memcached, TOXENV=py3.5-django1.10-postgresql-locmem, TOXENV=py3.5-django1.10-sqlite3-memcached, TOXENV=py3.5-django1.10-postgresql-pylibmc, TOXENV=py3.5-django1.10-mysql-pylibmc, TOXENV=py3.5-django1.10-sqlite3-pylibmc, TOXENV=py3.5-django1.10-mysql-locmem, TOXENV=py3.5-django1.10-mysql-filebased, TOXENV=py3.5-django1.10-sqlite3-locmem, TOXENV=py3.5-django1.10-sqlite3-redis, TOXENV=py3.5-django1.10-mysql-redis, TOXENV=py3.5-django1.10-mysql-memcached, and TOXENV=py3.5-django1.10-postgresql-filebased ✔
131	elif rhs_class in UNCACHABLE_FUNCS:	45✔
132	raise UncachableQuery	45✔
133
134
135	def is_cachable(table):	45✔
136	whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES	45✔
137	if whitelist and table not in whitelist:	45✔
138	return False	×
139	return table not in cachalot_settings.CACHALOT_UNCACHABLE_TABLES	45✔
140
141
142	def are_all_cachable(tables):	45✔
143	whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES	45✔
144	if whitelist and not tables.issubset(whitelist):	45✔
145	return False	45✔
146	return tables.isdisjoint(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)	45✔
147
148
149	def filter_cachable(tables):	45✔
150	whitelist = cachalot_settings.CACHALOT_ONLY_CACHABLE_TABLES	45✔
151	tables = tables.difference(cachalot_settings.CACHALOT_UNCACHABLE_TABLES)	45✔
152	if whitelist:	45✔
153	return tables.intersection(whitelist)	×
154	return tables	45✔
155
156
157	def _get_tables(db_alias, query):	45✔
158	if query.select_for_update or (	45✔
159	not cachalot_settings.CACHALOT_CACHE_RANDOM
160	and '?' in query.order_by):
161	raise UncachableQuery	45✔
162
163	try:	45✔
164	if query.extra_select or getattr(query, 'subquery', False):	45✔
165	raise IsRawQuery	45✔
166	tables = set(query.table_map)	45✔
167	tables.add(query.get_meta().db_table)	45✔
168	for subquery in _find_subqueries(query.where.children):	45✔
169	tables.update(_get_tables(db_alias, subquery))	45✔
170	except IsRawQuery:	45✔
171	sql = query.get_compiler(db_alias).as_sql()[0].lower()	45✔
172	tables = _get_tables_from_sql(connections[db_alias], sql)	45✔
173
174	if not are_all_cachable(tables):	45✔
175	raise UncachableQuery	45✔
176	return tables	45✔
177
178
179	def _get_table_cache_keys(compiler):	45✔
180	db_alias = compiler.using	45✔
181	get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN	45✔
182	return [get_table_cache_key(db_alias, t)	45✔
183	for t in _get_tables(db_alias, compiler.query)]
184
185
186	def _invalidate_tables(cache, db_alias, tables):	45✔
187	tables = filter_cachable(set(tables))	45✔
188	if not tables:	45✔
189	return	45✔
190	now = time()	45✔
191	get_table_cache_key = cachalot_settings.CACHALOT_TABLE_KEYGEN	45✔
192	cache.set_many(	45✔
193	{get_table_cache_key(db_alias, t): now for t in tables},
194	cachalot_settings.CACHALOT_TIMEOUT)
195
196	if isinstance(cache, AtomicCache):	45✔
197	cache.to_be_invalidated.update(tables)	45✔

BertrandBordage / django-cachalot / 471

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous