1764227661

Build Type

github

Committed by GitHub

Commit Message

Merge ef848e047 into 6ea264820

Pull Request Pull Request #99: better updatedata command

Run Details

608 of 681 branches covered (89.28%)

Branch coverage included in aggregate %.

112 of 170 new or added lines in 5 files covered. (65.88%)

36 existing lines in 5 files now uncovered.

1433 of 1535 relevant lines covered (93.36%)

12.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.92

/computedfields/management/commands/updatedata.py

1	from django.core.management.base import BaseCommand, CommandError	15✔
2	from django.apps import apps	15✔
3	from django.db import transaction	15✔
4	from computedfields.models import active_resolver	15✔
5	from computedfields.helper import modelname	15✔
6	from computedfields.resolver import COMPUTEDFIELDS_FASTUPDATE, COMPUTEDFIELDS_BATCHSIZE_FAST	15✔
7	from time import time	15✔
8	from datetime import timedelta	15✔
9
10
11	class _Tqdm:	15✔
12	def __init__(self, args, *kwargs):	15✔
NEW 13	pass	×
14	def __enter__(self):	15✔
NEW 15	return self	×
16	def __exit__(self, type, value, traceback):	15✔
NEW 17	pass	×
18	def update(self, *args):	15✔
NEW 19	pass	×
20
21
22	try:	15✔
23	from tqdm import tqdm	15✔
NEW UNCOV 24	except ImportError:	×
NEW UNCOV 25	tqdm = _Tqdm	×
26
27
28	class Command(BaseCommand):	15✔
29	help = 'Update computed fields.'	15✔
30
31	def add_arguments(self, parser):	15✔
32	parser.add_argument(	15✔
33	'args', metavar='app_label[.ModelName]', nargs='*',
34	help='Update computed fields in specified app_label or app_label.ModelName.',
35	)
36	parser.add_argument(	15✔
37	'-p', '--progress',
38	action='store_true',
39	help='show update progress',
40	)
41	parser.add_argument(	15✔
42	'-m', '--mode',
43	default='default',
44	type=str,
45	choices=('loop', 'bulk', 'fast'),
46	help='update mode, default: bulk/fast from settings.py'
47	)
48	parser.add_argument(	15✔
49	'-q', '--querysize',
50	default=1000,
51	type=int,
52	help='queryset size, default: 1000'
53	)
54
55	def handle(self, app_labels, *options):	15✔
56	# FIXME: remove once tests are fixed
57	#for model in active_resolver.computed_models:
58	# for obj in model.objects.all():
59	# obj.save()
60	#return
61	start_time = time()	15✔
62	progress = options['progress']	15✔
63	mode = options['mode']	15✔
64	size = options['querysize']	15✔
65	if progress and tqdm == _Tqdm:	15!
NEW 66	raise CommandError("Package 'tqdm' needed for the progressbar.")	×
67	models = retrieve_computed_models(app_labels)	15✔
68	getattr(self, 'action_' + mode, self.action_default)(models, size, progress)	15✔
69	end_time = time()	15✔
70	duration = int(end_time - start_time)	15✔
71	print(f'\nTotal time: {timedelta(seconds=duration)}')	15✔
72
73	@transaction.atomic	15✔
74	def action_default(self, models, size, show_progress, mode=''):	15✔
75	"""
76	Runs either in fast or bulk mode, whatever was set in settings.
77	"""
78	# TODO: move this outside of transaction?
79	if not mode:	15!
80	from django.conf import settings	15✔
81	if getattr(settings, 'COMPUTEDFIELDS_FASTUPDATE', COMPUTEDFIELDS_FASTUPDATE):	15!
82	from computedfields.fast_update import check_support	15✔
83	if check_support():	15✔
84	mode = 'fast'	4✔
85	print(f'Update mode: settings.py --> {mode or "bulk"}')	15✔
86
87	print(f'QuerySet size: {size}')	15✔
88	print('Models:')	15✔
89	for model in models:	15✔
90	qs = model.objects.all()	15✔
91	amount = qs.count()	15✔
92	fields = set(active_resolver.computed_models[model].keys())	15✔
93	print(f'- {self.style.MIGRATE_LABEL(modelname(model))}')	15✔
94	print(f' Fields: {", ".join(fields)}')	15✔
95	print(f' Records: {amount}')	15✔
96	if not amount:	15✔
97	continue	15✔
98	# FIXME: use slice interface from bulk_updater, once we have it
99	pos = 0	15✔
100	with tqdm(total=amount, desc=' Progress', unit=' rec', disable=not show_progress) as bar:	15✔
101	while pos < amount:	15✔
102	active_resolver.update_dependent(	15✔
103	qs.order_by('pk')[pos:pos+size+1], update_fields=fields)
104	progress = min(pos+size, amount) - pos	15✔
105	bar.update(progress)	15✔
106	pos += size	15✔
107
108	def action_bulk(self, models, size, show_progress):	15✔
NEW 109	active_resolver.use_fastupdate = False	×
NEW 110	print('Update mode: bulk')	×
NEW 111	self.action_default(models, size, show_progress, 'bulk')	×
112
113	def action_fast(self, models, size, show_progress):	15✔
NEW 114	from computedfields.fast_update import check_support	×
NEW 115	if not check_support():	×
NEW 116	raise CommandError('Current db backend does not support fast_update.')	×
NEW 117	active_resolver.use_fastupdate = True	×
NEW 118	from django.conf import settings	×
NEW 119	active_resolver._batchsize = getattr(	×
120	settings, 'COMPUTEDFIELDS_BATCHSIZE_FAST', COMPUTEDFIELDS_BATCHSIZE_FAST)
NEW 121	print('Update mode: fast')	×
NEW 122	self.action_default(models, size, show_progress, 'fast')	×
123
124	@transaction.atomic	15✔
125	def action_loop(self, models, size, show_progress):	12✔
126	# TODO: load select/prefetch from cfs?
NEW 127	print('Update mode: loop')	×
NEW 128	print(f'QuerySet size: {size}')	×
NEW 129	print('Models:')	×
NEW 130	for model in models:	×
NEW 131	qs = model.objects.all()	×
NEW 132	amount = qs.count()	×
NEW 133	fields = list(active_resolver.computed_models[model].keys())	×
NEW 134	print(f'- {self.style.MIGRATE_LABEL(modelname(model))}')	×
NEW 135	print(f' Fields: {", ".join(fields)}')	×
NEW 136	print(f' Records: {amount}')	×
NEW 137	if not amount:	×
NEW 138	continue	×
139	# also apply select/prefetch rules
NEW 140	select = active_resolver.get_select_related(model, fields)	×
NEW 141	prefetch = active_resolver.get_prefetch_related(model, fields)	×
NEW 142	if select:	×
NEW 143	qs = qs.select_related(*select)	×
NEW 144	if prefetch:	×
NEW 145	qs = qs.prefetch_related(*prefetch)	×
NEW 146	with tqdm(total=amount, desc=' Progress', unit=' rec', disable=not show_progress) as bar:	×
NEW 147	for obj in qs.iterator(size):	×
NEW 148	obj.save(update_fields=fields)	×
NEW 149	bar.update(1)	×
150
151
152	def retrieve_computed_models(app_labels):	15✔
153	computed_models = set(active_resolver.computed_models.keys())	15✔
154	if not app_labels:	15!
155	return computed_models	15✔
NEW 156	update_models = set()	×
NEW 157	for label in app_labels:	×
NEW 158	app_model = label.split('.')	×
NEW 159	try:	×
NEW 160	app_config = apps.get_app_config(app_model[0])	×
NEW 161	except LookupError as e:	×
NEW 162	raise CommandError(str(e))	×
NEW 163	if len(app_model) == 1:	×
NEW 164	update_models \|= set(app_config.get_models()) & computed_models	×
NEW 165	elif len(app_model) == 2:	×
NEW 166	try:	×
NEW 167	update_models \|= set([app_config.get_model(app_model[1])]) & computed_models	×
NEW 168	except LookupError:	×
NEW 169	raise CommandError(f'Unknown model: {label}')	×
170	else:
NEW 171	raise CommandError(f'Unsupported app_label.ModelName specification: {label}')	×
NEW 172	return update_models	×

netzkolchose / django-computedfields / 1764227661

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous