1764024207

Build Type

github

Committed by GitHub

Commit Message

Merge 67f8f326c into 6ea264820

Pull Request Pull Request #99: better updatedata command

Run Details

602 of 667 branches covered (90.25%)

Branch coverage included in aggregate %.

82 of 130 new or added lines in 4 files covered. (63.08%)

34 existing lines in 5 files now uncovered.

1415 of 1507 relevant lines covered (93.9%)

13.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.17

/computedfields/management/commands/updatedata.py

1	from django.core.management.base import BaseCommand, CommandError	15✔
2	from django.apps import apps	15✔
3	from django.db import transaction	15✔
4	from computedfields.models import active_resolver	15✔
5	from computedfields.helper import modelname	15✔
6	from time import time	15✔
7	from datetime import timedelta	15✔
8
9
10	class _Tqdm:	15✔
11	def __init__(self, args, *kwargs):	15✔
NEW 12	pass	×
13	def __enter__(self):	15✔
NEW 14	return self	×
15	def __exit__(self, type, value, traceback):	15✔
NEW 16	pass	×
17	def update(self, *args):	15✔
NEW 18	pass	×
19
20
21	try:	15✔
22	from tqdm import tqdm	15✔
NEW 23	except ImportError:	×
NEW UNCOV 24	tqdm = _Tqdm	×
25
26
27	# TODO: skip progress for 0 records
28
29	class Command(BaseCommand):	15✔
30	help = 'Update computed fields.'	15✔
31
32	def add_arguments(self, parser):	15✔
33	parser.add_argument(	15✔
34	'args', metavar='app_label[.ModelName]', nargs='*',
35	help='Update computed fields in specified app_label or app_label.ModelName.',
36	)
37	parser.add_argument(	15✔
38	'-p', '--progress',
39	action='store_true',
40	help='show update progress',
41	)
42	parser.add_argument(	15✔
43	'-m', '--mode',
44	default='default',
45	type=str,
46	choices=('loop', 'bulk', 'fast'),
47	help='update mode, default: bulk/fast from settings.py'
48	)
49	parser.add_argument(	15✔
50	'-q', '--querysize',
51	default=1000,
52	type=int,
53	help='queryset size, default: 1000'
54	)
55
56	def handle(self, app_labels, *options):	15✔
57	# FIXME: remove once tests are fixed
58	#for model in active_resolver.computed_models:
59	# for obj in model.objects.all():
60	# obj.save()
61	#return
62	start_time = time()	15✔
63	progress = options['progress']	15✔
64	mode = options['mode']	15✔
65	size = options['querysize']	15✔
66	if progress and tqdm == _Tqdm:	15!
NEW 67	raise CommandError("Package 'tqdm' needed for the progressbar.")	×
68	models = retrieve_computed_models(app_labels)	15✔
69	getattr(self, 'action_' + mode, self.action_default)(models, size, progress)	15✔
70	end_time = time()	15✔
71	duration = int(end_time - start_time)	15✔
72	print(f'\nTotal time: {timedelta(seconds=duration)}')	15✔
73
74	@transaction.atomic	15✔
75	def action_default(self, models, size, show_progress, mode=''):	15✔
76	"""
77	Runs either in fast or bulk mode, whatever was set in settings.
78	"""
79	# TODO: move this outside of transaction?
80	if not mode:	15!
81	from django.conf import settings	15✔
82	if getattr(settings, 'COMPUTEDFIELDS_FASTUPDATE', False):	15!
83	from computedfields.fast_update import check_support	15✔
84	if check_support():	15✔
85	mode = 'fast'	4✔
86	print(f'Update mode: settings.py --> {mode or "bulk"}')	15✔
87
88	print(f'QuerySet size: {size}')	15✔
89	print('Models:')	15✔
90	for model in models:	15✔
91	qs = model.objects.all()	15✔
92	amount = qs.count()	15✔
93	print(f'- {self.style.MIGRATE_LABEL(modelname(model))}')	15✔
94	print(f' Fields: {", ".join(active_resolver.computed_models[model].keys())}')	15✔
95	print(f' Records: {amount}')	15✔
96	fields = set(active_resolver.computed_models[model].keys())	15✔
97	# FIXME: use slice interface from bulk_updater, once we have it
98	pos = 0	15✔
99	with tqdm(total=amount, desc=' Progress', unit=' rec', disable=not show_progress) as bar:	15✔
100	while pos < amount:	15✔
101	active_resolver.update_dependent(	15✔
102	qs.order_by('pk')[pos:pos+size+1], update_fields=fields)
103	progress = min(pos+size, amount) - pos	15✔
104	bar.update(progress)	15✔
105	pos += size	15✔
106
107	def action_bulk(self, models, size, show_progress):	15✔
NEW 108	active_resolver.use_fastupdate = False	×
NEW 109	print('Update mode: bulk')	×
NEW 110	self.action_default(models, size, show_progress, 'bulk')	×
111
112	def action_fast(self, models, size, show_progress):	15✔
NEW 113	from computedfields.fast_update import check_support	×
NEW 114	if not check_support():	×
NEW 115	raise CommandError('Current db backend does not support fast_update.')	×
NEW 116	active_resolver.use_fastupdate = True	×
NEW 117	from django.conf import settings	×
NEW 118	active_resolver._batchsize = getattr(	×
119	settings, 'COMPUTEDFIELDS_BATCHSIZE_FAST', active_resolver._batchsize * 10)
NEW 120	print('Update mode: fast')	×
NEW 121	self.action_default(models, size, show_progress, 'fast')	×
122
123	@transaction.atomic	15✔
124	def action_loop(self, models, size, show_progress):	12✔
125	# TODO: load select/prefetch from cfs?
NEW 126	print('Update mode: loop')	×
NEW 127	print(f'QuerySet size: {size}')	×
NEW 128	print('Models:')	×
NEW 129	for model in models:	×
NEW 130	qs = model.objects.all()	×
NEW 131	amount = qs.count()	×
NEW 132	print(f'- {self.style.MIGRATE_LABEL(modelname(model))}')	×
NEW 133	print(f' Fields: {", ".join(active_resolver.computed_models[model].keys())}')	×
NEW 134	print(f' Records: {amount}')	×
NEW 135	fields = list(active_resolver.computed_models[model].keys())	×
NEW 136	with tqdm(total=amount, desc=' Progress', unit=' rec', disable=not show_progress) as bar:	×
NEW 137	for obj in qs.iterator(size):	×
NEW 138	obj.save(update_fields=fields)	×
NEW 139	bar.update(1)	×
140
141
142	def retrieve_computed_models(app_labels):	15✔
143	computed_models = set(active_resolver.computed_models.keys())	15✔
144	if not app_labels:	15!
145	return computed_models	15✔
NEW 146	update_models = set()	×
NEW 147	for label in app_labels:	×
NEW 148	app_model = label.split('.')	×
NEW 149	try:	×
NEW 150	app_config = apps.get_app_config(app_model[0])	×
NEW 151	except LookupError as e:	×
NEW 152	raise CommandError(str(e))	×
NEW 153	if len(app_model) == 1:	×
NEW 154	update_models \|= set(app_config.get_models()) & computed_models	×
NEW 155	elif len(app_model) == 2:	×
NEW 156	try:	×
NEW 157	update_models \|= set([app_config.get_model(app_model[1])]) & computed_models	×
NEW 158	except LookupError:	×
NEW 159	raise CommandError(f'Unknown model: {label}')	×
160	else:
NEW 161	raise CommandError(f'Unsupported app_label.ModelName specification: {label}')	×
NEW 162	return update_models	×

netzkolchose / django-computedfields / 1764024207

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous