18634744198

Committed 19 Oct 2025 07:00PM UTC coverage: 31.618% (-29.9%) from 61.514%

Build # 18634744198

Build Type

push

github

Committed by

mpasternak

Commit Message

Merge branch 'release/v202510.1270'

Coverage Stats

657 of 9430 branches covered (6.97%)

Branch coverage included in aggregate %.

229 of 523 new or added lines in 42 files covered. (43.79%)

11303 existing lines in 316 files now uncovered.

14765 of 39346 relevant lines covered (37.53%)

0.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

2.84

src/pbn_api/tasks.py

from django.core.management import call_command

from django_bpp.celery_tasks import app


@app.task
def download_institution_publications(user_id):
    """
    Download institution publications using PBN API management commands.
    Uses database-based locking to ensure only one instance runs at a time.

    Args:
        user_id: ID of the user initiating the download (must have valid PBN token)
    """
    from django.db import transaction
    from django.utils import timezone

    from bpp.models.profile import BppUser
    from pbn_downloader_app.models import PbnDownloadTask

    # Check if there's already a running task
    running_task = PbnDownloadTask.objects.filter(status="running").first()
    if running_task:
        raise ValueError(
            "Another download task is already running. Please wait for it to complete."
        )

    task_record = None
    try:
        user = BppUser.objects.get(pk=user_id)
        pbn_user = user.get_pbn_user()

        if not pbn_user.pbn_token:
            raise ValueError(
                f"User {user.username} is not authorized in PBN (no pbn_token)"
            )

        if not pbn_user.pbn_token_possibly_valid():
            raise ValueError(
                f"User {user.username} has an invalid or expired PBN token"
            )

        # Create task record with atomic transaction to prevent race conditions
        with transaction.atomic():
            # Double-check there's no running task within the transaction
            if PbnDownloadTask.objects.filter(status="running").exists():
                raise ValueError(
                    "Another download task is already running. Please wait for it to complete."
                )

            task_record = PbnDownloadTask.objects.create(
                user=user,
                status="running",
                started_at=timezone.now(),
                current_step="Inicjalizacja pobierania...",
                progress_percentage=0,
            )

        # Monkey-patch tqdm to capture progress and save to database
        from threading import Lock

        import tqdm

        original_tqdm_init = tqdm.tqdm.__init__
        original_tqdm_update = tqdm.tqdm.update
        original_tqdm_close = tqdm.tqdm.close
        progress_lock = Lock()

        def patched_tqdm_init(
            self, iterable=None, desc=None, total=None, *args, **kwargs
        ):
            # Store task record reference for progress updates
            self._task_record = task_record
            self._desc = desc or ""
            # Determine phase offset based on current step
            if task_record.current_step and "Faza 1" in task_record.current_step:
                self._phase_offset = 10  # First command: 10-45%
            elif task_record.current_step and "Faza 2" in task_record.current_step:
                self._phase_offset = 50  # Second command: 50-85%
            else:
                self._phase_offset = 10  # Default

            # Call original init
            original_tqdm_init(self, iterable, desc, total, *args, **kwargs)

        def patched_tqdm_update(self, n=1):
            result = original_tqdm_update(self, n)

            # Update database progress
            if hasattr(self, "_task_record") and self._task_record:
                with progress_lock:
                    try:
                        # Calculate progress percentage (10-45% for first command, 50-85% for second)
                        if self.total and self.total > 0:
                            command_progress = (
                                self.n / self.total
                            ) * 35  # Each command gets 35% of total progress
                            total_progress = self._phase_offset + command_progress

                            self._task_record.progress_percentage = min(
                                90, max(10, int(total_progress))
                            )

                            # Update current step with more detail
                            if self._desc:
                                if "publikacj" in self._desc.lower():
                                    self._task_record.publications_processed = self.n
                                    if self.total:
                                        self._task_record.total_publications = (
                                            self.total
                                        )
                                elif (
                                    "oświadczen" in self._desc.lower()
                                    or "statement" in self._desc.lower()
                                ):
                                    self._task_record.statements_processed = self.n
                                    if self.total:
                                        self._task_record.total_statements = self.total

                                # Update current step with progress details
                                progress_text = f"{self._desc} ({self.n}"
                                if self.total:
                                    progress_text += f"/{self.total}"
                                progress_text += ")"
                                self._task_record.current_step = progress_text

                            self._task_record.save()
                    except Exception:
                        pass  # Don't let database errors break the download

            return result

        def patched_tqdm_close(self):
            result = original_tqdm_close(self)

            # Final update when tqdm closes
            if hasattr(self, "_task_record") and self._task_record:
                with progress_lock:
                    try:
                        if self.total and self.n >= self.total:
                            # Command completed, update to end of phase
                            total_progress = self._phase_offset + 35
                            self._task_record.progress_percentage = min(
                                90, int(total_progress)
                            )
                            self._task_record.save()
                    except Exception:
                        pass

            return result

        # Apply monkey patches
        tqdm.tqdm.__init__ = patched_tqdm_init
        tqdm.tqdm.update = patched_tqdm_update
        tqdm.tqdm.close = patched_tqdm_close

        try:
            # Run the management commands with progress reporting
            task_record.current_step = "Pobieranie publikacji instytucji (Faza 1/2)"
            task_record.progress_percentage = 10
            task_record.save()

            call_command(
                "pbn_pobierz_publikacje_z_instytucji_v2", user_token=pbn_user.pbn_token
            )

            task_record.current_step = "Pobieranie oświadczeń i publikacji (Faza 2/2)"
            task_record.progress_percentage = 50
            task_record.save()

            call_command(
                "pbn_pobierz_oswiadczenia_i_publikacje_v1",
                user_token=pbn_user.pbn_token,
            )

            task_record.current_step = "Finalizowanie pobierania..."
            task_record.progress_percentage = 90
            task_record.save()

        finally:
            # Restore original tqdm methods
            tqdm.tqdm.__init__ = original_tqdm_init
            tqdm.tqdm.update = original_tqdm_update
            tqdm.tqdm.close = original_tqdm_close

        # Mark as completed
        task_record.status = "completed"
        task_record.current_step = "Pobieranie zakończone pomyślnie"
        task_record.progress_percentage = 100
        task_record.completed_at = timezone.now()
        task_record.save()

    except Exception as e:
        # Record the error
        if task_record:
            task_record.status = "failed"
            task_record.error_message = str(e)
            task_record.completed_at = timezone.now()
            task_record.save()
        raise

1	from django.core.management import call_command	1✔
2
3	from django_bpp.celery_tasks import app	1✔
4
5
6	@app.task	1✔
7	def download_institution_publications(user_id):	1✔
8	"""
9	Download institution publications using PBN API management commands.
10	Uses database-based locking to ensure only one instance runs at a time.
11
12	Args:
13	user_id: ID of the user initiating the download (must have valid PBN token)
14	"""
UNCOV 15	from django.db import transaction	×
UNCOV 16	from django.utils import timezone	×
17
UNCOV 18	from bpp.models.profile import BppUser	×
UNCOV 19	from pbn_downloader_app.models import PbnDownloadTask	×
20
21	# Check if there's already a running task
UNCOV 22	running_task = PbnDownloadTask.objects.filter(status="running").first()	×
UNCOV 23	if running_task:	×
UNCOV 24	raise ValueError(	×
25	"Another download task is already running. Please wait for it to complete."
26	)
27
UNCOV 28	task_record = None	×
UNCOV 29	try:	×
UNCOV 30	user = BppUser.objects.get(pk=user_id)	×
UNCOV 31	pbn_user = user.get_pbn_user()	×
32
UNCOV 33	if not pbn_user.pbn_token:	×
UNCOV 34	raise ValueError(	×
35	f"User {user.username} is not authorized in PBN (no pbn_token)"
36	)
37
UNCOV 38	if not pbn_user.pbn_token_possibly_valid():	×
UNCOV 39	raise ValueError(	×
40	f"User {user.username} has an invalid or expired PBN token"
41	)
42
43	# Create task record with atomic transaction to prevent race conditions
UNCOV 44	with transaction.atomic():	×
45	# Double-check there's no running task within the transaction
UNCOV 46	if PbnDownloadTask.objects.filter(status="running").exists():	×
47	raise ValueError(	×
48	"Another download task is already running. Please wait for it to complete."
49	)
50
UNCOV 51	task_record = PbnDownloadTask.objects.create(	×
52	user=user,
53	status="running",
54	started_at=timezone.now(),
55	current_step="Inicjalizacja pobierania...",
56	progress_percentage=0,
57	)
58
59	# Monkey-patch tqdm to capture progress and save to database
UNCOV 60	from threading import Lock	×
61
UNCOV 62	import tqdm	×
63
UNCOV 64	original_tqdm_init = tqdm.tqdm.__init__	×
UNCOV 65	original_tqdm_update = tqdm.tqdm.update	×
UNCOV 66	original_tqdm_close = tqdm.tqdm.close	×
UNCOV 67	progress_lock = Lock()	×
68
UNCOV 69	def patched_tqdm_init(	×
70	self, iterable=None, desc=None, total=None, args, *kwargs
71	):
72	# Store task record reference for progress updates
73	self._task_record = task_record	×
74	self._desc = desc or ""	×
75	# Determine phase offset based on current step
76	if task_record.current_step and "Faza 1" in task_record.current_step:	×
77	self._phase_offset = 10 # First command: 10-45%	×
78	elif task_record.current_step and "Faza 2" in task_record.current_step:	×
79	self._phase_offset = 50 # Second command: 50-85%	×
80	else:
81	self._phase_offset = 10 # Default	×
82
83	# Call original init
84	original_tqdm_init(self, iterable, desc, total, args, *kwargs)	×
85
UNCOV 86	def patched_tqdm_update(self, n=1):	×
87	result = original_tqdm_update(self, n)	×
88
89	# Update database progress
90	if hasattr(self, "_task_record") and self._task_record:	×
91	with progress_lock:	×
92	try:	×
93	# Calculate progress percentage (10-45% for first command, 50-85% for second)
94	if self.total and self.total > 0:	×
95	command_progress = (	×
96	self.n / self.total
97	) * 35 # Each command gets 35% of total progress
98	total_progress = self._phase_offset + command_progress	×
99
100	self._task_record.progress_percentage = min(	×
101	90, max(10, int(total_progress))
102	)
103
104	# Update current step with more detail
105	if self._desc:	×
106	if "publikacj" in self._desc.lower():	×
107	self._task_record.publications_processed = self.n	×
108	if self.total:	×
109	self._task_record.total_publications = (	×
110	self.total
111	)
112	elif (	×
113	"oświadczen" in self._desc.lower()
114	or "statement" in self._desc.lower()
115	):
116	self._task_record.statements_processed = self.n	×
117	if self.total:	×
118	self._task_record.total_statements = self.total	×
119
120	# Update current step with progress details
121	progress_text = f"{self._desc} ({self.n}"	×
122	if self.total:	×
123	progress_text += f"/{self.total}"	×
124	progress_text += ")"	×
125	self._task_record.current_step = progress_text	×
126
127	self._task_record.save()	×
128	except Exception:	×
129	pass # Don't let database errors break the download	×
130
131	return result	×
132
UNCOV 133	def patched_tqdm_close(self):	×
134	result = original_tqdm_close(self)	×
135
136	# Final update when tqdm closes
137	if hasattr(self, "_task_record") and self._task_record:	×
138	with progress_lock:	×
139	try:	×
140	if self.total and self.n >= self.total:	×
141	# Command completed, update to end of phase
142	total_progress = self._phase_offset + 35	×
143	self._task_record.progress_percentage = min(	×
144	90, int(total_progress)
145	)
146	self._task_record.save()	×
147	except Exception:	×
148	pass	×
149
150	return result	×
151
152	# Apply monkey patches
UNCOV 153	tqdm.tqdm.__init__ = patched_tqdm_init	×
UNCOV 154	tqdm.tqdm.update = patched_tqdm_update	×
UNCOV 155	tqdm.tqdm.close = patched_tqdm_close	×
156
UNCOV 157	try:	×
158	# Run the management commands with progress reporting
UNCOV 159	task_record.current_step = "Pobieranie publikacji instytucji (Faza 1/2)"	×
UNCOV 160	task_record.progress_percentage = 10	×
UNCOV 161	task_record.save()	×
162
UNCOV 163	call_command(	×
164	"pbn_pobierz_publikacje_z_instytucji_v2", user_token=pbn_user.pbn_token
165	)
166
UNCOV 167	task_record.current_step = "Pobieranie oświadczeń i publikacji (Faza 2/2)"	×
UNCOV 168	task_record.progress_percentage = 50	×
UNCOV 169	task_record.save()	×
170
UNCOV 171	call_command(	×
172	"pbn_pobierz_oswiadczenia_i_publikacje_v1",
173	user_token=pbn_user.pbn_token,
174	)
175
UNCOV 176	task_record.current_step = "Finalizowanie pobierania..."	×
UNCOV 177	task_record.progress_percentage = 90	×
UNCOV 178	task_record.save()	×
179
180	finally:
181	# Restore original tqdm methods
UNCOV 182	tqdm.tqdm.__init__ = original_tqdm_init	×
UNCOV 183	tqdm.tqdm.update = original_tqdm_update	×
UNCOV 184	tqdm.tqdm.close = original_tqdm_close	×
185
186	# Mark as completed
UNCOV 187	task_record.status = "completed"	×
UNCOV 188	task_record.current_step = "Pobieranie zakończone pomyślnie"	×
UNCOV 189	task_record.progress_percentage = 100	×
UNCOV 190	task_record.completed_at = timezone.now()	×
UNCOV 191	task_record.save()	×
192
UNCOV 193	except Exception as e:	×
194	# Record the error
UNCOV 195	if task_record:	×
UNCOV 196	task_record.status = "failed"	×
UNCOV 197	task_record.error_message = str(e)	×
UNCOV 198	task_record.completed_at = timezone.now()	×
UNCOV 199	task_record.save()	×
UNCOV 200	raise	×

iplweb / bpp / 18634744198

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous