• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 18706668671

22 Oct 2025 05:48AM UTC coverage: 69.481% (-0.2%) from 69.702%
18706668671

push

github

web-flow
[FEATURE] Automatic reload process status without page reload (#1205)

21 of 39 new or added lines in 2 files covered. (53.85%)

1942 of 2795 relevant lines covered (69.48%)

3.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.96
/Classes/Domain/Repository/ProcessRepository.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Domain\Repository;
6

7
/***************************************************************
8
 *  Copyright notice
9
 *
10
 *  (c) 2019 AOE GmbH <dev@aoe.com>
11
 *
12
 *  All rights reserved
13
 *
14
 *  This script is part of the TYPO3 project. The TYPO3 project is
15
 *  free software; you can redistribute it and/or modify
16
 *  it under the terms of the GNU General Public License as published by
17
 *  the Free Software Foundation; either version 3 of the License, or
18
 *  (at your option) any later version.
19
 *
20
 *  The GNU General Public License can be found at
21
 *  http://www.gnu.org/copyleft/gpl.html.
22
 *
23
 *  This script is distributed in the hope that it will be useful,
24
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
25
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26
 *  GNU General Public License for more details.
27
 *
28
 *  This copyright notice MUST APPEAR in all copies of the script!
29
 ***************************************************************/
30

31
use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
32
use AOE\Crawler\Domain\Model\Process;
33
use AOE\Crawler\Domain\Model\ProcessCollection;
34
use Doctrine\DBAL\ArrayParameterType;
35
use Doctrine\DBAL\Exception;
36
use Symfony\Contracts\Service\Attribute\Required;
37
use TYPO3\CMS\Core\Database\ConnectionPool;
38
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
39
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
40
use TYPO3\CMS\Core\Database\Query\Restriction\HiddenRestriction;
41
use TYPO3\CMS\Core\Utility\GeneralUtility;
42

43
/**
44
 * @internal since v9.2.5
45
 */
46
class ProcessRepository
47
{
48
    final public const TABLE_NAME = 'tx_crawler_process';
49

50
    protected QueryBuilder $queryBuilder;
51
    protected array $extensionSettings = [];
52

53
    #[Required]
54
    public function setExtensionSettings(): void
55
    {
56
        /** @var ExtensionConfigurationProvider $configurationProvider */
57
        $configurationProvider = GeneralUtility::makeInstance(ExtensionConfigurationProvider::class);
4 ✔
58
        $this->extensionSettings = $configurationProvider->getExtensionConfiguration();
4 ✔
59
    }
60

61
    #[Required]
62
    public function setQueryBuilder(): void
63
    {
64
        $this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(
4 ✔
65
            self::TABLE_NAME
4 ✔
66
        );
4 ✔
67
    }
68

69
    /**
70
     * This method is used to find all cli processes within a limit.
71
     * @throws Exception
72
     */
73
    public function findAll(): ProcessCollection
74
    {
75
        /** @var ProcessCollection $collection */
76
        $collection = GeneralUtility::makeInstance(ProcessCollection::class);
4 ✔
77
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
4 ✔
78
        $queryBuilder->getRestrictions()
4 ✔
79
            ->removeByType(HiddenRestriction::class)
4 ✔
80
            ->removeByType(DeletedRestriction::class);
4 ✔
81

82
        $statement = $queryBuilder
4 ✔
83
            ->select('*')
4 ✔
84
            ->from(self::TABLE_NAME)
4 ✔
85
            ->orderBy('ttl', 'DESC')
4 ✔
86
            ->executeQuery();
4 ✔
87

88
        while ($row = $statement->fetchAssociative()) {
4 ✔
89
            $process = GeneralUtility::makeInstance(Process::class);
4 ✔
90
            $process->setProcessId((string) $row['process_id']);
4 ✔
91
            $process->setActive((bool) $row['active']);
4 ✔
92
            $process->setTtl((int) $row['ttl']);
4 ✔
93
            $process->setAssignedItemsCount((int) $row['assigned_items_count']);
4 ✔
94
            $process->setDeleted((bool) $row['deleted']);
4 ✔
95
            $process->setSystemProcessId((string) $row['system_process_id']);
4 ✔
96
            $collection->append($process);
4 ✔
97
        }
98

99
        return $collection;
4 ✔
100
    }
101

102
    public function findAllActive(): ProcessCollection
103
    {
104
        /** @var ProcessCollection $collection */
105
        $collection = GeneralUtility::makeInstance(ProcessCollection::class);
4 ✔
106
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
4 ✔
107

108
        $statement = $queryBuilder
4 ✔
109
            ->select('*')
4 ✔
110
            ->from(self::TABLE_NAME)
4 ✔
111
            ->where($queryBuilder->expr()->eq('active', 1), $queryBuilder->expr()->eq('deleted', 0))
4 ✔
112
            ->orderBy('ttl', 'DESC')
4 ✔
113
            ->executeQuery();
4 ✔
114

115
        while ($row = $statement->fetchAssociative()) {
4 ✔
116
            $process = new Process();
2 ✔
117
            $process->setProcessId((string) $row['process_id']);
2 ✔
118
            $process->setActive((bool) $row['active']);
2 ✔
119
            $process->setTtl((int) $row['ttl']);
2 ✔
120
            $process->setAssignedItemsCount((int) $row['assigned_items_count']);
2 ✔
121
            $process->setDeleted((bool) $row['deleted']);
2 ✔
122
            $process->setSystemProcessId((string) $row['system_process_id']);
2 ✔
123
            $collection->append($process);
2 ✔
124
        }
125

126
        return $collection;
4 ✔
127
    }
128

129
    public function findByProcessId(string $processId): ?Process
130
    {
NEW
131
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
×
132

NEW
133
        $row = $queryBuilder
×
NEW
134
            ->select('*')
×
NEW
135
            ->from(self::TABLE_NAME)
×
NEW
136
            ->where($queryBuilder->expr()->eq('process_id', $queryBuilder->createNamedParameter($processId)))
×
NEW
137
            ->executeQuery()
×
NEW
138
            ->fetchAssociative();
×
139

NEW
140
        if ($row === false) {
×
NEW
141
            return null;
×
142
        }
143

NEW
144
        $process = new Process();
×
NEW
145
        $process->setProcessId((string) $row['process_id']);
×
NEW
146
        $process->setActive((bool) $row['active']);
×
NEW
147
        $process->setTtl((int) $row['ttl']);
×
NEW
148
        $process->setAssignedItemsCount((int) $row['assigned_items_count']);
×
NEW
149
        $process->setDeleted((bool) $row['deleted']);
×
NEW
150
        $process->setSystemProcessId((string) $row['system_process_id']);
×
151

NEW
152
        return $process;
×
153
    }
154

155
    /**
156
     * @param string $processId
157
     */
158
    public function removeByProcessId($processId): void
159
    {
160
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
1 ✔
161

162
        $queryBuilder
1 ✔
163
            ->delete(self::TABLE_NAME)
1 ✔
164
            ->where(
1 ✔
165
                $queryBuilder->expr()->eq('process_id', $queryBuilder->createNamedParameter($processId))
1 ✔
166
            )->executeStatement();
1 ✔
167
    }
168

169
    /**
170
     * @return array|null
171
     *
172
     * Function is moved from ProcessCleanUpHook
173
     * TODO: Check why we need both getActiveProcessesOlderThanOneHour and getActiveOrphanProcesses, the get getActiveOrphanProcesses does not really check for Orphan in this implementation.
174
     */
175
    public function getActiveProcessesOlderThanOneHour()
176
    {
177
        $processMaxRunTime = $this->extensionSettings['processMaxRunTime'] ?? 360;
1 ✔
178

179
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
1 ✔
180
        $queryBuilder->getRestrictions()
1 ✔
181
            ->removeByType(HiddenRestriction::class)
1 ✔
182
            ->removeByType(DeletedRestriction::class);
1 ✔
183

184
        $activeProcesses = [];
1 ✔
185
        $statement = $queryBuilder
1 ✔
186
            ->select('process_id', 'system_process_id')
1 ✔
187
            ->from(self::TABLE_NAME)
1 ✔
188
            ->where(
1 ✔
189
                $queryBuilder->expr()->lte('ttl', intval(time() - $processMaxRunTime - 3600)),
1 ✔
190
                $queryBuilder->expr()->eq('active', 1)
1 ✔
191
            )
1 ✔
192
            ->executeQuery();
1 ✔
193

194
        while ($row = $statement->fetchAssociative()) {
1 ✔
195
            $activeProcesses[] = $row;
1 ✔
196
        }
197

198
        return $activeProcesses;
1 ✔
199
    }
200

201
    /**
202
     * Function is moved from ProcessCleanUpHook
203
     *
204
     * @return array
205
     * @see getActiveProcessesOlderThanOneHour
206
     */
207
    public function getActiveOrphanProcesses()
208
    {
209
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
1 ✔
210
        $queryBuilder->getRestrictions()
1 ✔
211
            ->removeByType(HiddenRestriction::class)
1 ✔
212
            ->removeByType(DeletedRestriction::class);
1 ✔
213

214
        $processMaxRunTime = $this->extensionSettings['processMaxRunTime'] ?? 360;
1 ✔
215

216
        return $queryBuilder
1 ✔
217
            ->select('process_id', 'system_process_id')
1 ✔
218
            ->from(self::TABLE_NAME)
1 ✔
219
            ->where(
1 ✔
220
                $queryBuilder->expr()->lte('ttl', intval(time() - $processMaxRunTime)),
1 ✔
221
                $queryBuilder->expr()->eq('active', 1)
1 ✔
222
            )
1 ✔
223
            ->executeQuery()->fetchAllAssociative();
1 ✔
224
    }
225

226
    /**
227
     * Returns the number of processes that live longer than the given timestamp.
228
     */
229
    public function countNotTimeouted(int $ttl): int
230
    {
231
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
1 ✔
232

233
        return (int) $queryBuilder
1 ✔
234
            ->count('*')
1 ✔
235
            ->from(self::TABLE_NAME)
1 ✔
236
            ->where($queryBuilder->expr()->eq('deleted', 0), $queryBuilder->expr()->gt('ttl', intval($ttl)))
1 ✔
237
            ->executeQuery()
1 ✔
238
            ->fetchOne();
1 ✔
239
    }
240

241
    public function deleteProcessesWithoutItemsAssigned(): void
242
    {
243
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
3 ✔
244
        $queryBuilder
3 ✔
245
            ->delete(self::TABLE_NAME)
3 ✔
246
            ->where($queryBuilder->expr()->eq('assigned_items_count', 0))
3 ✔
247
            ->executeStatement();
3 ✔
248
    }
249

250
    public function deleteProcessesMarkedAsDeleted(): void
251
    {
252
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
3 ✔
253
        $queryBuilder
3 ✔
254
            ->delete(self::TABLE_NAME)
3 ✔
255
            ->where($queryBuilder->expr()->eq('deleted', $queryBuilder->createNamedParameter(1)))
3 ✔
256
            ->executeStatement();
3 ✔
257
    }
258

259
    public function isProcessActive(string $processId): bool
260
    {
261
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
2 ✔
262
        $isActive = $queryBuilder
2 ✔
263
            ->select('active')
2 ✔
264
            ->from(self::TABLE_NAME)
2 ✔
265
            ->where($queryBuilder->expr()->eq('process_id', $queryBuilder->createNamedParameter($processId)))
2 ✔
266
            ->orderBy('ttl')
2 ✔
267
            ->executeQuery()
2 ✔
268
            ->fetchFirstColumn();
2 ✔
269

270
        return (bool) $isActive;
2 ✔
271
    }
272

273
    public function updateProcessAssignItemsCount(int $numberOfAffectedRows, string $processId): void
274
    {
275
        GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE_NAME)
3 ✔
276
            ->update(
3 ✔
277
                self::TABLE_NAME,
3 ✔
278
                [
3 ✔
279
                    'assigned_items_count' => $numberOfAffectedRows,
3 ✔
280
                ],
3 ✔
281
                [
3 ✔
282
                    'process_id' => $processId,
3 ✔
283
                ]
3 ✔
284
            );
3 ✔
285
    }
286

287
    public function markRequestedProcessesAsNotActive(array $processIds): void
288
    {
289
        $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME);
3 ✔
290
        $queryBuilder->update(self::TABLE_NAME)
3 ✔
291
            ->where(
3 ✔
292
                'NOT EXISTS (
3 ✔
293
                SELECT * FROM tx_crawler_queue
294
                    WHERE tx_crawler_queue.process_id = tx_crawler_process.process_id
295
                    AND tx_crawler_queue.exec_time = 0
296
                )',
3 ✔
297
                $queryBuilder->expr()->in(
3 ✔
298
                    'process_id',
3 ✔
299
                    $queryBuilder->createNamedParameter($processIds, ArrayParameterType::STRING)
3 ✔
300
                ),
3 ✔
301
                $queryBuilder->expr()->eq('deleted', 0)
3 ✔
302
            )
3 ✔
303
            ->set('active', '0')
3 ✔
304
            ->executeStatement();
3 ✔
305
    }
306

307
    public function addProcess(string $processId, int $systemProcessId): void
308
    {
309
        GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE_NAME)->insert(
3 ✔
310
            self::TABLE_NAME,
3 ✔
311
            [
3 ✔
312
                'process_id' => $processId,
3 ✔
313
                'active' => 1,
3 ✔
314
                'ttl' => time() + (int) $this->extensionSettings['processMaxRunTime'],
3 ✔
315
                'system_process_id' => $systemProcessId,
3 ✔
316
            ]
3 ✔
317
        );
3 ✔
318
    }
319
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc