• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

IGNF / validator-api / 15188458321

22 May 2025 01:53PM UTC coverage: 53.006% (+19.6%) from 33.386%
15188458321

push

github

web-flow
fix: change storage location based on environmental variable

Storage adapter

11 of 28 new or added lines in 4 files covered. (39.29%)

335 of 632 relevant lines covered (53.01%)

2.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.76
/src/Validation/ValidationManager.php
1
<?php
2

3
namespace App\Validation;
4

5
use App\Entity\Validation;
6
use App\Exception\ZipArchiveValidationException;
7
use App\Storage\ValidationsStorage;
8
use Doctrine\ORM\EntityManagerInterface;
9
use Psr\Log\LoggerInterface;
10
use RuntimeException;
11
use Symfony\Component\Filesystem\Filesystem;
12
use Symfony\Component\Process\Exception\ProcessFailedException;
13
use Symfony\Component\Process\Process;
14

15
class ValidationManager
16
{
17

18
    /**
19
     * @var EntityManagerInterface
20
     */
21
    private $em;
22

23
    /**
24
     * @var ValidationsStorage
25
     */
26
    private $storage;
27

28
    /**
29
     * @var ValidatorCLI
30
     */
31
    private $validatorCli;
32

33
    /**
34
     * @var LoggerInterface
35
     */
36
    private $logger;
37

38
    /**
39
     * @var ZipArchiveValidator
40
     */
41
    private $zipArchiveValidator;
42

43
    /**
44
     * Current validation (in order to handle SIGTERM)
45
     * @var Validation
46
     */
47
    private $currentValidation = null;
48

49
    public function __construct(
50
        EntityManagerInterface $em,
51
        ValidationsStorage $storage,
52
        ValidatorCLI $validatorCli,
53
        ZipArchiveValidator $zipArchiveValidator,
54
        LoggerInterface $logger
55
    ) {
56
        $this->em = $em;
1✔
57
        $this->storage = $storage;
1✔
58
        $this->validatorCli = $validatorCli;
1✔
59
        $this->zipArchiveValidator = $zipArchiveValidator;
1✔
60
        $this->logger = $logger;
1✔
61
    }
62

63
    /**
64
     * Archive a given validation removing all local files.
65
     *
66
     * @param Validation $validation
67
     * @return void
68
     */
69
    public function archive(Validation $validation)
70
    {
71
        $this->logger->info('Validation[{uid}] : archive removing all files...', [
1✔
72
            'uid' => $validation->getUid(),
1✔
73
        ]);
1✔
74
        $validationDirectory = $this->storage->getDirectory($validation);
1✔
75
        $fs = new Filesystem();
1✔
76
        if ($fs->exists($validationDirectory)) {
1✔
77
            $this->logger->debug('Validation[{uid}] : remove validation directory ...', [
1✔
78
                'uid' => $validation->getUid(),
1✔
79
                'validationDirectory' => $validationDirectory,
1✔
80
            ]);
1✔
81
            $fs->remove($validationDirectory);
1✔
82
        }
83

84
        // Delete from storage
85
        $this->logger->info('Validation[{uid}] : remove upload files', [
1✔
86
            'uid' => $validation->getUid(),
1✔
87
        ]);
1✔
88
        $uploadDirectory = $this->storage->getUploadDirectory($validation);
1✔
89
        if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {
1✔
NEW
90
            $this->storage->getStorage()->deleteDirectory($uploadDirectory);
×
91
        }
92
        $this->logger->info('Validation[{uid}] : remove output files', [
1✔
93
            'uid' => $validation->getUid(),
1✔
94
        ]);
1✔
95
        $outputDirectory = $this->storage->getOutputDirectory($validation);
1✔
96
        if ($this->storage->getStorage()->directoryExists($outputDirectory)) {
1✔
NEW
97
            $this->storage->getStorage()->deleteDirectory($outputDirectory);
×
98
        }
99
        $this->logger->info('Validation[{uid}] : archive removing all files : completed', [
1✔
100
            'uid' => $validation->getUid(),
1✔
101
            'status' => Validation::STATUS_ARCHIVED,
1✔
102
        ]);
1✔
103
        $validation->setStatus(Validation::STATUS_ARCHIVED);
1✔
104
        $this->em->persist($validation);
1✔
105
        $this->em->flush();
1✔
106
    }
107

108
    /**
109
     * Process next pending validation.
110
     *
111
     * @return void
112
     */
113
    public function processOne()
114
    {
115
        $validation = $this->getValidationRepository()->popNextPending();
×
116
        if (is_null($validation)) {
×
117
            $this->logger->debug("processOne : no validation pending, quitting");
×
118
            return;
×
119
        }
120
        $this->currentValidation = $validation;
×
121
        $this->doProcess($validation);
×
122
        $this->currentValidation = null;
×
123
    }
124

125
    /**
126
     * Stop currently running validation (invoked when SIGTERM is received)
127
     *
128
     * @return void
129
     */
130
    public function cancelProcessing(){
131
        if ( is_null($this->currentValidation) ){
×
132
            $this->logger->debug("SIGTERM received, no validation in progress");
×
133
            return ;
×
134
        }
135
        $this->logger->warning("Validation[{uid}]: SIGTERM received, changing state to pending",[
×
136
            "uid" => $this->currentValidation->getUid()
×
137
        ]);
×
138
        $this->currentValidation->setStatus(Validation::STATUS_PENDING);
×
139
        $this->em->persist($this->currentValidation);
×
140
        $this->em->flush();
×
141
    }
142

143
    /**
144
     * Process pending validation
145
     *
146
     * @param Validation $validation
147
     * @return void
148
     */
149
    private function doProcess(Validation $validation)
150
    {
151
        $this->logger->info("Validation[{uid}]: process pending validation...", ['uid' => $validation->getUid()]);
×
152

153
        /*
154
         * force usage of popNextPending to avoid concurrency problems.
155
         */
156
        if (Validation::STATUS_PROCESSING !== $validation->getStatus()) {
×
157
            $message = sprintf(
×
158
                'doProcess must be invoked on validation with status %s (current status is %s)',
×
159
                Validation::STATUS_PROCESSING,
×
160
                $validation->getStatus()
×
161
            );
×
162
            $this->logger->error($message, ['uid' => $validation->getUid()]);
×
163
            throw new RuntimeException($message);
×
164
        }
165

166
        try {
167
            /*
168
             * get files from storage
169
             */
170
            $this->getZip($validation);
×
171

172
            /*
173
             * pre-validating the names of the files in the zip archive
174
             */
175
            $this->validateZip($validation);
×
176

177
            /*
178
             * unzip dataset
179
             */
180
            $this->unzip($validation);
×
181

182
            /*
183
             * run validator-cli.jar command
184
             */
185
            $this->validatorCli->process($validation);
×
186

187
            /*
188
             * zip normalized results
189
             */
190
            $this->zipNormData($validation);
×
191

192
            /*
193
             * Save validation data to storage
194
             */
195
            $this->saveToStorage($validation);
×
196

197
            /*
198
             * cleanup data
199
             */
200
            $this->cleanUp($validation);
×
201

202
            $validation->setStatus(Validation::STATUS_FINISHED);
×
203
            $this->logger->info("Validation[{uid}]: validation carried out successfully", ['uid' => $validation->getUid()]);
×
204
        } catch (ZipArchiveValidationException $ex) {
×
205
            $validation->setStatus(Validation::STATUS_ERROR);
×
206
            $validation->setMessage($ex->getMessage());
×
207
            $validation->setResults($ex->getErrors());
×
208
            $this->logger->error("Validation[{uid}]: {message}: {errors}", ['uid' => $validation->getUid(), 'message' => $ex->getMessage(), 'errors' => $ex->getErrors()]);
×
209
        } catch (\Throwable $th) {
×
210
            $validation->setStatus(Validation::STATUS_ERROR);
×
211
            $validation->setMessage($th->getMessage());
×
212
            $this->logger->error("Validation[{uid}]: {message}", ['uid' => $validation->getUid(), 'message' => $th->getMessage()]);
×
213
        }
214

215
        $validation->setDateFinish(new \DateTime('now'));
×
216
        $this->em->persist($validation);
×
217
        $this->em->flush();
×
218
    }
219

220
    /**
221
     * Get Zip file from storage to validate
222
     * @param Validation $validation
223
     * @return void
224
     */
225
    private function getZip(Validation $validation)
226
    {
227
        $this->logger->info('Validation[{uid}] : get from storage...', [
×
228
            'uid' => $validation->getUid(),
×
229
            'datasetName' => $validation->getDatasetName(),
×
230
        ]);
×
231

232
        $validationDirectory = $this->storage->getDirectory($validation);
×
233
        $uploadFile = $this->storage->getUploadDirectory($validation) . $validation->getDatasetName() . '.zip';
×
234

235
        if (!is_dir($validationDirectory)) {
×
236
            mkdir($validationDirectory);
×
237
        }
238

239
        $zipPath = $validationDirectory . '/' . $validation->getDatasetName() . '.zip';
×
240

241
        file_put_contents(
×
242
            $zipPath,
×
NEW
243
            $this->storage->getStorage()->read($uploadFile)
×
244
        );
×
245
    }
246

247
    /**
248
     * Pre-validates the names of files in the zip
249
     *
250
     * @param Validation $validation
251
     * @return void
252
     * @throws ZipArchiveValidationException
253
     */
254
    private function validateZip($validation)
255
    {
256
        $this->logger->info('Validation[{uid}] : validate zip archive...', [
×
257
            'uid' => $validation->getUid(),
×
258
            'datasetName' => $validation->getDatasetName(),
×
259
        ]);
×
260
        $validationDirectory = $this->storage->getDirectory($validation);
×
261
        $zipPath = $validationDirectory . '/' . $validation->getDatasetName() . '.zip';
×
262
        $errors = $this->zipArchiveValidator->validate($zipPath);
×
263
        if (count($errors) > 0) {
×
264
            throw new ZipArchiveValidationException($errors);
×
265
        }
266
    }
267

268
    /**
269
     * Unzips the compressed dataset
270
     *
271
     * @param Validation $validation
272
     * @return void
273
     */
274
    private function unzip(Validation $validation)
275
    {
276
        $this->logger->info('Validation[{uid}] : extract source archive...', [
×
277
            'uid' => $validation->getUid(),
×
278
            'datasetName' => $validation->getDatasetName(),
×
279
        ]);
×
280
        $validationDirectory = $this->storage->getDirectory($validation);
×
281
        $zipFilename = $validationDirectory . '/' . $validation->getDatasetName() . '.zip';
×
282
        $zip = new \ZipArchive();
×
283

284
        if ($zip->open($zipFilename) === true) {
×
285
            $zip->extractTo($validationDirectory . '/' . $validation->getDatasetName());
×
286
            $zip->close();
×
287
        } else {
288
            throw new \Exception("Zip decompression failed");
×
289
        }
290
    }
291

292
    /**
293
     * Zips the generated normalized data
294
     *
295
     * @param Validation $validation
296
     * @return void
297
     */
298
    private function zipNormData(Validation $validation)
299
    {
300
        $this->logger->info('Validation[{uid}] : compress normalized data...', [
×
301
            'uid' => $validation->getUid(),
×
302
            'datasetName' => $validation->getDatasetName(),
×
303
        ]);
×
304
        $fs = new Filesystem();
×
305

306
        $validationDirectory = $this->storage->getDirectory($validation);
×
307
        $normDataParentDir = $validationDirectory . '/validation/';
×
308
        $datasetName = $validation->getDatasetName();
×
309

310
        // checking if normalized data is present
311
        if (!$fs->exists($normDataParentDir . $datasetName)) {
×
312
            return;
×
313
        }
314

315
        $process = new Process(["zip","-r","$datasetName.zip",$datasetName],$normDataParentDir);
×
316
        $process->setTimeout(600);
×
317
        $process->setIdleTimeout(600);
×
318
        $process->run();
×
319

320
        if (!$process->isSuccessful()) {
×
321
            throw new ProcessFailedException($process);
×
322
        }
323
    }
324

325
    /**
326
     * Saves output to storage
327
     */
328
    private function saveToStorage(Validation $validation)
329
    {
330
        // Saves normalized data to storage
331
        $this->logger->info('Validation[{uid}] : saving normalized data...', [
×
332
            'uid' => $validation->getUid(),
×
333
            'datasetName' => $validation->getDatasetName(),
×
334
        ]);
×
335
        $validationDirectory = $this->storage->getDirectory($validation);
×
336
        $normDataPath = $validationDirectory . '/validation/' . $validation->getDatasetName() . '.zip';
×
337
        $outputDirectory = $this->storage->getOutputDirectory($validation);
×
NEW
338
        if (! $this->storage->getStorage()->directoryExists($outputDirectory)){
×
NEW
339
            $this->storage->getStorage()->createDirectory($outputDirectory);
×
340
        }
341
        $outputPath = $outputDirectory . $validation->getDatasetName() . '.zip';
×
NEW
342
        if ($this->storage->getStorage()->fileExists($outputPath)){
×
NEW
343
            $this->storage->getStorage()->delete($outputPath);
×
344
        }
345
        $stream = fopen($normDataPath, 'r+');
×
NEW
346
        $this->storage->getStorage()->writeStream($outputPath, $stream);
×
347
        fclose($stream);
×
348

349
        // Saves validator logs to storage
350
        $this->logger->info('Validation[{uid}] : saving logs...', [
×
351
            'uid' => $validation->getUid(),
×
352
            'datasetName' => $validation->getDatasetName(),
×
353
        ]);
×
354
        $logPath = $validationDirectory . '/validator-debug.log';
×
355
        $outputPath = $outputDirectory . '/validator-debug.log';
×
356

357
        $stream = fopen($logPath, 'r+');
×
NEW
358
        $this->storage->getStorage()->writeStream($outputPath, $stream);
×
359
        fclose($stream);
×
360
    }
361

362
    /**
363
     * Cleans up temporary files
364
     *
365
     * @return void
366
     */
367
    private function cleanUp(Validation $validation)
368
    {
369
        $this->logger->info('Validation[{uid}] : cleanup...', [
×
370
            'uid' => $validation->getUid(),
×
371
            'datasetName' => $validation->getDatasetName(),
×
372
        ]);
×
373
        $validationDirectory = $this->storage->getDirectory($validation);
×
374

375
        $fs = new FileSystem();
×
376
        if ($fs->exists($validationDirectory)) {
×
377
            $this->logger->debug('Validation[{uid}] : rm -rf {uid}/{datasetName}/...', [
×
378
                'uid' => $validation->getUid(),
×
379
                'datasetName' => $validation->getDatasetName(),
×
380
            ]);
×
381
            $fs->remove($validationDirectory);
×
382
        }
383
    }
384

385
    /**
386
     * @return ValidationRepository
387
     */
388
    protected function getValidationRepository()
389
    {
390
        return $this->em->getRepository(Validation::class);
×
391
    }
392

393
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc