14596906002

Committed 22 Apr 2025 02:09PM UTC coverage: 33.386% (-12.1%) from 45.455%

Build # 14596906002

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #78 from IGNF/fix_timeout_error

Fix timeout error

Run Details

0 of 61 new or added lines in 3 files covered. (0.0%)

72 existing lines in 5 files now uncovered.

210 of 629 relevant lines covered (33.39%)

2.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

36.3

/src/Controller/Api/ValidationsController.php

<?php

namespace App\Controller\Api;

use App\Entity\Validation;
use App\Exception\ApiException;
use App\Export\CsvReportWriter;
use App\Repository\ValidationRepository;
use App\Service\MimeTypeGuesserService;
use App\Service\ValidatorArgumentsService;
use App\Storage\ValidationsStorage;
use JMS\Serializer\Serializer;
use JMS\Serializer\SerializerInterface;
use League\Flysystem\FilesystemOperator;
use Psr\Log\LoggerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\Filesystem\Filesystem;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\StreamedResponse;
use Symfony\Component\Routing\Annotation\Route;

/**
 * @Route("/api/validations")
 */
class ValidationsController extends AbstractController
{
    public function __construct(
        private ValidationRepository $repository,
        private SerializerInterface $serializer,
        private ValidationsStorage $storage,
        private FilesystemOperator $dataStorage,
        private ValidatorArgumentsService $valArgsService,
        private MimeTypeGuesserService $mimeTypeGuesserService,
        private LoggerInterface $logger
    ) {

    }

    /**
     * @Route(
     *      "/",
     *      name="validator_api_disabled_routes",
     *      methods={"GET","DELETE","PATCH","PUT"}
     * )
     */
    public function disabledRoutes()
    {
        return new JsonResponse(['error' => "This route is not allowed"], Response::HTTP_METHOD_NOT_ALLOWED);
    }

    /**
     * @Route(
     *      "/{uid}",
     *      name="validator_api_get_validation",
     *      methods={"GET"}
     * )
     */
    public function getValidation($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        return new JsonResponse($this->serializer->toArray($validation), Response::HTTP_OK);
    }

    /**
     * @Route(
     *      "/{uid}/logs",
     *      name="validator_api_read_logs",
     *      methods={"GET"}
     * )
     */
    public function readConsole($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }

        $outputDirectory = $this->storage->getOutputDirectory($validation);
        $filepath = $outputDirectory . '/validator-debug.log';

        $content = $this->dataStorage->read($filepath);

        return new Response(
            $content,
            Response::HTTP_CREATED
        );
    }


    /**
     * @Route(
     *      "/{uid}/results.csv",
     *      name="validator_api_get_validation_csv",
     *      methods={"GET"}
     * )
     */
    public function getValidationCsv($uid, CsvReportWriter $csvWriter)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        $response = new StreamedResponse(function () use ($validation, $csvWriter) {
            $csvWriter->write($validation);
        });
        $response->headers->set('Content-Type', 'application/force-download');
        $filename = $uid . '-results.csv';
        $response->headers->set('Content-Disposition', 'attachment; filename="' . $filename . '"');

        return $response;
    }

    /**
     * @Route(
     *      "/",
     *      name="validator_api_upload_dataset",
     *      methods={"POST"}
     * )
     */
    public function uploadDataset(Request $request)
    {
        $files = $request->files;
        /*
         * Ensure that input file is submitted
         */
        $file = $files->get('dataset');
        if (!$file) {
            throw new ApiException("Argument [dataset] is missing", Response::HTTP_BAD_REQUEST);
        }

        $this->logger->info('handle new upload...',[
            'path_name' => $file->getPathName(),
            'client_original_name' => $file->getClientOriginalName()
        ]);

        /*
         * Ensure that input file is a ZIP file.
         */
        $mimeType = $this->mimeTypeGuesserService->guessMimeType($file->getPathName());
        if ($mimeType !== 'application/zip') {
            throw new ApiException("Dataset must be in a compressed [.zip] file", Response::HTTP_BAD_REQUEST);
        }

        /*
         * create validation and same validation
         */
        $validation = new Validation();
        // TODO : check getClientOriginalName
        $datasetName = str_replace('.zip', '', $file->getClientOriginalName());
        $validation->setDatasetName($datasetName);

        // Save file to storage
        $uploadDirectory = $this->storage->getUploadDirectory($validation);
        if (!$this->dataStorage->directoryExists($uploadDirectory)) {
            $this->dataStorage->createDirectory($uploadDirectory);
        }
        $fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip';
        if ($this->dataStorage->fileExists($fileLocation)) {
            $this->dataStorage->delete($fileLocation);
        }
        $stream = fopen($file->getRealPath(), 'r+');
        $this->dataStorage->writeStream($fileLocation, $stream);
        fclose($stream);

        $fs = new Filesystem;
        if ($fs->exists($file->getRealPath())) {
            $this->logger->debug('Validation[{uid}] : rm -rf {path}...', [
                'uid' => $validation->getUid(),
                'path' => $file->getRealPath(),
            ]);
            $fs->remove($file->getRealPath());
        }

        $em = $this->getDoctrine()->getManager();
        $em->persist($validation);
        $em->flush();
        $em->refresh($validation);

        return new JsonResponse(
            $this->serializer->toArray($validation),
            Response::HTTP_CREATED
        );
    }

    /**
     * @Route(
     *      "/{uid}",
     *      name="validator_api_update_arguments",
     *      methods={"PATCH"}
     * )
     */
    public function updateArguments(Request $request, $uid)
    {
        $data = $request->getContent();

        if (!json_decode($data, true)) {
            throw new ApiException("Request body must be a valid JSON string", Response::HTTP_BAD_REQUEST);
        }

        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);

        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }
        // TODO : review (json_decode in this method and inside of validate)
        $arguments = $this->valArgsService->validate($data);

        $validation->reset();
        $validation->setArguments($arguments);
        $validation->setStatus(Validation::STATUS_PENDING);

        $em = $this->getDoctrine()->getManager();
        $em->flush();
        $em->refresh($validation);

        return new JsonResponse(
            $this->serializer->toArray($validation),
            Response::HTTP_OK
        );
    }

    /**
     * @Route(
     *      "/{uid}",
     *      name="validator_api_delete_validation",
     *      methods={"DELETE"}
     * )
     */
    public function deleteValidation($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        $this->logger->info('Validation[{uid}] : removing all saved data...', [
            'uid' => $validation->getUid(),
            'datasetName' => $validation->getDatasetName(),
        ]);

        $em = $this->getDoctrine()->getManager();
        $em->remove($validation);
        $em->flush();

        // Delete from storage
        $uploadDirectory = $this->storage->getUploadDirectory($validation);
        if ($this->dataStorage->directoryExists($uploadDirectory)) {
            $this->dataStorage->deleteDirectory($uploadDirectory);
        }
        $outputDirectory = $this->storage->getOutputDirectory($validation);
        if ($this->dataStorage->directoryExists($outputDirectory)) {
            $this->dataStorage->deleteDirectory($outputDirectory);
        }

        return new JsonResponse(null, Response::HTTP_NO_CONTENT);
    }

    /**
     * @Route(
     *      "/{uid}/files/normalized",
     *      name="validator_api_download_normalized_data",
     *      methods={"GET"}
     * )
     */
    public function downloadNormalizedData($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }

        if ($validation->getStatus() == Validation::STATUS_ERROR) {
            throw new ApiException("Validation failed, no normalized data", Response::HTTP_FORBIDDEN);
        }

        if (in_array($validation->getStatus(), [Validation::STATUS_PENDING, Validation::STATUS_PROCESSING, Validation::STATUS_WAITING_ARGS])) {
            throw new ApiException("Validation hasn't been executed yet", Response::HTTP_FORBIDDEN);
        }

        $outputDirectory = $this->storage->getOutputDirectory($validation);
        $zipFilepath = $outputDirectory . $validation->getDatasetName() . '.zip';
        return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-normalized.zip");
    }

    /**
     * @Route(
     *      "/{uid}/files/source",
     *      name="validator_api_download_source_data",
     *      methods={"GET"}
     * )
     */
    public function downloadSourceData($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }

        $uploadDirectory = $this->storage->getUploadDirectory($validation);
        $zipFilepath = $uploadDirectory . $validation->getDatasetName() . '.zip';
        return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-source.zip");
    }

    /**
     * Returns binary response of the specified file
     *
     * @param string $dirpath
     * @param string $filename
     * @return StreamedResponse
     */
    private function getDownloadResponse($filepath, $filename)
    {
        if (!$this->dataStorage->has($filepath)) {
            throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN);
        }

        $stream = $this->dataStorage->readStream($filepath);

        return new StreamedResponse(function () use ($stream) {
            fpassthru($stream);
            exit();
        }, 200, [
            'Content-Transfer-Encoding',
            'binary',
            'Content-Type' => 'application/zip',
            'Content-Disposition' => sprintf('attachment; filename="%s"', $filename),
            'Content-Length' => fstat($stream)['size']
        ]);
    }
}

1	<?php
2
3	namespace App\Controller\Api;
4
5	use App\Entity\Validation;
6	use App\Exception\ApiException;
7	use App\Export\CsvReportWriter;
8	use App\Repository\ValidationRepository;
9	use App\Service\MimeTypeGuesserService;
10	use App\Service\ValidatorArgumentsService;
11	use App\Storage\ValidationsStorage;
12	use JMS\Serializer\Serializer;
13	use JMS\Serializer\SerializerInterface;
14	use League\Flysystem\FilesystemOperator;
15	use Psr\Log\LoggerInterface;
16	use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
17	use Symfony\Component\Filesystem\Filesystem;
18	use Symfony\Component\HttpFoundation\JsonResponse;
19	use Symfony\Component\HttpFoundation\Request;
20	use Symfony\Component\HttpFoundation\Response;
21	use Symfony\Component\HttpFoundation\StreamedResponse;
22	use Symfony\Component\Routing\Annotation\Route;
23
24	/**
25	* @Route("/api/validations")
26	*/
27	class ValidationsController extends AbstractController
28	{
29	public function __construct(
30	private ValidationRepository $repository,
31	private SerializerInterface $serializer,
32	private ValidationsStorage $storage,
33	private FilesystemOperator $dataStorage,
34	private ValidatorArgumentsService $valArgsService,
35	private MimeTypeGuesserService $mimeTypeGuesserService,
36	private LoggerInterface $logger
37	) {
38
39	}	20✔
40
41	/**
42	* @Route(
43	* "/",
44	* name="validator_api_disabled_routes",
45	* methods={"GET","DELETE","PATCH","PUT"}
46	* )
47	*/
48	public function disabledRoutes()
49	{
50	return new JsonResponse(['error' => "This route is not allowed"], Response::HTTP_METHOD_NOT_ALLOWED);	1✔
51	}
52
53	/**
54	* @Route(
55	* "/{uid}",
56	* name="validator_api_get_validation",
57	* methods={"GET"}
58	* )
59	*/
60	public function getValidation($uid)
61	{
62	$validation = $this->repository->findOneByUid($uid);	2✔
63	if (!$validation) {	2✔
64	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
65	}
66
67	return new JsonResponse($this->serializer->toArray($validation), Response::HTTP_OK);	1✔
68	}
69
70	/**
71	* @Route(
72	* "/{uid}/logs",
73	* name="validator_api_read_logs",
74	* methods={"GET"}
75	* )
76	*/
77	public function readConsole($uid)
78	{
79	$validation = $this->repository->findOneByUid($uid);	×
80	if (!$validation) {	×
81	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	×
82	}
83
84	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	×
85	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	×
86	}
87
NEW 88	$outputDirectory = $this->storage->getOutputDirectory($validation);	×
NEW 89	$filepath = $outputDirectory . '/validator-debug.log';	×
90
91	$content = $this->dataStorage->read($filepath);	×
92
93	return new Response(	×
94	$content,	×
95	Response::HTTP_CREATED	×
96	);	×
97	}
98
99
100	/**
101	* @Route(
102	* "/{uid}/results.csv",
103	* name="validator_api_get_validation_csv",
104	* methods={"GET"}
105	* )
106	*/
107	public function getValidationCsv($uid, CsvReportWriter $csvWriter)
108	{
109	$validation = $this->repository->findOneByUid($uid);	×
110	if (!$validation) {	×
111	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	×
112	}
113
114	$response = new StreamedResponse(function () use ($validation, $csvWriter) {	×
115	$csvWriter->write($validation);	×
116	});	×
117	$response->headers->set('Content-Type', 'application/force-download');	×
118	$filename = $uid . '-results.csv';	×
119	$response->headers->set('Content-Disposition', 'attachment; filename="' . $filename . '"');	×
120
121	return $response;	×
122	}
123
124	/**
125	* @Route(
126	* "/",
127	* name="validator_api_upload_dataset",
128	* methods={"POST"}
129	* )
130	*/
131	public function uploadDataset(Request $request)
132	{
133	$files = $request->files;	3✔
134	/*
135	* Ensure that input file is submitted
136	*/
137	$file = $files->get('dataset');	3✔
138	if (!$file) {	3✔
139	throw new ApiException("Argument [dataset] is missing", Response::HTTP_BAD_REQUEST);	2✔
140	}
141
142	$this->logger->info('handle new upload...',[	1✔
143	'path_name' => $file->getPathName(),	1✔
144	'client_original_name' => $file->getClientOriginalName()	1✔
145	]);	1✔
146
147	/*
148	* Ensure that input file is a ZIP file.
149	*/
150	$mimeType = $this->mimeTypeGuesserService->guessMimeType($file->getPathName());	1✔
151	if ($mimeType !== 'application/zip') {	1✔
152	throw new ApiException("Dataset must be in a compressed [.zip] file", Response::HTTP_BAD_REQUEST);	1✔
153	}
154
155	/*
156	* create validation and same validation
157	*/
158	$validation = new Validation();	×
159	// TODO : check getClientOriginalName
160	$datasetName = str_replace('.zip', '', $file->getClientOriginalName());	×
161	$validation->setDatasetName($datasetName);	×
162
163	// Save file to storage
NEW 164	$uploadDirectory = $this->storage->getUploadDirectory($validation);	×
165	if (!$this->dataStorage->directoryExists($uploadDirectory)) {	×
166	$this->dataStorage->createDirectory($uploadDirectory);	×
167	}
168	$fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip';	×
169	if ($this->dataStorage->fileExists($fileLocation)) {	×
170	$this->dataStorage->delete($fileLocation);	×
171	}
172	$stream = fopen($file->getRealPath(), 'r+');	×
173	$this->dataStorage->writeStream($fileLocation, $stream);	×
174	fclose($stream);	×
175
NEW 176	$fs = new Filesystem;	×
NEW 177	if ($fs->exists($file->getRealPath())) {	×
NEW 178	$this->logger->debug('Validation[{uid}] : rm -rf {path}...', [	×
NEW 179	'uid' => $validation->getUid(),	×
NEW 180	'path' => $file->getRealPath(),	×
NEW 181	]);	×
NEW 182	$fs->remove($file->getRealPath());	×
183	}
184
185	$em = $this->getDoctrine()->getManager();	×
186	$em->persist($validation);	×
187	$em->flush();	×
188	$em->refresh($validation);	×
189
190	return new JsonResponse(	×
191	$this->serializer->toArray($validation),	×
192	Response::HTTP_CREATED	×
193	);	×
194	}
195
196	/**
197	* @Route(
198	* "/{uid}",
199	* name="validator_api_update_arguments",
200	* methods={"PATCH"}
201	* )
202	*/
203	public function updateArguments(Request $request, $uid)
204	{
205	$data = $request->getContent();	10✔
206
207	if (!json_decode($data, true)) {	10✔
208	throw new ApiException("Request body must be a valid JSON string", Response::HTTP_BAD_REQUEST);	2✔
209	}
210
211	$validation = $this->repository->findOneByUid($uid);	8✔
212	if (!$validation) {	8✔
213	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
214
215	}
216
217	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	7✔
218	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	1✔
219	}
220	// TODO : review (json_decode in this method and inside of validate)
221	$arguments = $this->valArgsService->validate($data);	6✔
222
223	$validation->reset();	1✔
224	$validation->setArguments($arguments);	1✔
225	$validation->setStatus(Validation::STATUS_PENDING);	1✔
226
227	$em = $this->getDoctrine()->getManager();	1✔
228	$em->flush();	1✔
229	$em->refresh($validation);	1✔
230
231	return new JsonResponse(	1✔
232	$this->serializer->toArray($validation),	1✔
233	Response::HTTP_OK	1✔
234	);	1✔
235	}
236
237	/**
238	* @Route(
239	* "/{uid}",
240	* name="validator_api_delete_validation",
241	* methods={"DELETE"}
242	* )
243	*/
244	public function deleteValidation($uid)
245	{
246	$validation = $this->repository->findOneByUid($uid);	×
247	if (!$validation) {	×
248	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	×
249	}
250
NEW 251	$this->logger->info('Validation[{uid}] : removing all saved data...', [	×
NEW 252	'uid' => $validation->getUid(),	×
NEW 253	'datasetName' => $validation->getDatasetName(),	×
NEW 254	]);	×
255
256	$em = $this->getDoctrine()->getManager();	×
257	$em->remove($validation);	×
258	$em->flush();	×
259
260	// Delete from storage
NEW 261	$uploadDirectory = $this->storage->getUploadDirectory($validation);	×
NEW 262	if ($this->dataStorage->directoryExists($uploadDirectory)) {	×
NEW 263	$this->dataStorage->deleteDirectory($uploadDirectory);	×
264	}
NEW 265	$outputDirectory = $this->storage->getOutputDirectory($validation);	×
NEW 266	if ($this->dataStorage->directoryExists($outputDirectory)) {	×
NEW 267	$this->dataStorage->deleteDirectory($outputDirectory);	×
268	}
269
270	return new JsonResponse(null, Response::HTTP_NO_CONTENT);	×
271	}
272
273	/**
274	* @Route(
275	* "/{uid}/files/normalized",
276	* name="validator_api_download_normalized_data",
277	* methods={"GET"}
278	* )
279	*/
280	public function downloadNormalizedData($uid)
281	{
282	$validation = $this->repository->findOneByUid($uid);	2✔
283	if (!$validation) {	2✔
284	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
285	}
286
287	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	1✔
288	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	1✔
289	}
290
291	if ($validation->getStatus() == Validation::STATUS_ERROR) {	1✔
292	throw new ApiException("Validation failed, no normalized data", Response::HTTP_FORBIDDEN);	×
293	}
294
295	if (in_array($validation->getStatus(), [Validation::STATUS_PENDING, Validation::STATUS_PROCESSING, Validation::STATUS_WAITING_ARGS])) {	1✔
296	throw new ApiException("Validation hasn't been executed yet", Response::HTTP_FORBIDDEN);	1✔
297	}
298
NEW 299	$outputDirectory = $this->storage->getOutputDirectory($validation);	×
NEW 300	$zipFilepath = $outputDirectory . $validation->getDatasetName() . '.zip';	×
UNCOV 301	return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-normalized.zip");	×
302	}
303
304	/**
305	* @Route(
306	* "/{uid}/files/source",
307	* name="validator_api_download_source_data",
308	* methods={"GET"}
309	* )
310	*/
311	public function downloadSourceData($uid)
312	{
313	$validation = $this->repository->findOneByUid($uid);	2✔
314	if (!$validation) {	2✔
315	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
316	}
317
318	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	1✔
319	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	1✔
320	}
321
NEW 322	$uploadDirectory = $this->storage->getUploadDirectory($validation);	×
323	$zipFilepath = $uploadDirectory . $validation->getDatasetName() . '.zip';	×
324	return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-source.zip");	×
325	}
326
327	/**
328	* Returns binary response of the specified file
329	*
330	* @param string $dirpath
331	* @param string $filename
332	* @return StreamedResponse
333	*/
334	private function getDownloadResponse($filepath, $filename)
335	{
336	if (!$this->dataStorage->has($filepath)) {	×
337	throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN);	×
338	}
339
340	$stream = $this->dataStorage->readStream($filepath);	×
341
342	return new StreamedResponse(function () use ($stream) {	×
343	fpassthru($stream);	×
344	exit();	×
345	}, 200, [	×
346	'Content-Transfer-Encoding',	×
347	'binary',	×
348	'Content-Type' => 'application/zip',	×
349	'Content-Disposition' => sprintf('attachment; filename="%s"', $filename),	×
350	'Content-Length' => fstat($stream)['size']	×
351	]);	×
352	}
353	}

IGNF / validator-api / 14596906002

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous