15188458321

Committed 22 May 2025 01:53PM UTC coverage: 53.006% (+19.6%) from 33.386%

Build # 15188458321

Build Type

push

github

Committed by

web-flow

Commit Message

fix: change storage location based on environmental variable

Storage adapter

Coverage Stats

11 of 28 new or added lines in 4 files covered. (39.29%)

335 of 632 relevant lines covered (53.01%)

2.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67

/src/Controller/Api/ValidationsController.php

<?php

namespace App\Controller\Api;

use App\Entity\Validation;
use App\Exception\ApiException;
use App\Export\CsvReportWriter;
use App\Repository\ValidationRepository;
use App\Service\MimeTypeGuesserService;
use App\Service\ValidatorArgumentsService;
use App\Storage\ValidationsStorage;
use JMS\Serializer\Serializer;
use JMS\Serializer\SerializerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\Filesystem\Filesystem;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\StreamedResponse;
use Symfony\Component\Routing\Annotation\Route;

/**
 * @Route("/api/validations")
 */
class ValidationsController extends AbstractController
{
    public function __construct(
        private ValidationRepository $repository,
        private SerializerInterface $serializer,
        private ValidationsStorage $storage,
        private ValidatorArgumentsService $valArgsService,
        private MimeTypeGuesserService $mimeTypeGuesserService,
        private LoggerInterface $logger
    ) {

    }

    /**
     * @Route(
     *      "/",
     *      name="validator_api_disabled_routes",
     *      methods={"GET","DELETE","PATCH","PUT"}
     * )
     */
    public function disabledRoutes()
    {
        return new JsonResponse(['error' => "This route is not allowed"], Response::HTTP_METHOD_NOT_ALLOWED);
    }

    /**
     * @Route(
     *      "/{uid}",
     *      name="validator_api_get_validation",
     *      methods={"GET"}
     * )
     */
    public function getValidation($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        return new JsonResponse($this->serializer->toArray($validation), Response::HTTP_OK);
    }

    /**
     * @Route(
     *      "/{uid}/logs",
     *      name="validator_api_read_logs",
     *      methods={"GET"}
     * )
     */
    public function readConsole($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }

        $outputDirectory = $this->storage->getOutputDirectory($validation);
        $filepath = $outputDirectory . '/validator-debug.log';

        $content = $this->storage->getStorage()->read($filepath);

        return new Response(
            $content,
            Response::HTTP_CREATED
        );
    }


    /**
     * @Route(
     *      "/{uid}/results.csv",
     *      name="validator_api_get_validation_csv",
     *      methods={"GET"}
     * )
     */
    public function getValidationCsv($uid, CsvReportWriter $csvWriter)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        $response = new StreamedResponse(function () use ($validation, $csvWriter) {
            $csvWriter->write($validation);
        });
        $response->headers->set('Content-Type', 'application/force-download');
        $filename = $uid . '-results.csv';
        $response->headers->set('Content-Disposition', 'attachment; filename="' . $filename . '"');

        return $response;
    }

    /**
     * @Route(
     *      "/",
     *      name="validator_api_upload_dataset",
     *      methods={"POST"}
     * )
     */
    public function uploadDataset(Request $request)
    {
        $files = $request->files;
        /*
         * Ensure that input file is submitted
         */
        $file = $files->get('dataset');
        if (!$file) {
            throw new ApiException("Argument [dataset] is missing", Response::HTTP_BAD_REQUEST);
        }

        $this->logger->info('handle new upload...',[
            'path_name' => $file->getPathName(),
            'client_original_name' => $file->getClientOriginalName()
        ]);

        /*
         * Ensure that input file is a ZIP file.
         */
        $mimeType = $this->mimeTypeGuesserService->guessMimeType($file->getPathName());
        if ($mimeType !== 'application/zip') {
            throw new ApiException("Dataset must be in a compressed [.zip] file", Response::HTTP_BAD_REQUEST);
        }

        /*
         * create validation and same validation
         */
        $validation = new Validation();
        // TODO : check getClientOriginalName
        $datasetName = str_replace('.zip', '', $file->getClientOriginalName());
        $validation->setDatasetName($datasetName);

        // Save file to storage
        $uploadDirectory = $this->storage->getUploadDirectory($validation);
        if (!$this->storage->getStorage()->directoryExists($uploadDirectory)) {
            $this->storage->getStorage()->createDirectory($uploadDirectory);
        }
        $fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip';
        if ($this->storage->getStorage()->fileExists($fileLocation)) {
            $this->storage->getStorage()->delete($fileLocation);
        }
        $stream = fopen($file->getRealPath(), 'r+');
        $this->storage->getStorage()->writeStream($fileLocation, $stream);
        fclose($stream);

        $fs = new Filesystem;
        if ($fs->exists($file->getRealPath())) {
            $this->logger->debug('Validation[{uid}] : rm -rf {path}...', [
                'uid' => $validation->getUid(),
                'path' => $file->getRealPath(),
            ]);
            $fs->remove($file->getRealPath());
        }

        $em = $this->getDoctrine()->getManager();
        $em->persist($validation);
        $em->flush();
        $em->refresh($validation);

        return new JsonResponse(
            $this->serializer->toArray($validation),
            Response::HTTP_CREATED
        );
    }

    /**
     * @Route(
     *      "/{uid}",
     *      name="validator_api_update_arguments",
     *      methods={"PATCH"}
     * )
     */
    public function updateArguments(Request $request, $uid)
    {
        $data = $request->getContent();

        if (!json_decode($data, true)) {
            throw new ApiException("Request body must be a valid JSON string", Response::HTTP_BAD_REQUEST);
        }

        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);

        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }
        // TODO : review (json_decode in this method and inside of validate)
        $arguments = $this->valArgsService->validate($data);

        $validation->reset();
        $validation->setArguments($arguments);
        $validation->setStatus(Validation::STATUS_PENDING);

        $em = $this->getDoctrine()->getManager();
        $em->flush();
        $em->refresh($validation);

        return new JsonResponse(
            $this->serializer->toArray($validation),
            Response::HTTP_OK
        );
    }

    /**
     * @Route(
     *      "/{uid}",
     *      name="validator_api_delete_validation",
     *      methods={"DELETE"}
     * )
     */
    public function deleteValidation($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        $this->logger->info('Validation[{uid}] : removing all saved data...', [
            'uid' => $validation->getUid(),
            'datasetName' => $validation->getDatasetName(),
        ]);

        $em = $this->getDoctrine()->getManager();
        $em->remove($validation);
        $em->flush();

        // Delete from storage
        $uploadDirectory = $this->storage->getUploadDirectory($validation);
        if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {
            $this->storage->getStorage()->deleteDirectory($uploadDirectory);
        }
        $outputDirectory = $this->storage->getOutputDirectory($validation);
        if ($this->storage->getStorage()->directoryExists($outputDirectory)) {
            $this->storage->getStorage()->deleteDirectory($outputDirectory);
        }

        return new JsonResponse(null, Response::HTTP_NO_CONTENT);
    }

    /**
     * @Route(
     *      "/{uid}/files/normalized",
     *      name="validator_api_download_normalized_data",
     *      methods={"GET"}
     * )
     */
    public function downloadNormalizedData($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }

        if ($validation->getStatus() == Validation::STATUS_ERROR) {
            throw new ApiException("Validation failed, no normalized data", Response::HTTP_FORBIDDEN);
        }

        if (in_array($validation->getStatus(), [Validation::STATUS_PENDING, Validation::STATUS_PROCESSING, Validation::STATUS_WAITING_ARGS])) {
            throw new ApiException("Validation hasn't been executed yet", Response::HTTP_FORBIDDEN);
        }

        $outputDirectory = $this->storage->getOutputDirectory($validation);
        $zipFilepath = $outputDirectory . $validation->getDatasetName() . '.zip';
        return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-normalized.zip");
    }

    /**
     * @Route(
     *      "/{uid}/files/source",
     *      name="validator_api_download_source_data",
     *      methods={"GET"}
     * )
     */
    public function downloadSourceData($uid)
    {
        $validation = $this->repository->findOneByUid($uid);
        if (!$validation) {
            throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);
        }

        if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {
            throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);
        }

        $uploadDirectory = $this->storage->getUploadDirectory($validation);
        $zipFilepath = $uploadDirectory . $validation->getDatasetName() . '.zip';
        return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-source.zip");
    }

    /**
     * Returns binary response of the specified file
     *
     * @param string $dirpath
     * @param string $filename
     * @return StreamedResponse
     */
    private function getDownloadResponse($filepath, $filename)
    {
        if (!$this->storage->getStorage()->has($filepath)) {
            throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN);
        }

        $stream = $this->storage->getStorage()->readStream($filepath);

        return new StreamedResponse(function () use ($stream) {
            fpassthru($stream);
            exit();
        }, 200, [
            'Content-Transfer-Encoding',
            'binary',
            'Content-Type' => 'application/zip',
            'Content-Disposition' => sprintf('attachment; filename="%s"', $filename),
            'Content-Length' => fstat($stream)['size']
        ]);
    }
}

1	<?php
2
3	namespace App\Controller\Api;
4
5	use App\Entity\Validation;
6	use App\Exception\ApiException;
7	use App\Export\CsvReportWriter;
8	use App\Repository\ValidationRepository;
9	use App\Service\MimeTypeGuesserService;
10	use App\Service\ValidatorArgumentsService;
11	use App\Storage\ValidationsStorage;
12	use JMS\Serializer\Serializer;
13	use JMS\Serializer\SerializerInterface;
14	use Psr\Log\LoggerInterface;
15	use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
16	use Symfony\Component\Filesystem\Filesystem;
17	use Symfony\Component\HttpFoundation\JsonResponse;
18	use Symfony\Component\HttpFoundation\Request;
19	use Symfony\Component\HttpFoundation\Response;
20	use Symfony\Component\HttpFoundation\StreamedResponse;
21	use Symfony\Component\Routing\Annotation\Route;
22
23	/**
24	* @Route("/api/validations")
25	*/
26	class ValidationsController extends AbstractController
27	{
28	public function __construct(
29	private ValidationRepository $repository,
30	private SerializerInterface $serializer,
31	private ValidationsStorage $storage,
32	private ValidatorArgumentsService $valArgsService,
33	private MimeTypeGuesserService $mimeTypeGuesserService,
34	private LoggerInterface $logger
35	) {
36
37	}	22✔
38
39	/**
40	* @Route(
41	* "/",
42	* name="validator_api_disabled_routes",
43	* methods={"GET","DELETE","PATCH","PUT"}
44	* )
45	*/
46	public function disabledRoutes()
47	{
48	return new JsonResponse(['error' => "This route is not allowed"], Response::HTTP_METHOD_NOT_ALLOWED);	1✔
49	}
50
51	/**
52	* @Route(
53	* "/{uid}",
54	* name="validator_api_get_validation",
55	* methods={"GET"}
56	* )
57	*/
58	public function getValidation($uid)
59	{
60	$validation = $this->repository->findOneByUid($uid);	2✔
61	if (!$validation) {	2✔
62	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
63	}
64
65	return new JsonResponse($this->serializer->toArray($validation), Response::HTTP_OK);	1✔
66	}
67
68	/**
69	* @Route(
70	* "/{uid}/logs",
71	* name="validator_api_read_logs",
72	* methods={"GET"}
73	* )
74	*/
75	public function readConsole($uid)
76	{
77	$validation = $this->repository->findOneByUid($uid);	×
78	if (!$validation) {	×
79	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	×
80	}
81
82	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	×
83	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	×
84	}
85
86	$outputDirectory = $this->storage->getOutputDirectory($validation);	×
87	$filepath = $outputDirectory . '/validator-debug.log';	×
88
NEW 89	$content = $this->storage->getStorage()->read($filepath);	×
90
91	return new Response(	×
92	$content,	×
93	Response::HTTP_CREATED	×
94	);	×
95	}
96
97
98	/**
99	* @Route(
100	* "/{uid}/results.csv",
101	* name="validator_api_get_validation_csv",
102	* methods={"GET"}
103	* )
104	*/
105	public function getValidationCsv($uid, CsvReportWriter $csvWriter)
106	{
107	$validation = $this->repository->findOneByUid($uid);	×
108	if (!$validation) {	×
109	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	×
110	}
111
112	$response = new StreamedResponse(function () use ($validation, $csvWriter) {	×
113	$csvWriter->write($validation);	×
114	});	×
115	$response->headers->set('Content-Type', 'application/force-download');	×
116	$filename = $uid . '-results.csv';	×
117	$response->headers->set('Content-Disposition', 'attachment; filename="' . $filename . '"');	×
118
119	return $response;	×
120	}
121
122	/**
123	* @Route(
124	* "/",
125	* name="validator_api_upload_dataset",
126	* methods={"POST"}
127	* )
128	*/
129	public function uploadDataset(Request $request)
130	{
131	$files = $request->files;	4✔
132	/*
133	* Ensure that input file is submitted
134	*/
135	$file = $files->get('dataset');	4✔
136	if (!$file) {	4✔
137	throw new ApiException("Argument [dataset] is missing", Response::HTTP_BAD_REQUEST);	2✔
138	}
139
140	$this->logger->info('handle new upload...',[	2✔
141	'path_name' => $file->getPathName(),	2✔
142	'client_original_name' => $file->getClientOriginalName()	2✔
143	]);	2✔
144
145	/*
146	* Ensure that input file is a ZIP file.
147	*/
148	$mimeType = $this->mimeTypeGuesserService->guessMimeType($file->getPathName());	2✔
149	if ($mimeType !== 'application/zip') {	2✔
150	throw new ApiException("Dataset must be in a compressed [.zip] file", Response::HTTP_BAD_REQUEST);	1✔
151	}
152
153	/*
154	* create validation and same validation
155	*/
156	$validation = new Validation();	1✔
157	// TODO : check getClientOriginalName
158	$datasetName = str_replace('.zip', '', $file->getClientOriginalName());	1✔
159	$validation->setDatasetName($datasetName);	1✔
160
161	// Save file to storage
162	$uploadDirectory = $this->storage->getUploadDirectory($validation);	1✔
163	if (!$this->storage->getStorage()->directoryExists($uploadDirectory)) {	1✔
164	$this->storage->getStorage()->createDirectory($uploadDirectory);	1✔
165	}
166	$fileLocation = $uploadDirectory . $validation->getDatasetName() . '.zip';	1✔
167	if ($this->storage->getStorage()->fileExists($fileLocation)) {	1✔
NEW 168	$this->storage->getStorage()->delete($fileLocation);	×
169	}
170	$stream = fopen($file->getRealPath(), 'r+');	1✔
171	$this->storage->getStorage()->writeStream($fileLocation, $stream);	1✔
172	fclose($stream);	1✔
173
174	$fs = new Filesystem;	1✔
175	if ($fs->exists($file->getRealPath())) {	1✔
176	$this->logger->debug('Validation[{uid}] : rm -rf {path}...', [	1✔
177	'uid' => $validation->getUid(),	1✔
178	'path' => $file->getRealPath(),	1✔
179	]);	1✔
180	$fs->remove($file->getRealPath());	1✔
181	}
182
183	$em = $this->getDoctrine()->getManager();	1✔
184	$em->persist($validation);	1✔
185	$em->flush();	1✔
186	$em->refresh($validation);	1✔
187
188	return new JsonResponse(	1✔
189	$this->serializer->toArray($validation),	1✔
190	Response::HTTP_CREATED	1✔
191	);	1✔
192	}
193
194	/**
195	* @Route(
196	* "/{uid}",
197	* name="validator_api_update_arguments",
198	* methods={"PATCH"}
199	* )
200	*/
201	public function updateArguments(Request $request, $uid)
202	{
203	$data = $request->getContent();	10✔
204
205	if (!json_decode($data, true)) {	10✔
206	throw new ApiException("Request body must be a valid JSON string", Response::HTTP_BAD_REQUEST);	2✔
207	}
208
209	$validation = $this->repository->findOneByUid($uid);	8✔
210	if (!$validation) {	8✔
211	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
212
213	}
214
215	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	7✔
216	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	1✔
217	}
218	// TODO : review (json_decode in this method and inside of validate)
219	$arguments = $this->valArgsService->validate($data);	6✔
220
221	$validation->reset();	1✔
222	$validation->setArguments($arguments);	1✔
223	$validation->setStatus(Validation::STATUS_PENDING);	1✔
224
225	$em = $this->getDoctrine()->getManager();	1✔
226	$em->flush();	1✔
227	$em->refresh($validation);	1✔
228
229	return new JsonResponse(	1✔
230	$this->serializer->toArray($validation),	1✔
231	Response::HTTP_OK	1✔
232	);	1✔
233	}
234
235	/**
236	* @Route(
237	* "/{uid}",
238	* name="validator_api_delete_validation",
239	* methods={"DELETE"}
240	* )
241	*/
242	public function deleteValidation($uid)
243	{
244	$validation = $this->repository->findOneByUid($uid);	1✔
245	if (!$validation) {	1✔
246	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
247	}
248
249	$this->logger->info('Validation[{uid}] : removing all saved data...', [	1✔
250	'uid' => $validation->getUid(),	1✔
251	'datasetName' => $validation->getDatasetName(),	1✔
252	]);	1✔
253
254	$em = $this->getDoctrine()->getManager();	1✔
255	$em->remove($validation);	1✔
256	$em->flush();	1✔
257
258	// Delete from storage
259	$uploadDirectory = $this->storage->getUploadDirectory($validation);	1✔
260	if ($this->storage->getStorage()->directoryExists($uploadDirectory)) {	1✔
NEW 261	$this->storage->getStorage()->deleteDirectory($uploadDirectory);	×
262	}
263	$outputDirectory = $this->storage->getOutputDirectory($validation);	1✔
264	if ($this->storage->getStorage()->directoryExists($outputDirectory)) {	1✔
NEW 265	$this->storage->getStorage()->deleteDirectory($outputDirectory);	×
266	}
267
268	return new JsonResponse(null, Response::HTTP_NO_CONTENT);	1✔
269	}
270
271	/**
272	* @Route(
273	* "/{uid}/files/normalized",
274	* name="validator_api_download_normalized_data",
275	* methods={"GET"}
276	* )
277	*/
278	public function downloadNormalizedData($uid)
279	{
280	$validation = $this->repository->findOneByUid($uid);	2✔
281	if (!$validation) {	2✔
282	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
283	}
284
285	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	1✔
286	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	1✔
287	}
288
289	if ($validation->getStatus() == Validation::STATUS_ERROR) {	1✔
290	throw new ApiException("Validation failed, no normalized data", Response::HTTP_FORBIDDEN);	×
291	}
292
293	if (in_array($validation->getStatus(), [Validation::STATUS_PENDING, Validation::STATUS_PROCESSING, Validation::STATUS_WAITING_ARGS])) {	1✔
294	throw new ApiException("Validation hasn't been executed yet", Response::HTTP_FORBIDDEN);	1✔
295	}
296
297	$outputDirectory = $this->storage->getOutputDirectory($validation);	×
298	$zipFilepath = $outputDirectory . $validation->getDatasetName() . '.zip';	×
299	return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-normalized.zip");	×
300	}
301
302	/**
303	* @Route(
304	* "/{uid}/files/source",
305	* name="validator_api_download_source_data",
306	* methods={"GET"}
307	* )
308	*/
309	public function downloadSourceData($uid)
310	{
311	$validation = $this->repository->findOneByUid($uid);	2✔
312	if (!$validation) {	2✔
313	throw new ApiException("No record found for uid=$uid", Response::HTTP_NOT_FOUND);	1✔
314	}
315
316	if ($validation->getStatus() == Validation::STATUS_ARCHIVED) {	1✔
317	throw new ApiException("Validation has been archived", Response::HTTP_FORBIDDEN);	1✔
318	}
319
320	$uploadDirectory = $this->storage->getUploadDirectory($validation);	×
321	$zipFilepath = $uploadDirectory . $validation->getDatasetName() . '.zip';	×
322	return $this->getDownloadResponse($zipFilepath, $validation->getDatasetName() . "-source.zip");	×
323	}
324
325	/**
326	* Returns binary response of the specified file
327	*
328	* @param string $dirpath
329	* @param string $filename
330	* @return StreamedResponse
331	*/
332	private function getDownloadResponse($filepath, $filename)
333	{
NEW 334	if (!$this->storage->getStorage()->has($filepath)) {	×
335	throw new ApiException("Requested files not found for this validation", Response::HTTP_FORBIDDEN);	×
336	}
337
NEW 338	$stream = $this->storage->getStorage()->readStream($filepath);	×
339
340	return new StreamedResponse(function () use ($stream) {	×
341	fpassthru($stream);	×
342	exit();	×
343	}, 200, [	×
344	'Content-Transfer-Encoding',	×
345	'binary',	×
346	'Content-Type' => 'application/zip',	×
347	'Content-Disposition' => sprintf('attachment; filename="%s"', $filename),	×
348	'Content-Length' => fstat($stream)['size']	×
349	]);	×
350	}
351	}

IGNF / validator-api / 15188458321

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous