• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

OSGeo / gdal / 12706066811

10 Jan 2025 08:38AM UTC coverage: 70.084% (-2.5%) from 72.549%
12706066811

Pull #11629

github

web-flow
Merge 9418dc48f into 0df468c56
Pull Request #11629: add uv documentation for python package

563296 of 803749 relevant lines covered (70.08%)

223434.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.37
/port/cpl_vsi_virtual.h
1
/******************************************************************************
2
 *
3
 * Project:  VSI Virtual File System
4
 * Purpose:  Declarations for classes related to the virtual filesystem.
5
 *           These would only be normally required by applications implementing
6
 *           their own virtual file system classes which should be rare.
7
 *           The class interface may be fragile through versions.
8
 * Author:   Frank Warmerdam, warmerdam@pobox.com
9
 *
10
 ******************************************************************************
11
 * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
12
 * Copyright (c) 2010-2014, Even Rouault <even dot rouault at spatialys.com>
13
 *
14
 * SPDX-License-Identifier: MIT
15
 ****************************************************************************/
16

17
#ifndef CPL_VSI_VIRTUAL_H_INCLUDED
18
#define CPL_VSI_VIRTUAL_H_INCLUDED
19

20
#include "cpl_vsi.h"
21
#include "cpl_vsi_error.h"
22
#include "cpl_string.h"
23
#include "cpl_multiproc.h"
24

25
#include <map>
26
#include <memory>
27
#include <vector>
28
#include <string>
29

30
// To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
31
#ifdef GetDiskFreeSpace
32
#undef GetDiskFreeSpace
33
#endif
34

35
// To avoid aliasing to CopyFile to CopyFileA on Windows
36
#ifdef CopyFile
37
#undef CopyFile
38
#endif
39

40
/************************************************************************/
41
/*                           VSIVirtualHandle                           */
42
/************************************************************************/
43

44
/** Virtual file handle */
45
struct CPL_DLL VSIVirtualHandle
46
{
47
  public:
48
    virtual int Seek(vsi_l_offset nOffset, int nWhence) = 0;
49
    virtual vsi_l_offset Tell() = 0;
50
    virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount) = 0;
51
    virtual int ReadMultiRange(int nRanges, void **ppData,
52
                               const vsi_l_offset *panOffsets,
53
                               const size_t *panSizes);
54

55
    /** This method is called when code plans to access soon one or several
56
     * ranges in a file. Some file systems may be able to use this hint to
57
     * for example asynchronously start such requests.
58
     *
59
     * Offsets may be given in a non-increasing order, and may potentially
60
     * overlap.
61
     *
62
     * @param nRanges Size of the panOffsets and panSizes arrays.
63
     * @param panOffsets Array containing the start offset of each range.
64
     * @param panSizes Array containing the size (in bytes) of each range.
65
     * @since GDAL 3.7
66
     */
67
    virtual void AdviseRead(CPL_UNUSED int nRanges,
90✔
68
                            CPL_UNUSED const vsi_l_offset *panOffsets,
69
                            CPL_UNUSED const size_t *panSizes)
70
    {
71
    }
90✔
72

73
    /** Return the total maximum number of bytes that AdviseRead() can handle
74
     * at once.
75
     *
76
     * Some AdviseRead() implementations may give up if the sum of the values
77
     * in the panSizes[] array provided to AdviseRead() exceeds a limit.
78
     *
79
     * Callers might use that threshold to optimize the efficiency of
80
     * AdviseRead().
81
     *
82
     * A returned value of 0 indicates a unknown limit.
83
     * @since GDAL 3.9
84
     */
85
    virtual size_t GetAdviseReadTotalBytesLimit() const
217✔
86
    {
87
        return 0;
217✔
88
    }
89

90
    virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount) = 0;
91

92
    int Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
93
        CPL_PRINT_FUNC_FORMAT(2, 3);
94

95
    virtual void ClearErr() = 0;
96

97
    virtual int Eof() = 0;
98

99
    virtual int Error() = 0;
100

101
    virtual int Flush()
34,984✔
102
    {
103
        return 0;
34,984✔
104
    }
105

106
    virtual int Close() = 0;
107
    // Base implementation that only supports file extension.
108
    virtual int Truncate(vsi_l_offset nNewSize);
109

110
    virtual void *GetNativeFileDescriptor()
9✔
111
    {
112
        return nullptr;
9✔
113
    }
114

115
    virtual VSIRangeStatus GetRangeStatus(CPL_UNUSED vsi_l_offset nOffset,
149✔
116
                                          CPL_UNUSED vsi_l_offset nLength)
117
    {
118
        return VSI_RANGE_STATUS_UNKNOWN;
149✔
119
    }
120

121
    virtual bool HasPRead() const;
122
    virtual size_t PRead(void *pBuffer, size_t nSize,
123
                         vsi_l_offset nOffset) const;
124

125
    /** Ask current operations to be interrupted.
126
     * Implementations must be thread-safe, as this will typically be called
127
     * from another thread than the active one for this file.
128
     */
129
    virtual void Interrupt()
×
130
    {
131
    }
×
132

133
    // NOTE: when adding new methods, besides the "actual" implementations,
134
    // also consider the VSICachedFile one.
135

136
    virtual ~VSIVirtualHandle()
300,893✔
137
    {
300,893✔
138
    }
300,893✔
139
};
140

141
/************************************************************************/
142
/*                        VSIVirtualHandleCloser                        */
143
/************************************************************************/
144

145
/** Helper close to use with a std:unique_ptr<VSIVirtualHandle>,
146
 *  such as VSIVirtualHandleUniquePtr. */
147
struct VSIVirtualHandleCloser
148
{
149
    /** Operator () that closes and deletes the file handle. */
150
    void operator()(VSIVirtualHandle *poHandle)
5,506✔
151
    {
152
        if (poHandle)
5,506✔
153
        {
154
            poHandle->Close();
5,488✔
155
            delete poHandle;
5,488✔
156
        }
157
    }
5,506✔
158
};
159

160
/** Unique pointer of VSIVirtualHandle that calls the Close() method */
161
typedef std::unique_ptr<VSIVirtualHandle, VSIVirtualHandleCloser>
162
    VSIVirtualHandleUniquePtr;
163

164
/************************************************************************/
165
/*                         VSIFilesystemHandler                         */
166
/************************************************************************/
167

168
#ifndef DOXYGEN_SKIP
169
class CPL_DLL VSIFilesystemHandler
170
{
171

172
  public:
173
    virtual ~VSIFilesystemHandler()
25,382✔
174
    {
25,382✔
175
    }
25,382✔
176

177
    VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess);
178

179
    virtual VSIVirtualHandle *Open(const char *pszFilename,
180
                                   const char *pszAccess, bool bSetError,
181
                                   CSLConstList papszOptions) = 0;
182
    virtual int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
183
                     int nFlags) = 0;
184

185
    virtual int Unlink(const char *pszFilename)
×
186
    {
187
        (void)pszFilename;
188
        errno = ENOENT;
×
189
        return -1;
×
190
    }
191

192
    virtual int *UnlinkBatch(CSLConstList papszFiles);
193

194
    virtual int Mkdir(const char *pszDirname, long nMode)
×
195
    {
196
        (void)pszDirname;
197
        (void)nMode;
198
        errno = ENOENT;
×
199
        return -1;
×
200
    }
201

202
    virtual int Rmdir(const char *pszDirname)
×
203
    {
204
        (void)pszDirname;
205
        errno = ENOENT;
×
206
        return -1;
×
207
    }
208

209
    virtual int RmdirRecursive(const char *pszDirname);
210

211
    char **ReadDir(const char *pszDirname)
4✔
212
    {
213
        return ReadDirEx(pszDirname, 0);
4✔
214
    }
215

216
    virtual char **ReadDirEx(const char * /*pszDirname*/, int /* nMaxFiles */)
3✔
217
    {
218
        return nullptr;
3✔
219
    }
220

221
    virtual char **SiblingFiles(const char * /*pszFilename*/)
71,850✔
222
    {
223
        return nullptr;
71,850✔
224
    }
225

226
    virtual int Rename(const char *oldpath, const char *newpath)
×
227
    {
228
        (void)oldpath;
229
        (void)newpath;
230
        errno = ENOENT;
×
231
        return -1;
×
232
    }
233

234
    virtual int IsCaseSensitive(const char *pszFilename)
26,042✔
235
    {
236
        (void)pszFilename;
237
        return TRUE;
26,042✔
238
    }
239

240
    virtual GIntBig GetDiskFreeSpace(const char * /* pszDirname */)
×
241
    {
242
        return -1;
×
243
    }
244

245
    virtual int SupportsSparseFiles(const char * /* pszPath */)
×
246
    {
247
        return FALSE;
×
248
    }
249

250
    virtual int HasOptimizedReadMultiRange(const char * /* pszPath */)
12,499✔
251
    {
252
        return FALSE;
12,499✔
253
    }
254

255
    virtual const char *GetActualURL(const char * /*pszFilename*/)
1✔
256
    {
257
        return nullptr;
1✔
258
    }
259

260
    virtual const char *GetOptions()
11✔
261
    {
262
        return nullptr;
11✔
263
    }
264

265
    virtual char *GetSignedURL(const char * /*pszFilename*/,
1✔
266
                               CSLConstList /* papszOptions */)
267
    {
268
        return nullptr;
1✔
269
    }
270

271
    virtual bool Sync(const char *pszSource, const char *pszTarget,
272
                      const char *const *papszOptions,
273
                      GDALProgressFunc pProgressFunc, void *pProgressData,
274
                      char ***ppapszOutputs);
275

276
    virtual int CopyFile(const char *pszSource, const char *pszTarget,
277
                         VSILFILE *fpSource, vsi_l_offset nSourceSize,
278
                         const char *const *papszOptions,
279
                         GDALProgressFunc pProgressFunc, void *pProgressData);
280

281
    virtual int
282
    CopyFileRestartable(const char *pszSource, const char *pszTarget,
283
                        const char *pszInputPayload, char **ppszOutputPayload,
284
                        CSLConstList papszOptions,
285
                        GDALProgressFunc pProgressFunc, void *pProgressData);
286

287
    virtual VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
288
                            const char *const *papszOptions);
289

290
    virtual char **GetFileMetadata(const char *pszFilename,
291
                                   const char *pszDomain,
292
                                   CSLConstList papszOptions);
293

294
    virtual bool SetFileMetadata(const char *pszFilename,
295
                                 CSLConstList papszMetadata,
296
                                 const char *pszDomain,
297
                                 CSLConstList papszOptions);
298

299
    virtual bool
300
    MultipartUploadGetCapabilities(int *pbNonSequentialUploadSupported,
301
                                   int *pbParallelUploadSupported,
302
                                   int *pbAbortSupported, size_t *pnMinPartSize,
303
                                   size_t *pnMaxPartSize, int *pnMaxPartCount);
304

305
    virtual char *MultipartUploadStart(const char *pszFilename,
306
                                       CSLConstList papszOptions);
307

308
    virtual char *MultipartUploadAddPart(const char *pszFilename,
309
                                         const char *pszUploadId,
310
                                         int nPartNumber,
311
                                         vsi_l_offset nFileOffset,
312
                                         const void *pData, size_t nDataLength,
313
                                         CSLConstList papszOptions);
314

315
    virtual bool
316
    MultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
317
                       size_t nPartIdsCount, const char *const *apszPartIds,
318
                       vsi_l_offset nTotalSize, CSLConstList papszOptions);
319

320
    virtual bool MultipartUploadAbort(const char *pszFilename,
321
                                      const char *pszUploadId,
322
                                      CSLConstList papszOptions);
323

324
    virtual bool AbortPendingUploads(const char * /*pszFilename*/)
×
325
    {
326
        return true;
×
327
    }
328

329
    virtual std::string
330
    GetStreamingFilename(const std::string &osFilename) const
26,568✔
331
    {
332
        return osFilename;
26,568✔
333
    }
334

335
    virtual std::string
336
    GetNonStreamingFilename(const std::string &osFilename) const
1,418✔
337
    {
338
        return osFilename;
1,418✔
339
    }
340

341
    /** Return the canonical filename.
342
     *
343
     * May be implemented by case-insensitive filesystems
344
     * (currently Win32 and MacOSX)
345
     * to return the filename with its actual case (i.e. the one that would
346
     * be used when listing the content of the directory).
347
     */
348
    virtual std::string
349
    GetCanonicalFilename(const std::string &osFilename) const
248✔
350
    {
351
        return osFilename;
248✔
352
    }
353

354
    virtual bool IsLocal(const char * /* pszPath */)
99✔
355
    {
356
        return true;
99✔
357
    }
358

359
    virtual bool SupportsSequentialWrite(const char * /* pszPath */,
44✔
360
                                         bool /* bAllowLocalTempFile */)
361
    {
362
        return true;
44✔
363
    }
364

365
    virtual bool SupportsRandomWrite(const char * /* pszPath */,
286✔
366
                                     bool /* bAllowLocalTempFile */)
367
    {
368
        return true;
286✔
369
    }
370

371
    virtual bool SupportsRead(const char * /* pszPath */)
43✔
372
    {
373
        return true;
43✔
374
    }
375

376
    virtual VSIFilesystemHandler *Duplicate(const char * /* pszPrefix */)
2✔
377
    {
378
        CPLError(CE_Failure, CPLE_NotSupported,
2✔
379
                 "Duplicate() not supported on this file system");
380
        return nullptr;
2✔
381
    }
382

383
    /** Return the directory separator.
384
     *
385
     * Default is forward slash. The only exception currently is the Windows
386
     * file system which returns anti-slash, unless the specified path is of the
387
     * form "{drive_letter}:/{rest_of_the_path}".
388
     */
389
    virtual const char *GetDirectorySeparator(CPL_UNUSED const char *pszPath)
877,031✔
390
    {
391
        return "/";
877,031✔
392
    }
393
};
394
#endif /* #ifndef DOXYGEN_SKIP */
395

396
/************************************************************************/
397
/*                            VSIFileManager                            */
398
/************************************************************************/
399

400
#ifndef DOXYGEN_SKIP
401
class CPL_DLL VSIFileManager
402
{
403
  private:
404
    VSIFilesystemHandler *poDefaultHandler = nullptr;
405
    std::map<std::string, VSIFilesystemHandler *> oHandlers{};
406

407
    VSIFileManager();
408

409
    static VSIFileManager *Get();
410

411
    CPL_DISALLOW_COPY_ASSIGN(VSIFileManager)
412

413
  public:
414
    ~VSIFileManager();
415

416
    static VSIFilesystemHandler *GetHandler(const char *);
417
    static void InstallHandler(const std::string &osPrefix,
418
                               VSIFilesystemHandler *);
419
    static void RemoveHandler(const std::string &osPrefix);
420

421
    static char **GetPrefixes();
422
};
423
#endif /* #ifndef DOXYGEN_SKIP */
424

425
/************************************************************************/
426
/* ==================================================================== */
427
/*                       VSIArchiveFilesystemHandler                   */
428
/* ==================================================================== */
429
/************************************************************************/
430

431
#ifndef DOXYGEN_SKIP
432

433
class VSIArchiveEntryFileOffset
434
{
435
  public:
436
    virtual ~VSIArchiveEntryFileOffset();
437
};
438

439
typedef struct
440
{
441
    char *fileName;
442
    vsi_l_offset uncompressed_size;
443
    VSIArchiveEntryFileOffset *file_pos;
444
    int bIsDir;
445
    GIntBig nModifiedTime;
446
} VSIArchiveEntry;
447

448
class VSIArchiveContent
449
{
450
  public:
451
    time_t mTime = 0;
452
    vsi_l_offset nFileSize = 0;
453
    int nEntries = 0;
454
    VSIArchiveEntry *entries = nullptr;
455

456
    ~VSIArchiveContent();
457
};
458

459
class VSIArchiveReader
460
{
461
  public:
462
    virtual ~VSIArchiveReader();
463

464
    virtual int GotoFirstFile() = 0;
465
    virtual int GotoNextFile() = 0;
466
    virtual VSIArchiveEntryFileOffset *GetFileOffset() = 0;
467
    virtual GUIntBig GetFileSize() = 0;
468
    virtual CPLString GetFileName() = 0;
469
    virtual GIntBig GetModifiedTime() = 0;
470
    virtual int GotoFileOffset(VSIArchiveEntryFileOffset *pOffset) = 0;
471
};
472

473
class VSIArchiveFilesystemHandler : public VSIFilesystemHandler
474
{
475
    CPL_DISALLOW_COPY_ASSIGN(VSIArchiveFilesystemHandler)
476

477
  protected:
478
    CPLMutex *hMutex = nullptr;
479
    /* We use a cache that contains the list of files contained in a VSIArchive
480
     * file as */
481
    /* unarchive.c is quite inefficient in listing them. This speeds up access
482
     * to VSIArchive files */
483
    /* containing ~1000 files like a CADRG product */
484
    std::map<CPLString, VSIArchiveContent *> oFileList{};
485

486
    virtual const char *GetPrefix() = 0;
487
    virtual std::vector<CPLString> GetExtensions() = 0;
488
    virtual VSIArchiveReader *CreateReader(const char *pszArchiveFileName) = 0;
489

490
  public:
491
    VSIArchiveFilesystemHandler();
492
    virtual ~VSIArchiveFilesystemHandler();
493

494
    int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
495
             int nFlags) override;
496
    int Unlink(const char *pszFilename) override;
497
    int Rename(const char *oldpath, const char *newpath) override;
498
    int Mkdir(const char *pszDirname, long nMode) override;
499
    int Rmdir(const char *pszDirname) override;
500
    char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
501

502
    virtual const VSIArchiveContent *
503
    GetContentOfArchive(const char *archiveFilename,
504
                        VSIArchiveReader *poReader = nullptr);
505
    virtual char *SplitFilename(const char *pszFilename,
506
                                CPLString &osFileInArchive,
507
                                int bCheckMainFileExists);
508
    virtual VSIArchiveReader *OpenArchiveFile(const char *archiveFilename,
509
                                              const char *fileInArchiveName);
510
    virtual int FindFileInArchive(const char *archiveFilename,
511
                                  const char *fileInArchiveName,
512
                                  const VSIArchiveEntry **archiveEntry);
513

514
    virtual bool IsLocal(const char *pszPath) override;
515

516
    virtual bool
517
    SupportsSequentialWrite(const char * /* pszPath */,
×
518
                            bool /* bAllowLocalTempFile */) override
519
    {
520
        return false;
×
521
    }
522

523
    virtual bool SupportsRandomWrite(const char * /* pszPath */,
×
524
                                     bool /* bAllowLocalTempFile */) override
525
    {
526
        return false;
×
527
    }
528
};
529

530
/************************************************************************/
531
/*                              VSIDIR                                  */
532
/************************************************************************/
533

534
struct CPL_DLL VSIDIR
535
{
536
    VSIDIR() = default;
2,828✔
537
    virtual ~VSIDIR();
538

539
    virtual const VSIDIREntry *NextDirEntry() = 0;
540

541
  private:
542
    VSIDIR(const VSIDIR &) = delete;
543
    VSIDIR &operator=(const VSIDIR &) = delete;
544
};
545

546
#endif /* #ifndef DOXYGEN_SKIP */
547

548
VSIVirtualHandle CPL_DLL *
549
VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
550
VSIVirtualHandle *
551
VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
552
                              const GByte *pabyBeginningContent,
553
                              vsi_l_offset nCheatFileSize);
554
constexpr int VSI_CACHED_DEFAULT_CHUNK_SIZE = 32768;
555
VSIVirtualHandle CPL_DLL *
556
VSICreateCachedFile(VSIVirtualHandle *poBaseHandle,
557
                    size_t nChunkSize = VSI_CACHED_DEFAULT_CHUNK_SIZE,
558
                    size_t nCacheSize = 0);
559

560
const int CPL_DEFLATE_TYPE_GZIP = 0;
561
const int CPL_DEFLATE_TYPE_ZLIB = 1;
562
const int CPL_DEFLATE_TYPE_RAW_DEFLATE = 2;
563
VSIVirtualHandle CPL_DLL *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
564
                                                int nDeflateType,
565
                                                int bAutoCloseBaseHandle);
566

567
VSIVirtualHandle *VSICreateGZipWritable(VSIVirtualHandle *poBaseHandle,
568
                                        int nDeflateType,
569
                                        bool bAutoCloseBaseHandle, int nThreads,
570
                                        size_t nChunkSize,
571
                                        size_t nSOZIPIndexEltSize,
572
                                        std::vector<uint8_t> *panSOZIPIndex);
573

574
VSIVirtualHandle *
575
VSICreateUploadOnCloseFile(VSIVirtualHandleUniquePtr &&poWritableHandle,
576
                           VSIVirtualHandleUniquePtr &&poTmpFile,
577
                           const std::string &osTmpFilename);
578

579
#endif /* ndef CPL_VSI_VIRTUAL_H_INCLUDED */
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc