• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

PredatorCZ / PreCore / 536

22 Sep 2024 12:23PM UTC coverage: 52.238% (-3.0%) from 55.265%
536

push

github

PredatorCZ
remove old xml serializers

4084 of 7818 relevant lines covered (52.24%)

10609.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

9.95
/src/app/in_context.cpp
1
/*  Spike is universal dedicated module handler
2
    This source contains context for input data
3

4
    Copyright 2021-2023 Lukas Cone
5

6
    Licensed under the Apache License, Version 2.0 (the "License");
7
    you may not use this file except in compliance with the License.
8
    You may obtain a copy of the License at
9

10
        http://www.apache.org/licenses/LICENSE-2.0
11

12
    Unless required by applicable law or agreed to in writing, software
13
    distributed under the License is distributed on an "AS IS" BASIS,
14
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
    See the License for the specific language governing permissions and
16
    limitations under the License.
17
*/
18

19
#include "spike/app/context.hpp"
20
#include "spike/app/out_context.hpp"
21
#include "spike/app/texel.hpp"
22
#include "spike/app/tmp_storage.hpp"
23
#include "spike/format/ZIP_istream.inl"
24
#include "spike/io/binreader.hpp"
25
#include "spike/io/binwritter.hpp"
26
#include "spike/io/directory_scanner.hpp"
27
#include "spike/io/fileinfo.hpp"
28
#include "spike/io/stat.hpp"
29
#include "spike/master_printer.hpp"
30
#include <list>
31
#include <mutex>
32
#include <optional>
33
#include <spanstream>
34

35
static std::mutex simpleIOLock;
36

37
const std::vector<std::string> &AppContextShare::SupplementalFiles() {
×
38
  if (!supplementals) {
×
39
    throw std::runtime_error(
×
40
        "Invalid call of SupplementalFiles, module is not for batch.");
×
41
  }
42

43
  return supplementals.value();
×
44
}
45

46
const std::vector<std::string> &ZIPIOContext::SupplementalFiles() {
×
47
  if (!supplementals) {
×
48
    throw std::runtime_error(
×
49
        "Invalid call of SupplementalFiles, module is not for batch.");
×
50
  }
51

52
  return supplementals.value();
×
53
}
54

55
struct AppContextShareImpl : AppContextShare {
56
  NewTexelContext *NewImage(NewTexelContextCreate ctx,
56✔
57
                            const std::string *path) override {
58
    if (texelContext) {
56✔
59
      texelContext->Finish();
28✔
60
    }
61

62
    texelContext = CreateTexelContext(ctx, this);
56✔
63

64
    if (path) {
56✔
65
      texelContext->pathOverride.Load(*path);
28✔
66
    }
67

68
    return texelContext.get();
56✔
69
  }
70

71
  std::pair<std::string, size_t> IdealPath(const std::string &path) {
190✔
72
    std::string filePath;
73
    size_t delimeter = 0;
74

75
    if (basePathParts.empty()) {
190✔
76
      filePath = std::string(basePath.GetFullPath()) + path;
380✔
77
      delimeter = basePath.GetFullPath().size();
78
    } else {
79
      std::string pathCopy(path);
80

81
      if (substPath.size() && path.starts_with(substPath)) {
×
82
        pathCopy.erase(0, substPath.size());
×
83
      }
84
      AFileInfo pathInfo(pathCopy);
×
85
      auto exploded = pathInfo.Explode();
×
86
      const size_t numItems = std::min(exploded.size(), basePathParts.size());
×
87

88
      if (basePath.GetFullPath().at(0) == '/') {
×
89
        filePath.push_back('/');
×
90
      }
91

92
      size_t i = 0;
93

94
      for (; i < numItems; i++) {
×
95
        if (basePathParts[i] == exploded[i]) {
×
96
          filePath.append(exploded[i]);
97
          filePath.push_back('/');
×
98
          continue;
99
        }
100

101
        break;
102
      }
103

104
      for (size_t j = i; j < basePathParts.size(); j++) {
×
105
        filePath.append(basePathParts[j]);
106
        filePath.push_back('/');
×
107
      }
108

109
      delimeter = filePath.size();
110

111
      for (; i < exploded.size(); i++) {
×
112
        filePath.append(exploded[i]);
113
        filePath.push_back('/');
×
114
      }
115

116
      filePath.pop_back();
117
    }
118

119
    return {filePath, delimeter};
190✔
120
  }
121

122
  NewFileContext NewFile(const std::string &path) override {
190✔
123
    const auto [filePath, delimeter] = IdealPath(path);
190✔
124

125
    try {
126
      outFile = BinWritter(filePath);
379✔
127
    } catch (const es::FileInvalidAccessError &e) {
1✔
128
      mkdirs(filePath);
1✔
129
      outFile = BinWritter(filePath);
1✔
130
    }
1✔
131
    return {outFile.BaseStream(), filePath, delimeter};
190✔
132
  }
380✔
133

134
  void BaseOutputPath(std::string basePath_) override {
×
135
    if (size_t found = basePath_.find_first_of(':'); found != basePath_.npos) {
×
136
      substPath = basePath_.substr(0, found);
×
137
      basePath_.erase(0, found + 1);
×
138
    }
139

140
    if (basePath_.empty()) {
×
141
      throw std::runtime_error("Output path cannot be empty");
×
142
    }
143

144
    if (basePath_.back() != '/') {
×
145
      basePath_.push_back('/');
×
146
    }
147
    basePath = AFileInfo(basePath_);
×
148
    basePathParts = basePath.Explode();
×
149
  }
150

151
  JenHash Hash() override { return JenHash(FullPath()); }
×
152

153
  std::string FullPath() override {
×
154
    return std::string(basePath.GetFullPath()) +
×
155
           std::string(workingFile.GetFullPath());
×
156
  }
157

158
  ~AppContextShareImpl() {
28✔
159
    if (texelContext) {
28✔
160
      texelContext->Finish();
28✔
161
    }
162
  }
56✔
163

×
164
  BinWritter outFile;
165
  AFileInfo basePath;
166
  std::string substPath;
167
  std::vector<std::string_view> basePathParts;
168
  std::unique_ptr<NewTexelContextImpl> texelContext;
28✔
169
};
28✔
170

28✔
171
struct SimpleIOContext : AppContextShareImpl {
172
  SimpleIOContext(const std::string &path,
56✔
173
                  std::optional<std::vector<std::string>> supplementals_) {
174
    mainFile.Open(path);
175
    workingFile.Load(path);
176
    supplementals = std::move(supplementals_);
177

178
    if (!cliSettings.out.empty()) {
179
      BaseOutputPath(cliSettings.out);
180
    }
181
  }
182
  std::istream *OpenFile(const std::string &path);
28✔
183

924✔
184
  AppContextStream RequestFile(const std::string &path) override;
28✔
185

28✔
186
  AppContextFoundStream FindFile(const std::string &rootFolder,
187
                                 const std::string &pattern) override;
188
  std::istream &GetStream() override;
28✔
189
  std::string GetBuffer(size_t size, size_t begin) override;
×
190

191
  void DisposeFile(std::istream *str) override;
28✔
192

193
  AppExtractContext *ExtractContext(std::string_view name) override {
194
    if (ectx) [[unlikely]] {
195
      return ectx.get();
196
    }
197

198
    std::string arcPath(workingFile.GetFolder());
199
    arcPath.append(name);
200

201
    auto [outPath, delimeter] = IdealPath(arcPath);
202

203
    if (mainSettings.extractSettings.makeZIP) {
×
204
      if (workingFile.GetExtension() == ".zip") {
×
205
        outPath.append("_out");
×
206
      }
207

208
      outPath.append(".zip");
×
209

210
      auto uniq = std::make_unique<ZIPExtactContext>(outPath);
211
      uniq->forEachFile = forEachFile;
×
212
      ectx = std::move(uniq);
213
    } else {
×
214
      if (!mainSettings.extractSettings.folderPerArc) {
×
215
        outPath = workingFile.GetFolder();
×
216
      } else {
217
        es::mkdir(outPath);
218
        outPath.push_back('/');
×
219
      }
220

×
221
      auto uniq = std::make_unique<IOExtractContext>(outPath);
×
222
      uniq->forEachFile = forEachFile;
223
      ectx = std::move(uniq);
224
    }
×
225

226
    return ectx.get();
227
  }
228

×
229
  AppExtractContext *ExtractContext() override {
230
    return ExtractContext(workingFile.GetFilename());
231
  }
×
232

×
233
  void Finish() override {
234
    if (ectx && mainSettings.extractSettings.makeZIP) {
235
      static_cast<ZIPExtactContext *>(ectx.get())->FinishZIP([] {
236
        printinfo("Generating cache.");
237
      });
238
    }
239
  }
×
240

×
241
private:
242
  BinReader mainFile;
243
  BinReader streamedFiles[32];
×
244
  uint32 usedFiles = 0;
×
245
  std::unique_ptr<AppExtractContext> ectx;
246
};
×
247

×
248
std::istream *SimpleIOContext::OpenFile(const std::string &path) {
249
  std::lock_guard<std::mutex> guard(simpleIOLock);
250
  for (size_t b = 0; b < 32; b++) {
251
    uint32 bit = 1 << b;
252
    if (!(usedFiles & bit)) {
253
      streamedFiles[b].Open(path);
254
      usedFiles ^= bit;
255
      return &streamedFiles[b].BaseStream();
256
    }
257
  }
258

×
259
  throw std::out_of_range("Maximum opened files reached!");
260
}
×
261

×
262
AppContextStream SimpleIOContext::RequestFile(const std::string &path) {
×
263
  AFileInfo wFile(workingFile);
×
264
  AFileInfo pFile(path);
×
265
  auto catchedFile = pFile.CatchBranch(wFile.GetFolder());
×
266
  return {OpenFile(catchedFile), this};
267
}
268

269
AppContextFoundStream SimpleIOContext::FindFile(const std::string &rootFolder,
×
270
                                                const std::string &pattern) {
271
  DirectoryScanner sc;
272
  sc.AddFilter(pattern);
×
273
  sc.Scan(rootFolder);
274

×
275
  if (sc.Files().empty()) {
×
276
    throw es::FileNotFoundError(pattern);
×
277
  } else if (sc.Files().size() > 1) {
278
    std::string *winner = nullptr;
279
    size_t minFolder = 0x10000;
×
280
    size_t minLevel = 0x10000;
281

×
282
    for (auto &f : sc) {
×
283
      size_t foundIdx = f.find_last_of('/');
×
284

285
      if (foundIdx == f.npos) {
×
286
        throw std::runtime_error("Too many files found for pattern: " +
×
287
                                 pattern);
×
288
      }
289

290
      if (foundIdx < minFolder) {
291
        winner = &f;
292
        minFolder = foundIdx;
×
293
        minLevel = std::count(f.begin(), f.end(), '/');
294
      } else if (foundIdx == minFolder) {
295
        if (auto clevel = std::count(f.begin(), f.end(), '/');
×
296
            clevel < minLevel) {
×
297
          winner = &f;
×
298
          minFolder = foundIdx;
299
          minLevel = clevel;
300
        } else if (clevel == minLevel) {
×
301
          throw std::runtime_error("Too many files found for pattern: " +
302
                                   pattern);
303
        }
×
304
      }
×
305
    }
×
306

×
307
    if (!winner) {
308
      throw std::runtime_error("Too many files found for pattern: " + pattern);
309
    }
310

×
311
    return {OpenFile(*winner), this, AFileInfo(*winner)};
×
312
  }
×
313

314
  return {OpenFile(sc.Files().front()), this, AFileInfo(sc.Files().front())};
315
}
316

317
void SimpleIOContext::DisposeFile(std::istream *str) {
×
318
  size_t index = 0;
×
319

320
  for (auto &f : streamedFiles) {
321
    if (&f.BaseStream() == str) {
×
322
      uint32 bit = 1 << index;
323

324
      if (!(usedFiles & bit)) {
×
325
        throw std::runtime_error("Stream already freed.");
326
      }
327

×
328
      es::Dispose(f);
329
      usedFiles ^= bit;
330
      return;
×
331
    }
×
332

×
333
    index++;
334
  }
×
335

×
336
  throw std::runtime_error("Requested stream not found!");
337
}
338

×
339
std::istream &SimpleIOContext::GetStream() { return mainFile.BaseStream(); }
×
340

×
341
std::string SimpleIOContext::GetBuffer(size_t size, size_t begin) {
342
  mainFile.Push();
343
  mainFile.Seek(begin);
×
344
  std::string buffer;
345
  mainFile.ReadContainer(
346
      buffer, size == size_t(-1) ? (mainFile.GetSize() - begin) : size);
×
347
  mainFile.Pop();
348

349
  return buffer;
28✔
350
}
351

×
352
std::shared_ptr<AppContextShare>
×
353
MakeIOContext(const std::string &path,
354
              std::optional<std::vector<std::string>> supplementals) {
355
  return std::make_unique<SimpleIOContext>(path, supplementals);
×
356
}
×
357

358
struct ZIPIOContextInstance : AppContextShareImpl {
359
  ZIPIOContextInstance(const ZIPIOContextInstance &) = delete;
×
360
  ZIPIOContextInstance(ZIPIOContextInstance &&) = delete;
361
  ZIPIOContextInstance(ZIPIOContext *base_, ZIPIOEntry entry_)
362
      : base(base_), entry(entry_) {
363
    workingFile.Load(entry_.AsView());
28✔
364
    BaseOutputPath(base->basePath);
365
  }
56✔
366

367
  ~ZIPIOContextInstance() {
368
    if (stream) {
369
      base->DisposeFile(stream);
370
    }
371
  }
×
372

×
373
  AppContextStream RequestFile(const std::string &path) override {
×
374
    return base->RequestFile(path);
×
375
  }
376
  void DisposeFile(std::istream *file) override { base->DisposeFile(file); }
377
  AppContextFoundStream FindFile(const std::string &rootFolder,
×
378
                                 const std::string &pattern) override {
×
379
    return base->FindFile(rootFolder, pattern);
×
380
  }
381

382
  std::istream &GetStream() override {
×
383
    if (!stream) {
384
      stream = base->OpenFile(entry);
385
    }
386
    return *stream;
387
  }
×
388
  std::string GetBuffer(size_t size, size_t begin) override {
×
389
    BinReaderRef rd(GetStream());
×
390
    rd.Push();
391
    rd.Seek(begin);
392
    std::string buffer;
393
    rd.ReadContainer(buffer,
×
394
                     size == size_t(-1) ? (rd.GetSize() - begin) : size);
×
395
    rd.Pop();
396
    return buffer;
×
397
  }
×
398

399
  AppExtractContext *ExtractContext(std::string_view name) override {
×
400
    if (ectx) [[unlikely]] {
401
      return ectx.get();
402
    }
×
403

×
404
    if (mainSettings.extractSettings.makeZIP) {
×
405
      base->InitMerger();
406
      entriesPath = RequestTempFile();
×
407
      auto uniq = std::make_unique<ZIPExtactContext>(entriesPath, false);
408
      if (mainSettings.extractSettings.folderPerArc) {
×
409
        uniq->prefixPath = workingFile.GetFolder();
×
410
        uniq->prefixPath += name;
411
        uniq->prefixPath.push_back('/');
412
      }
413
      uniq->forEachFile = forEachFile;
×
414
      ectx = std::move(uniq);
×
415
    } else {
416
      std::string outPath;
×
417
      if (!mainSettings.extractSettings.folderPerArc) {
418
        outPath = workingFile.GetFolder();
419
      } else {
×
420
        outPath = std::string(basePath.GetFullPath());
×
421
        outPath += workingFile.GetFolder();
×
422
        outPath += name;
423
        mkdirs(outPath);
424
        outPath.push_back('/');
×
425
      }
×
426

×
427
      auto uniq = std::make_unique<IOExtractContext>(outPath);
×
428
      uniq->forEachFile = forEachFile;
×
429
      ectx = std::move(uniq);
×
430
    }
431

×
432
    return ectx.get();
433
  }
×
434

435
  AppExtractContext *ExtractContext() override {
436
    return ExtractContext(workingFile.GetFilename());
437
  }
×
438

439
  void Finish() override {
440
    if (ectx && mainSettings.extractSettings.makeZIP) {
×
441
      base->Merge(static_cast<ZIPExtactContext *>(ectx.get()), entriesPath);
442
      es::Dispose(ectx);
443
      es::RemoveFile(entriesPath);
×
444
    }
×
445
  }
446

447
  ZIPIOContext *base;
×
448
  std::istream *stream = nullptr;
×
449
  ZipEntry entry;
450
  std::unique_ptr<AppExtractContext> ectx;
451
  std::string entriesPath;
452
};
×
453

454
std::shared_ptr<AppContextShare> ZIPIOContext::Instance(ZIPIOEntry entry) {
455
  return std::make_unique<ZIPIOContextInstance>(this, entry);
×
456
}
×
457

458
static std::mutex ZIPLock;
459

×
460
struct ZIPDataHolder {
×
461
  virtual ~ZIPDataHolder() = default;
×
462
};
463

×
464
struct ZIPIOContext_implbase : ZIPIOContext {
465
  ZIPIOContext_implbase(const std::string &file) : zipMount(file) {}
466
  std::istream *OpenFile(const ZipEntry &entry) override;
467
  std::string GetChunk(const ZipEntry &entry, size_t offset,
468
                       size_t size) const override;
469
  void DisposeFile(std::istream *str) override;
470

471
  void Merge(ZIPExtactContext *eCtx, const std::string &records) override {
472
    std::lock_guard<std::mutex> lg(mergerMtx);
473
    merger->Merge(*eCtx, records);
474
  }
×
475

×
476
  void InitMerger() override {
477
    std::lock_guard<std::mutex> lg(mergerMtx);
478
    if (!merger) {
479
      merger.emplace(basePath + "_out.zip", RequestTempFile());
480
    }
481
  }
482

483
  void Finish() override {
484
    if (merger) {
485
      merger->FinishMerge([] { printinfo("Generating cache."); });
×
486
    }
487
  }
488

489
protected:
490
  std::list<std::spanstream> openedFiles;
491
  es::MappedFile zipMount;
×
492
  std::optional<ZIPMerger> merger;
×
493
  std::mutex mergerMtx;
×
494
};
495

496
struct ZIPMemoryStream : ZIPDataHolder {
×
497
  std::istringstream stream;
×
498

×
499
  ZIPMemoryStream(std::string &&input)
×
500
      : stream(input, std::ios::in | std::ios::binary) {}
501
};
502

503
struct ZIPFileStream : ZIPDataHolder {
×
504
  BinReader rd;
×
505
  std::string path;
×
506
  ZIPFileStream(const std::string &path_) : rd(path_), path(path_) {}
507
  ~ZIPFileStream() {
508
    es::Dispose(rd);
509
    try {
510
      es::RemoveFile(path);
511
    } catch (const std::exception &e) {
512
      printerror(e.what());
513
    }
514
  }
515
};
516

517
std::istream *ZIPIOContext_implbase::OpenFile(const ZipEntry &entry) {
518
  auto dataBegin = static_cast<char *>(zipMount.data) + entry.offset;
519
  auto dataEnd = dataBegin + entry.size;
520

521
  std::lock_guard<std::mutex> guard(ZIPLock);
522
  auto &str = openedFiles.emplace_back(std::span<char>(dataBegin, dataEnd),
523
                                       std::ios::binary | std::ios::in);
524
  return &str;
525
}
526

527
std::string ZIPIOContext_implbase::GetChunk(const ZipEntry &entry,
528
                                            size_t offset, size_t size) const {
529
  auto dataBegin = static_cast<char *>(zipMount.data) + entry.offset + offset;
530
  auto dataEnd = dataBegin + size;
531

532
  return {dataBegin, dataEnd};
533
}
534

535
void ZIPIOContext_implbase::DisposeFile(std::istream *str) {
536
  std::lock_guard<std::mutex> guard(ZIPLock);
537
  openedFiles.remove_if([&](auto &spanStr) {
×
538
    return static_cast<std::istream *>(&spanStr) == str;
×
539
  });
×
540
}
541

542
struct ZIPIOContextIter_impl : ZIPIOEntryRawIterator {
×
543
  using map_type = std::map<std::string_view, ZipEntry>;
×
544
  ZIPIOContextIter_impl(const map_type &map)
×
545
      : base(&map), current(map.begin()), end(map.end()) {}
546
  ZIPIOEntry Fist() const override {
547
    if (current == end) {
×
548
      return {};
549
    }
×
550
    return {current->second, current->first};
×
551
  }
552
  ZIPIOEntry Next() const override {
×
553
    if (current == end) {
554
      return {};
555
    }
×
556
    current++;
557
    if (current == end) {
×
558
      return {};
×
559
    }
560

561
    return {current->second, current->first};
562
  }
563
  size_t Count() const override { return base->size(); }
564

565
  const map_type *base;
×
566
  mutable map_type::const_iterator current;
×
567
  map_type::const_iterator end;
×
568
};
×
569

570
struct ZIPIOContext_impl : ZIPIOContext_implbase {
571
  AppContextStream RequestFile(const std::string &path) override;
572

×
573
  AppContextFoundStream FindFile(const std::string &rootFolder,
×
574
                                 const std::string &pattern) override;
×
575

576
  ZIPIOContextIterator Iter(ZIPIOEntryType) const override {
577
    return {std::make_unique<ZIPIOContextIter_impl>(vfs)};
×
578
  }
×
579

580
  ZIPIOContext_impl(const std::string &file, const PathFilter &pathFilter_,
581
                    const PathFilter &moduleFilter_)
582
      : ZIPIOContext_implbase(file), pathFilter(&pathFilter_),
583
        moduleFilter(&moduleFilter_) {
×
584
    Read();
585
    pathFilter = moduleFilter = nullptr;
586
  }
587

588
  ZIPIOContext_impl(const std::string &file) : ZIPIOContext_implbase(file) {
589
    Read();
590
  }
591

592
private:
593
  void Read();
594
  const PathFilter *pathFilter = nullptr;
595
  const PathFilter *moduleFilter = nullptr;
596
  std::map<std::string_view, ZipEntry> vfs;
×
597
};
×
598

599
AppContextStream ZIPIOContext_impl::RequestFile(const std::string &path) {
600
  auto found = vfs.find(path);
×
601

602
  if (es::IsEnd(vfs, found)) {
×
603
    throw es::FileNotFoundError(path);
×
604
  }
×
605

×
606
  return {OpenFile(found->second), this};
607
}
608

×
609
AppContextFoundStream ZIPIOContext_impl::FindFile(const std::string &,
×
610
                                                  const std::string &pattern) {
611
  PathFilter filter;
612
  filter.AddFilter(pattern);
613

614
  for (auto &f : vfs) {
615
    std::string_view kvi(f.first);
616
    size_t lastSlash = kvi.find_last_of('/');
617
    kvi.remove_prefix(lastSlash + 1);
618

619
    if (filter.IsFiltered(kvi)) {
×
620
      return {OpenFile(f.second), this, AFileInfo(f.first)};
×
621
    }
622
  }
×
623

×
624
  throw es::FileNotFoundError(pattern);
625
}
626

×
627
// Warning: Unaligned accesses
628
// Note: Multiple central directories? (unlikely)
629
void ZIPIOContext_impl::Read() {
×
630
  auto curEnd = static_cast<char *>(zipMount.data) + zipMount.fileSize -
631
                (sizeof(ZIPCentralDir) - 2);
×
632
  auto curLocator = reinterpret_cast<const ZIPCentralDir *>(curEnd);
×
633

634
  if (curLocator->id != ZIPCentralDir::ID) {
×
635
    int numIters = 4096;
×
636
    while (numIters > 0) {
637
      curEnd--;
×
638
      numIters--;
639

×
640
      curLocator = reinterpret_cast<const ZIPCentralDir *>(curEnd);
×
641
      if (curLocator->id == ZIPCentralDir::ID) {
642
        break;
643
      }
644
    }
×
645
  }
646

647
  if (curLocator->id != ZIPCentralDir::ID) {
648
    throw std::runtime_error("Cannot find ZIP central directory");
649
  }
×
650

×
651
  uint64 dirOffset = 0;
652
  uint64 numEntries = 0;
653
  uint64 dirSize = 0;
654

×
655
  if (curLocator->dirOffset == -1U || curLocator->numDirEntries == uint16(-1) ||
656
      curLocator->dirSize == -1U) {
×
657
    curEnd -= sizeof(ZIP64CentralDir) - 8;
×
658
    auto curLocatorX64 = reinterpret_cast<const ZIP64CentralDir *>(curEnd);
×
659
    if (curLocatorX64->id != ZIP64CentralDir::ID) {
660
      int numIters = 4096;
661
      while (numIters > 0) {
×
662
        curEnd--;
663
        numIters--;
664

665
        curLocatorX64 = reinterpret_cast<const ZIP64CentralDir *>(curEnd);
666
        if (curLocatorX64->id == ZIP64CentralDir::ID) {
667
          break;
×
668
        }
×
669
      }
670
    }
671

672
    if (curLocatorX64->id != ZIP64CentralDir::ID) {
673
      throw std::runtime_error("Cannot find ZIPx64 central directory");
674
    }
675

×
676
    std::spanstream entriesSpan(
×
677
        std::span<char>(curEnd,
×
678
                        static_cast<char *>(zipMount.data) + zipMount.fileSize),
679
        std::ios::binary | std::ios::in);
×
680
    BinReaderRef rd(entriesSpan);
681
    ZIP64CentralDir x64CentraDir;
×
682
    rd.Read(x64CentraDir);
×
683

×
684
    dirOffset = x64CentraDir.dirOffset;
685
    numEntries = x64CentraDir.numDirEntries;
686
    dirSize = x64CentraDir.dirSize;
×
687
  } else {
688
    dirOffset = curLocator->dirOffset;
689
    numEntries = curLocator->numDirEntries;
690
    dirSize = curLocator->dirSize;
691
  }
692

×
693
  auto entriesBegin = static_cast<char *>(zipMount.data) + dirOffset;
×
694
  auto entriesEnd = entriesBegin + dirSize;
695
  std::spanstream entriesSpan(std::span<char>(entriesBegin, entriesEnd),
696
                              std::ios::binary | std::ios::in);
697
  BinReaderRef rd(entriesSpan);
698
  std::spanstream localStreamSpan(
699
      std::span<char>(static_cast<char *>(zipMount.data), entriesBegin),
×
700
      std::ios::binary | std::ios::in);
701
  BinReaderRef localRd(localStreamSpan);
702

703
  for (size_t d = 0; d < numEntries; d++) {
704
    uint32 id;
×
705
    rd.Push();
×
706
    rd.Read(id);
×
707
    rd.Pop();
×
708

×
709
    switch (id) {
×
710
    case ZIPFile::ID: {
×
711
      ZIPFile hdr;
712
      rd.Read(hdr);
713

×
714
      [&] {
715
        std::string_view entryName(entriesBegin + rd.Tell(), hdr.fileNameSize);
716
        rd.Skip(hdr.fileNameSize);
×
717

718
        if (!hdr.compressedSize) {
719
          return;
×
720
        }
×
721

722
        if (hdr.flags[ZIPLocalFlag::Encrypted]) {
723
          throw std::runtime_error("ZIP cannot have encrypted files!");
×
724
        }
725

726
        if (hdr.compression != ZIPCompressionMethod::Store) {
727
          throw std::runtime_error("ZIP cannot have compressed files!");
728
        }
729

×
730
        if (!hdr.fileNameSize) {
731
          throw std::runtime_error("ZIP local file's path must be specified!");
732
        }
×
733

734
        size_t entrySize = hdr.compressedSize;
×
735
        size_t localOffset = hdr.localHeaderOffset;
×
736

×
737
        if (hdr.compressedSize == -1U || hdr.uncompressedSize == -1U ||
738
            hdr.localHeaderOffset == -1U) {
×
739
          const size_t extraEnd = rd.Push() + hdr.extraFieldSize;
×
740

741
          while (rd.Tell() < extraEnd) {
742
            ZIP64Extra extra;
×
743
            rd.Read(extra.id);
×
744
            rd.Read(extra.size);
745

746
            if (extra.id == 1) {
×
747
              if (hdr.uncompressedSize == -1U) {
×
748
                rd.Read(extra.uncompressedSize);
749
                entrySize = extra.uncompressedSize;
750
              }
×
751
              if (hdr.compressedSize == -1U) {
×
752
                rd.Read(extra.compressedSize);
753
              }
754
              if (hdr.localHeaderOffset == -1U) {
×
755
                rd.Read(extra.localHeaderOffset);
×
756
                localOffset = extra.localHeaderOffset;
757
              }
×
758
              break;
759
            } else {
×
760
              rd.Skip(extra.size);
761
            }
×
762
          }
763

764
          rd.Pop();
765
        }
766

×
767
        if (pathFilter && !pathFilter->IsFiltered(entryName)) {
×
768
          return;
769
        }
×
770

771
        if (moduleFilter && !moduleFilter->IsFiltered(entryName)) {
×
772
          return;
773
        }
774

×
775
        ZIPLocalFile localEntry;
776
        localRd.Seek(localOffset);
×
777
        localRd.Read(localEntry);
778
        ZipEntry entry;
×
779
        entry.size = entrySize;
780
        entry.offset = localRd.Tell() + localEntry.extraFieldSize +
×
781
                       localEntry.fileNameSize;
782
        vfs.emplace(entryName, entry);
783
      }();
784

785
      rd.Skip(hdr.extraFieldSize + hdr.fileCommentSize);
786
      break;
787
    }
×
788
    default:
789
      using std::to_string;
790
      throw std::runtime_error("Invalid dir entry " + to_string(id) + " at " +
791
                               to_string(rd.Tell() + dirOffset));
×
792
    }
793
  }
794
}
795

796
ZIPIOEntry::operator bool() const {
×
797
  return std::visit([](auto &name) { return !name.empty(); }, name);
×
798
}
799

×
800
struct ZIPIOContextCached : ZIPIOContext_implbase {
×
801
  AppContextStream RequestFile(const std::string &path) override {
×
802

×
803
    auto found = cache.RequestFile(path);
×
804

805
    if (!found.size) {
×
806
      throw es::FileNotFoundError(path);
807
    }
808

×
809
    return {OpenFile(found), this};
810
  }
×
811

×
812
  AppContextFoundStream FindFile(const std::string &,
813
                                 const std::string &pattern) override {
814
    auto found = cache.FindFile(pattern);
815

816
    if (!found.size) {
×
817
      throw es::FileNotFoundError(pattern);
×
818
    }
819

820
    return std::visit(
821
        [&](auto &item) -> AppContextFoundStream {
×
822
          return {OpenFile(found), this, AFileInfo(item)};
823
        },
×
824
        found.name);
825
  }
×
826

×
827
  ZIPIOContextIterator Iter(ZIPIOEntryType type) const override {
828
    return {cache.Iter(type)};
829
  }
×
830

831
  ZIPIOContextCached(const std::string &file, es::MappedFile &&cacheFile)
832
      : ZIPIOContext_implbase(file), cacheMount(std::move(cacheFile)) {
×
833
    cache.Mount(cacheMount.data);
834
    auto &cacheHdr = reinterpret_cast<const CacheBaseHeader &>(cache.Header());
×
835
    auto zipData = static_cast<const char *>(zipMount.data);
836
    auto zipHeader = reinterpret_cast<const CacheBaseHeader *>(
×
837
        zipData + cacheHdr.zipCheckupOffset);
×
838

839
    if (memcmp(zipHeader, &cacheHdr, sizeof(cacheHdr))) {
840
      throw std::runtime_error("Cache header and zip checkup are different.");
841
    }
×
842
  }
×
843

844
private:
×
845
  Cache cache;
×
846
  es::MappedFile cacheMount;
847
};
×
848

×
849
std::unique_ptr<ZIPIOContext> MakeZIPContext(const std::string &file,
850
                                             const PathFilter &pathFilter,
×
851
                                             const PathFilter &moduleFilter) {
852
  return std::make_unique<ZIPIOContext_impl>(file, pathFilter, moduleFilter);
853
}
×
854

×
855
std::unique_ptr<ZIPIOContext> MakeZIPContext(const std::string &file) {
856
  std::string cacheFile = file + ".cache";
857
  es::MappedFile mf;
×
858
  try {
×
859
    mf = es::MappedFile(cacheFile);
×
860
  } catch (const std::exception &e) {
×
861
    printwarning("Failed loading cache: " << e.what());
×
862
    return std::make_unique<ZIPIOContext_impl>(file);
×
863
  }
×
864
  try {
865
    printinfo("Found zip cache: " << cacheFile);
×
866
    return std::make_unique<ZIPIOContextCached>(file, std::move(mf));
×
867
  } catch (const std::exception &e) {
868
    printwarning("Failed loading cache: " << e.what());
869
    return std::make_unique<ZIPIOContext_impl>(file);
870
  }
871
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc