• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

PredatorCZ / PreCore / 477

25 Aug 2023 03:56PM UTC coverage: 55.317% (+2.7%) from 52.584%
477

push

github-actions-ci

PredatorCZ
add texel context

1382 of 1382 new or added lines in 8 files covered. (100.0%)

4146 of 7495 relevant lines covered (55.32%)

8934.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

9.47
/src/app/in_context.cpp
1
/*  Spike is universal dedicated module handler
2
    This source contains context for input data
3

4
    Copyright 2021-2023 Lukas Cone
5

6
    Licensed under the Apache License, Version 2.0 (the "License");
7
    you may not use this file except in compliance with the License.
8
    You may obtain a copy of the License at
9

10
        http://www.apache.org/licenses/LICENSE-2.0
11

12
    Unless required by applicable law or agreed to in writing, software
13
    distributed under the License is distributed on an "AS IS" BASIS,
14
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
    See the License for the specific language governing permissions and
16
    limitations under the License.
17
*/
18

19
#include "spike/app/context.hpp"
20
#include "spike/app/out_context.hpp"
21
#include "spike/app/texel.hpp"
22
#include "spike/app/tmp_storage.hpp"
23
#include "spike/format/ZIP_istream.inl"
24
#include "spike/io/binreader.hpp"
25
#include "spike/io/binwritter.hpp"
26
#include "spike/io/directory_scanner.hpp"
27
#include "spike/io/fileinfo.hpp"
28
#include "spike/io/stat.hpp"
29
#include "spike/master_printer.hpp"
30
#include <list>
31
#include <mutex>
32
#include <optional>
33
#include <spanstream>
34

35
static std::mutex simpleIOLock;
36

37
const std::vector<std::string> &AppContextShare::SupplementalFiles() {
×
38
  if (!supplementals) {
×
39
    throw std::runtime_error(
×
40
        "Invalid call of SupplementalFiles, module is not for batch.");
×
41
  }
42

43
  return supplementals.value();
×
44
}
45

46
const std::vector<std::string> &ZIPIOContext::SupplementalFiles() {
×
47
  if (!supplementals) {
×
48
    throw std::runtime_error(
×
49
        "Invalid call of SupplementalFiles, module is not for batch.");
×
50
  }
51

52
  return supplementals.value();
×
53
}
54

55
struct AppContextShareImpl : AppContextShare {
56
  NewTexelContext *NewImage(NewTexelContextCreate ctx,
56✔
57
                            const std::string *path) override {
58
    if (texelContext) {
56✔
59
      texelContext->Finish();
28✔
60
    }
61

62
    texelContext = CreateTexelContext(ctx, this);
56✔
63

64
    if (path) {
56✔
65
      texelContext->pathOverride.Load(*path);
28✔
66
    }
67

68
    return texelContext.get();
56✔
69
  }
70

71
  NewFileContext NewFile(const std::string &path) override {
190✔
72
    std::string filePath;
73
    size_t delimeter = 0;
74

75
    if (basePathParts.empty()) {
190✔
76
      filePath = std::string(basePath.GetFullPath()) + path;
380✔
77
      delimeter = basePath.GetFullPath().size();
78
    } else {
79
      AFileInfo pathInfo(path);
×
80
      auto exploded = pathInfo.Explode();
×
81
      const size_t numItems = std::min(exploded.size(), basePathParts.size());
×
82

83
      if (basePath.GetFullPath().at(0) == '/') {
×
84
        filePath.push_back('/');
×
85
      }
86

87
      size_t i = 0;
88

89
      for (; i < numItems; i++) {
×
90
        if (basePathParts[i] == exploded[i]) {
×
91
          filePath.append(exploded[i]);
92
          filePath.push_back('/');
×
93
          continue;
94
        }
95

96
        break;
97
      }
98

99
      for (size_t j = i; j < basePathParts.size(); j++) {
×
100
        filePath.append(basePathParts[j]);
101
        filePath.push_back('/');
×
102
      }
103

104
      delimeter = filePath.size();
105

106
      for (; i < exploded.size(); i++) {
×
107
        filePath.append(exploded[i]);
108
        filePath.push_back('/');
×
109
      }
110

111
      filePath.pop_back();
112
    }
113

114
    try {
115
      outFile = BinWritter(filePath);
379✔
116
    } catch (const es::FileInvalidAccessError &e) {
1✔
117
      mkdirs(filePath);
1✔
118
      outFile = BinWritter(filePath);
1✔
119
    }
1✔
120
    return {outFile.BaseStream(), filePath, delimeter};
190✔
121
  }
380✔
122

123
  void BaseOutputPath(std::string basePath_) override {
×
124
    if (basePath_.back() != '/') {
×
125
      basePath_.push_back('/');
×
126
    }
127
    basePath = AFileInfo(basePath_);
×
128
    basePathParts = basePath.Explode();
×
129
  }
130

131
  JenHash Hash() override { return JenHash(FullPath()); }
×
132

133
  std::string FullPath() override {
×
134
    return std::string(basePath.GetFullPath()) +
×
135
           std::string(workingFile.GetFullPath());
×
136
  }
137

138
  ~AppContextShareImpl() {
28✔
139
    if (texelContext) {
28✔
140
      texelContext->Finish();
28✔
141
    }
142
  }
28✔
143

×
144
  BinWritter outFile;
145
  AFileInfo basePath;
146
  std::vector<std::string_view> basePathParts;
147
  std::unique_ptr<NewTexelContextImpl> texelContext;
148
};
28✔
149

28✔
150
struct SimpleIOContext : AppContextShareImpl {
28✔
151
  SimpleIOContext(const std::string &path,
152
                  std::optional<std::vector<std::string>> supplementals_) {
28✔
153
    mainFile.Open(path);
154
    workingFile.Load(path);
155
    supplementals = std::move(supplementals_);
156

157
    if (!cliSettings.out.empty()) {
158
      BaseOutputPath(cliSettings.out);
159
    }
160
  }
161
  std::istream *OpenFile(const std::string &path);
28✔
162

924✔
163
  AppContextStream RequestFile(const std::string &path) override;
28✔
164

28✔
165
  AppContextFoundStream FindFile(const std::string &rootFolder,
166
                                 const std::string &pattern) override;
167
  std::istream &GetStream() override;
28✔
168
  std::string GetBuffer(size_t size, size_t begin) override;
×
169

170
  void DisposeFile(std::istream *str) override;
28✔
171

172
  AppExtractContext *ExtractContext(std::string_view name) override {
173
    if (ectx) [[unlikely]] {
174
      return ectx.get();
175
    }
176

177
    std::string outPath(basePath.GetFullPath());
178
    outPath += workingFile.GetFolder();
179
    outPath += name;
180

181
    if (mainSettings.extractSettings.makeZIP) {
182
      if (workingFile.GetExtension() == ".zip") {
×
183
        outPath.append("_out");
×
184
      }
×
185

186
      outPath.append(".zip");
187

×
188
      auto uniq = std::make_unique<ZIPExtactContext>(outPath);
189
      uniq->forEachFile = forEachFile;
190
      ectx = std::move(uniq);
191
    } else {
×
192
      if (!mainSettings.extractSettings.folderPerArc) {
×
193
        outPath = workingFile.GetFolder();
×
194
      } else {
195
        es::mkdir(outPath);
196
        outPath.push_back('/');
×
197
      }
198

×
199
      auto uniq = std::make_unique<IOExtractContext>(outPath);
×
200
      uniq->forEachFile = forEachFile;
201
      ectx = std::move(uniq);
202
    }
×
203

204
    return ectx.get();
205
  }
206

×
207
  AppExtractContext *ExtractContext() override {
208
    return ExtractContext(workingFile.GetFilename());
209
  }
×
210

×
211
  void Finish() override {
212
    if (ectx && mainSettings.extractSettings.makeZIP) {
213
      static_cast<ZIPExtactContext *>(ectx.get())->FinishZIP([] {
214
        printinfo("Generating cache.");
215
      });
216
    }
217
  }
×
218

×
219
private:
220
  BinReader mainFile;
221
  BinReader streamedFiles[32];
×
222
  uint32 usedFiles = 0;
×
223
  std::unique_ptr<AppExtractContext> ectx;
224
};
×
225

×
226
std::istream *SimpleIOContext::OpenFile(const std::string &path) {
227
  std::lock_guard<std::mutex> guard(simpleIOLock);
228
  for (size_t b = 0; b < 32; b++) {
229
    uint32 bit = 1 << b;
230
    if (!(usedFiles & bit)) {
231
      streamedFiles[b].Open(path);
232
      usedFiles ^= bit;
233
      return &streamedFiles[b].BaseStream();
234
    }
235
  }
236

×
237
  throw std::out_of_range("Maximum opened files reached!");
238
}
×
239

×
240
AppContextStream SimpleIOContext::RequestFile(const std::string &path) {
×
241
  AFileInfo wFile(workingFile);
×
242
  AFileInfo pFile(path);
×
243
  auto catchedFile = pFile.CatchBranch(wFile.GetFolder());
×
244
  return {OpenFile(catchedFile), this};
245
}
246

247
AppContextFoundStream SimpleIOContext::FindFile(const std::string &rootFolder,
×
248
                                                const std::string &pattern) {
249
  DirectoryScanner sc;
250
  sc.AddFilter(pattern);
×
251
  sc.Scan(rootFolder);
252

×
253
  if (sc.Files().empty()) {
×
254
    throw es::FileNotFoundError(pattern);
×
255
  } else if (sc.Files().size() > 1) {
256
    std::string *winner = nullptr;
257
    size_t minFolder = 0x10000;
×
258
    size_t minLevel = 0x10000;
259

×
260
    for (auto &f : sc) {
×
261
      size_t foundIdx = f.find_last_of('/');
×
262

263
      if (foundIdx == f.npos) {
×
264
        throw std::runtime_error("Too many files found.");
×
265
      }
×
266

267
      if (foundIdx < minFolder) {
268
        winner = &f;
269
        minFolder = foundIdx;
270
        minLevel = std::count(f.begin(), f.end(), '/');
×
271
      } else if (foundIdx == minFolder) {
272
        if (auto clevel = std::count(f.begin(), f.end(), '/');
273
            clevel < minLevel) {
×
274
          winner = &f;
×
275
          minFolder = foundIdx;
276
          minLevel = clevel;
277
        } else if (clevel == minLevel) {
×
278
          throw std::runtime_error("Too many files found.");
279
        }
280
      }
×
281
    }
×
282

×
283
    if (!winner) {
×
284
      throw std::runtime_error("Too many files found.");
285
    }
286

287
    return {OpenFile(*winner), this, AFileInfo(*winner)};
×
288
  }
×
289

290
  return {OpenFile(sc.Files().front()), this, AFileInfo(sc.Files().front())};
291
}
292

293
void SimpleIOContext::DisposeFile(std::istream *str) {
×
294
  size_t index = 0;
×
295

296
  for (auto &f : streamedFiles) {
297
    if (&f.BaseStream() == str) {
×
298
      uint32 bit = 1 << index;
299

300
      if (!(usedFiles & bit)) {
×
301
        throw std::runtime_error("Stream already freed.");
302
      }
303

×
304
      es::Dispose(f);
305
      usedFiles ^= bit;
306
      return;
×
307
    }
×
308

×
309
    index++;
310
  }
×
311

×
312
  throw std::runtime_error("Requested stream not found!");
313
}
314

×
315
std::istream &SimpleIOContext::GetStream() { return mainFile.BaseStream(); }
×
316

×
317
std::string SimpleIOContext::GetBuffer(size_t size, size_t begin) {
318
  mainFile.Push();
319
  mainFile.Seek(begin);
×
320
  std::string buffer;
321
  mainFile.ReadContainer(buffer,
322
                         size == size_t(-1) ? mainFile.GetSize() : size);
×
323
  mainFile.Pop();
324

325
  return buffer;
28✔
326
}
327

×
328
std::shared_ptr<AppContextShare>
×
329
MakeIOContext(const std::string &path,
330
              std::optional<std::vector<std::string>> supplementals) {
331
  return std::make_unique<SimpleIOContext>(path, supplementals);
×
332
}
×
333

334
struct ZIPIOContextInstance : AppContextShareImpl {
335
  ZIPIOContextInstance(const ZIPIOContextInstance &) = delete;
×
336
  ZIPIOContextInstance(ZIPIOContextInstance &&) = delete;
337
  ZIPIOContextInstance(ZIPIOContext *base_, ZIPIOEntry entry_)
338
      : base(base_), entry(entry_) {
339
    workingFile.Load(entry_.AsView());
28✔
340
    BaseOutputPath(base->basePath);
341
  }
56✔
342

343
  ~ZIPIOContextInstance() {
344
    if (stream) {
345
      base->DisposeFile(stream);
346
    }
347
  }
×
348

×
349
  AppContextStream RequestFile(const std::string &path) override {
×
350
    return base->RequestFile(path);
×
351
  }
352
  void DisposeFile(std::istream *file) override { base->DisposeFile(file); }
353
  AppContextFoundStream FindFile(const std::string &rootFolder,
×
354
                                 const std::string &pattern) override {
×
355
    return base->FindFile(rootFolder, pattern);
×
356
  }
357

358
  std::istream &GetStream() override {
×
359
    if (!stream) {
360
      stream = base->OpenFile(entry);
361
    }
362
    return *stream;
363
  }
×
364
  std::string GetBuffer(size_t size, size_t begin) override {
×
365
    BinReaderRef rd(GetStream());
×
366
    rd.Push();
367
    rd.Seek(begin);
368
    std::string buffer;
369
    rd.ReadContainer(buffer, size == size_t(-1) ? rd.GetSize() : size);
×
370
    rd.Pop();
×
371
    return buffer;
372
  }
×
373

×
374
  AppExtractContext *ExtractContext(std::string_view name) override {
375
    if (ectx) [[unlikely]] {
×
376
      return ectx.get();
377
    }
378

×
379
    if (mainSettings.extractSettings.makeZIP) {
×
380
      base->InitMerger();
×
381
      entriesPath = RequestTempFile();
382
      auto uniq = std::make_unique<ZIPExtactContext>(entriesPath, false);
×
383
      if (mainSettings.extractSettings.folderPerArc) {
384
        uniq->prefixPath = workingFile.GetFolder();
×
385
        uniq->prefixPath += name;
×
386
        uniq->prefixPath.push_back('/');
387
      }
388
      uniq->forEachFile = forEachFile;
389
      ectx = std::move(uniq);
×
390
    } else {
391
      std::string outPath;
×
392
      if (!mainSettings.extractSettings.folderPerArc) {
393
        outPath = workingFile.GetFolder();
394
      } else {
×
395
        outPath = std::string(basePath.GetFullPath());
×
396
        outPath += workingFile.GetFolder();
×
397
        outPath += name;
398
        mkdirs(outPath);
399
        outPath.push_back('/');
×
400
      }
×
401

×
402
      auto uniq = std::make_unique<IOExtractContext>(outPath);
×
403
      uniq->forEachFile = forEachFile;
×
404
      ectx = std::move(uniq);
×
405
    }
406

×
407
    return ectx.get();
408
  }
×
409

410
  AppExtractContext *ExtractContext() override {
411
    return ExtractContext(workingFile.GetFilename());
412
  }
×
413

414
  void Finish() override {
415
    if (ectx && mainSettings.extractSettings.makeZIP) {
×
416
      base->Merge(static_cast<ZIPExtactContext *>(ectx.get()), entriesPath);
417
      es::Dispose(ectx);
418
      es::RemoveFile(entriesPath);
×
419
    }
×
420
  }
421

422
  ZIPIOContext *base;
×
423
  std::istream *stream = nullptr;
×
424
  ZipEntry entry;
425
  std::unique_ptr<AppExtractContext> ectx;
426
  std::string entriesPath;
427
};
×
428

429
std::shared_ptr<AppContextShare> ZIPIOContext::Instance(ZIPIOEntry entry) {
430
  return std::make_unique<ZIPIOContextInstance>(this, entry);
×
431
}
×
432

433
static std::mutex ZIPLock;
434

×
435
struct ZIPDataHolder {
×
436
  virtual ~ZIPDataHolder() = default;
×
437
};
438

×
439
struct ZIPIOContext_implbase : ZIPIOContext {
440
  ZIPIOContext_implbase(const std::string &file) : zipMount(file) {}
441
  std::istream *OpenFile(const ZipEntry &entry) override;
442
  std::string GetChunk(const ZipEntry &entry, size_t offset,
443
                       size_t size) const override;
444
  void DisposeFile(std::istream *str) override;
445

446
  void Merge(ZIPExtactContext *eCtx, const std::string &records) override {
447
    std::lock_guard<std::mutex> lg(mergerMtx);
448
    merger->Merge(*eCtx, records);
449
  }
×
450

×
451
  void InitMerger() override {
452
    std::lock_guard<std::mutex> lg(mergerMtx);
453
    if (!merger) {
454
      merger.emplace(basePath + "_out.zip", RequestTempFile());
455
    }
456
  }
457

458
  void Finish() override {
459
    if (merger) {
460
      merger->FinishMerge([] { printinfo("Generating cache."); });
×
461
    }
462
  }
463

464
protected:
465
  std::list<std::spanstream> openedFiles;
466
  es::MappedFile zipMount;
×
467
  std::optional<ZIPMerger> merger;
×
468
  std::mutex mergerMtx;
×
469
};
470

471
struct ZIPMemoryStream : ZIPDataHolder {
×
472
  std::istringstream stream;
×
473

×
474
  ZIPMemoryStream(std::string &&input)
×
475
      : stream(input, std::ios::in | std::ios::binary) {}
476
};
477

478
struct ZIPFileStream : ZIPDataHolder {
×
479
  BinReader rd;
×
480
  std::string path;
×
481
  ZIPFileStream(const std::string &path_) : rd(path_), path(path_) {}
482
  ~ZIPFileStream() {
483
    es::Dispose(rd);
484
    try {
485
      es::RemoveFile(path);
486
    } catch (const std::exception &e) {
487
      printerror(e.what());
488
    }
489
  }
490
};
491

492
std::istream *ZIPIOContext_implbase::OpenFile(const ZipEntry &entry) {
493
  auto dataBegin = static_cast<char *>(zipMount.data) + entry.offset;
494
  auto dataEnd = dataBegin + entry.size;
495

496
  std::lock_guard<std::mutex> guard(ZIPLock);
497
  auto &str = openedFiles.emplace_back(std::span<char>(dataBegin, dataEnd),
498
                                       std::ios::binary | std::ios::in);
499
  return &str;
500
}
501

502
std::string ZIPIOContext_implbase::GetChunk(const ZipEntry &entry,
503
                                            size_t offset, size_t size) const {
504
  auto dataBegin = static_cast<char *>(zipMount.data) + entry.offset + offset;
505
  auto dataEnd = dataBegin + size;
506

507
  return {dataBegin, dataEnd};
508
}
509

510
void ZIPIOContext_implbase::DisposeFile(std::istream *str) {
511
  std::lock_guard<std::mutex> guard(ZIPLock);
512
  openedFiles.remove_if([&](auto &spanStr) {
×
513
    return static_cast<std::istream *>(&spanStr) == str;
×
514
  });
×
515
}
516

517
struct ZIPIOContextIter_impl : ZIPIOEntryRawIterator {
×
518
  using map_type = std::map<std::string_view, ZipEntry>;
×
519
  ZIPIOContextIter_impl(const map_type &map)
×
520
      : base(&map), current(map.begin()), end(map.end()) {}
521
  ZIPIOEntry Fist() const override {
522
    if (current == end) {
×
523
      return {};
524
    }
×
525
    return {current->second, current->first};
×
526
  }
527
  ZIPIOEntry Next() const override {
×
528
    if (current == end) {
529
      return {};
530
    }
×
531
    current++;
532
    if (current == end) {
×
533
      return {};
×
534
    }
535

536
    return {current->second, current->first};
537
  }
538
  size_t Count() const override { return base->size(); }
539

540
  const map_type *base;
×
541
  mutable map_type::const_iterator current;
×
542
  map_type::const_iterator end;
×
543
};
×
544

545
struct ZIPIOContext_impl : ZIPIOContext_implbase {
546
  AppContextStream RequestFile(const std::string &path) override;
547

×
548
  AppContextFoundStream FindFile(const std::string &rootFolder,
×
549
                                 const std::string &pattern) override;
×
550

551
  ZIPIOContextIterator Iter(ZIPIOEntryType) const override {
552
    return {std::make_unique<ZIPIOContextIter_impl>(vfs)};
×
553
  }
×
554

555
  ZIPIOContext_impl(const std::string &file, const PathFilter &pathFilter_,
556
                    const PathFilter &moduleFilter_)
557
      : ZIPIOContext_implbase(file), pathFilter(&pathFilter_),
558
        moduleFilter(&moduleFilter_) {
×
559
    Read();
560
    pathFilter = moduleFilter = nullptr;
561
  }
562

563
  ZIPIOContext_impl(const std::string &file) : ZIPIOContext_implbase(file) {
564
    Read();
565
  }
566

567
private:
568
  void Read();
569
  const PathFilter *pathFilter = nullptr;
570
  const PathFilter *moduleFilter = nullptr;
571
  std::map<std::string_view, ZipEntry> vfs;
×
572
};
×
573

574
AppContextStream ZIPIOContext_impl::RequestFile(const std::string &path) {
575
  auto found = vfs.find(path);
×
576

577
  if (es::IsEnd(vfs, found)) {
×
578
    throw es::FileNotFoundError(path);
×
579
  }
×
580

×
581
  return {OpenFile(found->second), this};
582
}
583

×
584
AppContextFoundStream ZIPIOContext_impl::FindFile(const std::string &,
×
585
                                                  const std::string &pattern) {
586
  PathFilter filter;
587
  filter.AddFilter(pattern);
588

589
  for (auto &f : vfs) {
590
    std::string_view kvi(f.first);
591
    size_t lastSlash = kvi.find_last_of('/');
592
    kvi.remove_prefix(lastSlash + 1);
593

594
    if (filter.IsFiltered(kvi)) {
×
595
      return {OpenFile(f.second), this, AFileInfo(f.first)};
×
596
    }
597
  }
×
598

×
599
  throw es::FileNotFoundError(pattern);
600
}
601

×
602
// Warning: Unaligned accesses
603
// Note: Multiple central directories? (unlikely)
604
void ZIPIOContext_impl::Read() {
×
605
  auto curEnd = static_cast<char *>(zipMount.data) + zipMount.fileSize -
606
                (sizeof(ZIPCentralDir) - 2);
×
607
  auto curLocator = reinterpret_cast<const ZIPCentralDir *>(curEnd);
×
608

609
  if (curLocator->id != ZIPCentralDir::ID) {
×
610
    int numIters = 4096;
×
611
    while (numIters > 0) {
612
      curEnd--;
×
613
      numIters--;
614

×
615
      curLocator = reinterpret_cast<const ZIPCentralDir *>(curEnd);
×
616
      if (curLocator->id == ZIPCentralDir::ID) {
617
        break;
618
      }
619
    }
×
620
  }
621

622
  if (curLocator->id != ZIPCentralDir::ID) {
623
    throw std::runtime_error("Cannot find ZIP central directory");
624
  }
×
625

×
626
  uint64 dirOffset = 0;
627
  uint64 numEntries = 0;
628
  uint64 dirSize = 0;
629

×
630
  if (curLocator->dirOffset == -1U || curLocator->numDirEntries == uint16(-1) ||
631
      curLocator->dirSize == -1U) {
×
632
    curEnd -= sizeof(ZIP64CentralDir) - 8;
×
633
    auto curLocatorX64 = reinterpret_cast<const ZIP64CentralDir *>(curEnd);
×
634
    if (curLocatorX64->id != ZIP64CentralDir::ID) {
635
      int numIters = 4096;
636
      while (numIters > 0) {
×
637
        curEnd--;
638
        numIters--;
639

640
        curLocatorX64 = reinterpret_cast<const ZIP64CentralDir *>(curEnd);
641
        if (curLocatorX64->id == ZIP64CentralDir::ID) {
642
          break;
×
643
        }
×
644
      }
645
    }
646

647
    if (curLocatorX64->id != ZIP64CentralDir::ID) {
648
      throw std::runtime_error("Cannot find ZIPx64 central directory");
649
    }
650

×
651
    std::spanstream entriesSpan(
×
652
        std::span<char>(curEnd,
×
653
                        static_cast<char *>(zipMount.data) + zipMount.fileSize),
654
        std::ios::binary | std::ios::in);
×
655
    BinReaderRef rd(entriesSpan);
656
    ZIP64CentralDir x64CentraDir;
×
657
    rd.Read(x64CentraDir);
×
658

×
659
    dirOffset = x64CentraDir.dirOffset;
660
    numEntries = x64CentraDir.numDirEntries;
661
    dirSize = x64CentraDir.dirSize;
×
662
  } else {
663
    dirOffset = curLocator->dirOffset;
664
    numEntries = curLocator->numDirEntries;
665
    dirSize = curLocator->dirSize;
666
  }
667

×
668
  auto entriesBegin = static_cast<char *>(zipMount.data) + dirOffset;
×
669
  auto entriesEnd = entriesBegin + dirSize;
670
  std::spanstream entriesSpan(std::span<char>(entriesBegin, entriesEnd),
671
                              std::ios::binary | std::ios::in);
672
  BinReaderRef rd(entriesSpan);
673
  std::spanstream localStreamSpan(
674
      std::span<char>(static_cast<char *>(zipMount.data), entriesBegin),
×
675
      std::ios::binary | std::ios::in);
676
  BinReaderRef localRd(localStreamSpan);
677

678
  for (size_t d = 0; d < numEntries; d++) {
679
    uint32 id;
×
680
    rd.Push();
×
681
    rd.Read(id);
×
682
    rd.Pop();
×
683

×
684
    switch (id) {
×
685
    case ZIPFile::ID: {
×
686
      ZIPFile hdr;
687
      rd.Read(hdr);
688

×
689
      [&] {
690
        std::string_view entryName(entriesBegin + rd.Tell(), hdr.fileNameSize);
691
        rd.Skip(hdr.fileNameSize);
×
692

693
        if (!hdr.compressedSize) {
694
          return;
×
695
        }
×
696

697
        if (hdr.flags[ZIPLocalFlag::Encrypted]) {
698
          throw std::runtime_error("ZIP cannot have encrypted files!");
×
699
        }
700

701
        if (hdr.compression != ZIPCompressionMethod::Store) {
702
          throw std::runtime_error("ZIP cannot have compressed files!");
703
        }
704

×
705
        if (!hdr.fileNameSize) {
706
          throw std::runtime_error("ZIP local file's path must be specified!");
707
        }
×
708

709
        size_t entrySize = hdr.compressedSize;
×
710
        size_t localOffset = hdr.localHeaderOffset;
×
711

×
712
        if (hdr.compressedSize == -1U || hdr.uncompressedSize == -1U ||
713
            hdr.localHeaderOffset == -1U) {
×
714
          const size_t extraEnd = rd.Push() + hdr.extraFieldSize;
×
715

716
          while (rd.Tell() < extraEnd) {
717
            ZIP64Extra extra;
×
718
            rd.Read(extra.id);
×
719
            rd.Read(extra.size);
720

721
            if (extra.id == 1) {
×
722
              if (hdr.uncompressedSize == -1U) {
×
723
                rd.Read(extra.uncompressedSize);
724
                entrySize = extra.uncompressedSize;
725
              }
×
726
              if (hdr.compressedSize == -1U) {
×
727
                rd.Read(extra.compressedSize);
728
              }
729
              if (hdr.localHeaderOffset == -1U) {
×
730
                rd.Read(extra.localHeaderOffset);
×
731
                localOffset = extra.localHeaderOffset;
732
              }
×
733
              break;
734
            } else {
×
735
              rd.Skip(extra.size);
736
            }
×
737
          }
738

739
          rd.Pop();
740
        }
741

×
742
        if (pathFilter && !pathFilter->IsFiltered(entryName)) {
×
743
          return;
744
        }
×
745

746
        if (moduleFilter && !moduleFilter->IsFiltered(entryName)) {
×
747
          return;
748
        }
749

×
750
        ZIPLocalFile localEntry;
751
        localRd.Seek(localOffset);
×
752
        localRd.Read(localEntry);
753
        ZipEntry entry;
×
754
        entry.size = entrySize;
755
        entry.offset = localRd.Tell() + localEntry.extraFieldSize +
×
756
                       localEntry.fileNameSize;
757
        vfs.emplace(entryName, entry);
758
      }();
759

760
      rd.Skip(hdr.extraFieldSize + hdr.fileCommentSize);
761
      break;
762
    }
×
763
    default:
764
      using std::to_string;
765
      throw std::runtime_error("Invalid dir entry " + to_string(id) + " at " +
766
                               to_string(rd.Tell() + dirOffset));
×
767
    }
768
  }
769
}
770

771
ZIPIOEntry::operator bool() const {
×
772
  return std::visit([](auto &name) { return !name.empty(); }, name);
×
773
}
774

×
775
struct ZIPIOContextCached : ZIPIOContext_implbase {
×
776
  AppContextStream RequestFile(const std::string &path) override {
×
777

×
778
    auto found = cache.RequestFile(path);
×
779

780
    if (!found.size) {
×
781
      throw es::FileNotFoundError(path);
782
    }
783

×
784
    return {OpenFile(found), this};
785
  }
×
786

×
787
  AppContextFoundStream FindFile(const std::string &,
788
                                 const std::string &pattern) override {
789
    auto found = cache.FindFile(pattern);
790

791
    if (!found.size) {
×
792
      throw es::FileNotFoundError(pattern);
×
793
    }
794

795
    return std::visit(
796
        [&](auto &item) -> AppContextFoundStream {
×
797
          return {OpenFile(found), this, AFileInfo(item)};
798
        },
×
799
        found.name);
800
  }
×
801

×
802
  ZIPIOContextIterator Iter(ZIPIOEntryType type) const override {
803
    return {cache.Iter(type)};
804
  }
×
805

806
  ZIPIOContextCached(const std::string &file, es::MappedFile &&cacheFile)
807
      : ZIPIOContext_implbase(file), cacheMount(std::move(cacheFile)) {
×
808
    cache.Mount(cacheMount.data);
809
    auto &cacheHdr = reinterpret_cast<const CacheBaseHeader &>(cache.Header());
×
810
    auto zipData = static_cast<const char *>(zipMount.data);
811
    auto zipHeader = reinterpret_cast<const CacheBaseHeader *>(
×
812
        zipData + cacheHdr.zipCheckupOffset);
×
813

814
    if (memcmp(zipHeader, &cacheHdr, sizeof(cacheHdr))) {
815
      throw std::runtime_error("Cache header and zip checkup are different.");
816
    }
×
817
  }
×
818

819
private:
×
820
  Cache cache;
×
821
  es::MappedFile cacheMount;
822
};
×
823

×
824
std::unique_ptr<ZIPIOContext> MakeZIPContext(const std::string &file,
825
                                             const PathFilter &pathFilter,
×
826
                                             const PathFilter &moduleFilter) {
827
  return std::make_unique<ZIPIOContext_impl>(file, pathFilter, moduleFilter);
828
}
×
829

×
830
std::unique_ptr<ZIPIOContext> MakeZIPContext(const std::string &file) {
831
  std::string cacheFile = file + ".cache";
832
  es::MappedFile mf;
×
833
  try {
×
834
    mf = es::MappedFile(cacheFile);
×
835
  } catch (const std::exception &e) {
×
836
    printwarning("Failed loading cache: " << e.what());
×
837
    return std::make_unique<ZIPIOContext_impl>(file);
×
838
  }
×
839
  try {
840
    printinfo("Found zip cache: " << cacheFile);
×
841
    return std::make_unique<ZIPIOContextCached>(file, std::move(mf));
×
842
  } catch (const std::exception &e) {
843
    printwarning("Failed loading cache: " << e.what());
844
    return std::make_unique<ZIPIOContext_impl>(file);
845
  }
846
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc