PredatorCZ / PreCore / 460

pending completion

Build # 460

Build Type

push

github-actions-ci

Committed by

PredatorCZ

Commit Message

try fix coverage

Run Details

3204 of 6095 relevant lines covered (52.57%)

354.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/app/out_context.cpp

/*  Spike is universal dedicated module handler
    This souce contains data output context
    Part of PreCore project

    Copyright 2021-2022 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "spike/app/out_context.hpp"
#include "spike/app/console.hpp"
#include "spike/crypto/crc32.hpp"
#include "spike/format/ZIP_istream.inl"
#include "spike/format/ZIP_ostream.inl"
#include "spike/io/binreader.hpp"
#include "spike/io/fileinfo.hpp"
#include "spike/io/stat.hpp"
#include <chrono>
#include <mutex>

void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {
  FinishFile(true);

  auto entriesStr = std::move(entriesStream).str();
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();
  size_t dirSize = entriesStr.size();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  records.WriteContainer(entriesStr);
  es::Dispose(entriesStr);

  if (cache) {
    records.Write<uint16>(0x4353);
    records.Write<uint16>(sizeof(CacheBaseHeader));
    cache->meta.zipCheckupOffset = records.Tell();
    records.Write(cache->meta);
    dirSize += sizeof(CacheBaseHeader) + 4;
  }

  SafeCast(zCentral.dirSize, dirSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.dirRecord = 0x2C;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = dirSize;
    zCentral64.dirOffset = dirOffset;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  if (cache) {
    cacheBeginCB();
    cache->meta.zipSize = records.Tell();
    BinWritter cacheWr(outputFile + ".cache");
    cache->WaitAndWrite(cacheWr);
    records.Seek(cache->meta.zipCheckupOffset);
    records.Write(cache->meta);
  }
}

void ZIPExtactContext::FinishFile(bool final) {
  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    bool forcex64 = false;

    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }

    return forcex64;
  };

  const bool useLocalExtendedData =
      SafeCast(zLocalFile.uncompressedSize, curFileSize);
  zLocalFile.compressedSize = zLocalFile.uncompressedSize;

  if (useLocalExtendedData) {
    ZIP64Extra extra;
    extra.compressedSize = curFileSize;
    extra.uncompressedSize = curFileSize;
    records.Write(extra);
    zLocalFile.extraFieldSize = 20;
  }

  numEntries++;
  records.Push();
  records.Seek(curLocalFileOffset);
  records.Write(zLocalFile);
  const size_t fileDataBegin =
      records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;

  if (cache) {
    cache->AddFile(curFileName, fileDataBegin, curFileSize);
    cache->meta.zipCRC = crc32b(
        cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);
  } else {
    fileOffsets.push_back(fileDataBegin);
  }

  records.Pop();

  ZIPFile zFile{};
  zFile.id = ZIPFile::ID;
  zFile.madeBy = 10;
  zFile.extractVersion = 10;
  zFile.lastModFileDate = zLocalFile.lastModFileDate;
  zFile.lastModFileTime = zLocalFile.lastModFileTime;
  zFile.compression = ZIPCompressionMethod::Store;
  zFile.compressedSize = zLocalFile.compressedSize;
  zFile.uncompressedSize = zLocalFile.uncompressedSize;
  zFile.fileNameSize = zLocalFile.fileNameSize;
  zFile.crc = zLocalFile.crc;
  const bool useFileExtendedData =
      SafeCast(zFile.localHeaderOffset, curLocalFileOffset);

  const bool useFileExtra = useFileExtendedData || useLocalExtendedData;
  ZIP64Extra extra;

  if (useFileExtra) {
    zFile.extraFieldSize = 4;

    if (useLocalExtendedData) {
      extra.uncompressedSize = curFileSize;
      extra.compressedSize = curFileSize;
      zFile.extraFieldSize += 16;
    }

    if (useFileExtendedData) {
      extra.localHeaderOffset = curLocalFileOffset;
      zFile.extraFieldSize += 8;
    }
  }

  if (final && cache) {
    zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;
  }

  entries.Write(zFile);
  entries.WriteContainer(prefixPath);
  entries.WriteContainer(curFileName);

  if (useFileExtra) {
    entries.Write(extra);
  }
}

inline std::tm localtime(std::time_t t) {
#ifdef _MSC_VER
  return *std::localtime(&t);
#else
  std::tm temp;
  localtime_r(&t, &temp);
  return temp;
#endif
}

void ZIPExtactContext::NewFile(const std::string &path) {
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo pathInfo(path);
  auto pathSv = pathInfo.GetFullPath();
  if (!curFileName.empty()) {
    FinishFile();
  }

  time_t curTime =
      std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
  std::tm ts = localtime(curTime);

  struct {
    uint16 day : 5, month : 4, year : 7;
  } dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};

  struct {
    uint16 second : 5, minute : 6, hour : 5;
  } dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};

  zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);
  zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);
  zLocalFile.compression = ZIPCompressionMethod::Store;
  zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();
  zLocalFile.crc = 0;
  curFileSize = 0;

  curFileName = pathSv;
  curLocalFileOffset = records.Tell();
  records.Write(zLocalFile);
  records.WriteContainer(prefixPath);
  records.WriteContainer(pathSv);

  if (forEachFile) {
    forEachFile();
  }
}

void ZIPExtactContext::SendData(std::string_view data) {
  curFileSize += data.size();
  zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());
  records.WriteContainer(data);
}

bool ZIPExtactContext::RequiresFolders() const { return false; }

void ZIPExtactContext::AddFolderPath(const std::string &) {
  throw std::logic_error(
      "AddFolderPath not supported, use RequiresFolders to check.");
}

void ZIPExtactContext::GenerateFolders() {
  throw std::logic_error(
      "GenerateFolders not supported, use RequiresFolders to check.");
}

void IOExtractContext::NewFile(const std::string &path) {
  Close_();
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo cfleWrap(path);
  std::string cfle(cfleWrap.GetFullPath());
  Open(outDir + cfle);

  if (forEachFile) {
    forEachFile();
  }
}

void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }

bool IOExtractContext::RequiresFolders() const { return true; }

void IOExtractContext::AddFolderPath(const std::string &path) {
  AFileInfo cfleWrap(path);
  auto cfle = cfleWrap.GetFullPath();

  for (auto it = cfle.begin(); it != cfle.end(); it++) {
    if (*it == '/') {
      folderTree.emplace(cfle.begin(), it);
    }
  }

  folderTree.emplace(path);
}

void IOExtractContext::GenerateFolders() {
  for (auto &f : folderTree) {
    auto genFolder = outDir + f;
    es::mkdir(genFolder);
  }

  folderTree.clear();
}

static std::mutex ZIPLock;

void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {
  if (!other.curFileName.empty()) {
    other.FinishFile();
  }

  BinReaderRef localEntries(other.entriesStream);
  char buffer[0x80000];
  std::lock_guard<std::mutex> guard(ZIPLock);
  const size_t filesSize = records.Tell();

  numEntries += other.numEntries;

  for (auto o : other.fileOffsets) {
    ZIPFile zFile;
    localEntries.Read(zFile);

    const size_t localHeaderOffset = zFile.localHeaderOffset;
    const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&
                               localHeaderOffset + filesSize >= 0xffffffff;
    bool newExtra = false;

    if (newExtraField) {
      zFile.localHeaderOffset = 0xffffffff;

      if (!zFile.extraFieldSize) {
        zFile.extraFieldSize = 12;
        newExtra = true;
      } else {
        zFile.extraFieldSize += 8;
      }
    }

    if (zFile.localHeaderOffset < 0xffffffff) {
      zFile.localHeaderOffset += filesSize;
    }

    entries.Write(zFile);
    localEntries.ReadBuffer(buffer, zFile.fileNameSize);
    cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,
                  zFile.uncompressedSize);
    cache.meta.zipCRC = crc32b(cache.meta.zipCRC,
                               reinterpret_cast<const char *>(&zFile.crc), 4);
    entries.WriteBuffer(buffer, zFile.fileNameSize);

    if (newExtra) {
      ZIP64Extra extra{};
      extra.localHeaderOffset = localHeaderOffset + filesSize;
      entries.Write(extra);
    } else if (zFile.extraFieldSize) {
      ZIP64Extra extra{};
      localEntries.Read(extra.id);
      localEntries.Read(extra.size);

      if (zFile.compressedSize == 0xffffffff) {
        localEntries.Read(extra.compressedSize);
        localEntries.Read(extra.uncompressedSize);
      }

      if (zFile.localHeaderOffset == 0xffffffff) {
        if (newExtraField) {
          extra.localHeaderOffset = localHeaderOffset + filesSize;
        } else {
          localEntries.Read(extra.localHeaderOffset);
          extra.localHeaderOffset += filesSize;
        }
      }

      entries.Write(extra);
    }
  }

  es::Dispose(other.entriesStream);

  const size_t recordsSize = other.records.Tell();
  es::Dispose(other.records);
  BinReader rd(recordsFile);
  const size_t numBlocks = recordsSize / sizeof(buffer);
  const size_t restBytes = recordsSize % sizeof(buffer);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }
}

void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {
  size_t entriesSize = entries.Tell();
  es::Dispose(entries);
  char buffer[0x80000];
  BinReader rd(entriesFile);
  const size_t numBlocks = entriesSize / sizeof(buffer);
  const size_t restBytes = entriesSize % sizeof(buffer);
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }

  bool validCacheEntry = false;

  {
    // Find last zipfile entry and modify extraFieldSize
    const size_t skipValue = std::min(entriesSize, size_t(0x11000));
    rd.Skip(-skipValue);
    rd.ReadBuffer(buffer, skipValue);
    std::string_view sv(buffer, skipValue);
    size_t foundLastEntry = sv.rfind("PK\x01\x02");
    validCacheEntry = foundLastEntry != sv.npos;

    if (validCacheEntry) {
      foundLastEntry += offsetof(ZIPFile, extraFieldSize);
      char *data = buffer + foundLastEntry;
      uint16 extraFieldSize = *reinterpret_cast<uint16 *>(data);
      records.Push();
      records.Skip(-(skipValue - foundLastEntry));
      records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));
      records.Pop();

      records.Write<uint16>(0x4353);
      records.Write<uint16>(sizeof(CacheBaseHeader));
      cache.meta.zipCheckupOffset = records.Tell();
      records.Write(cache.meta);
      entriesSize += sizeof(CacheBaseHeader) + 4;
    }
  }

  SafeCast(zCentral.dirSize, entriesSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = entriesSize;
    zCentral64.dirOffset = dirOffset;
    zCentral64.dirRecord = 0x2C;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  es::Dispose(rd);
  es::RemoveFile(entriesFile);

  if (validCacheEntry) {
    cacheBeginCB();
    cache.meta.zipSize = records.Tell();
    BinWritter cacheWr(outFile + ".cache");
    cache.WaitAndWrite(cacheWr);
    records.Seek(cache.meta.zipCheckupOffset);
    records.Write(cache.meta);
  }
}

1	/* Spike is universal dedicated module handler
2	This souce contains data output context
3	Part of PreCore project
4
5	Copyright 2021-2022 Lukas Cone
6
7	Licensed under the Apache License, Version 2.0 (the "License");
8	you may not use this file except in compliance with the License.
9	You may obtain a copy of the License at
10
11	http://www.apache.org/licenses/LICENSE-2.0
12
13	Unless required by applicable law or agreed to in writing, software
14	distributed under the License is distributed on an "AS IS" BASIS,
15	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16	See the License for the specific language governing permissions and
17	limitations under the License.
18	*/
19
20	#include "spike/app/out_context.hpp"
21	#include "spike/app/console.hpp"
22	#include "spike/crypto/crc32.hpp"
23	#include "spike/format/ZIP_istream.inl"
24	#include "spike/format/ZIP_ostream.inl"
25	#include "spike/io/binreader.hpp"
26	#include "spike/io/fileinfo.hpp"
27	#include "spike/io/stat.hpp"
28	#include <chrono>
29	#include <mutex>
30
31	void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {	×
32	FinishFile(true);	×
33
34	auto entriesStr = std::move(entriesStream).str();
35	bool forcex64 = false;	×
36	const size_t dirOffset = records.Tell();
37	size_t dirSize = entriesStr.size();
38
39	auto SafeCast = [&](auto &where, auto &&what) {
40	const uint64 limit =
41	std::numeric_limits<std::decay_t<decltype(where)>>::max();
42	if (what >= limit) {	×
43	forcex64 = true;	×
44	where = limit;	×
45	} else {
46	where = what;	×
47	}
48	};
49
50	ZIPCentralDir zCentral{};	×
51	zCentral.id = ZIPCentralDir::ID;	×
52	SafeCast(zCentral.numDirEntries, numEntries);
53	SafeCast(zCentral.numDiskEntries, numEntries);
54	SafeCast(zCentral.dirOffset, dirOffset);
55
56	records.WriteContainer(entriesStr);
57	es::Dispose(entriesStr);	×
58
59	if (cache) {	×
60	records.Write<uint16>(0x4353);	×
61	records.Write<uint16>(sizeof(CacheBaseHeader));	×
62	cache->meta.zipCheckupOffset = records.Tell();	×
63	records.Write(cache->meta);	×
64	dirSize += sizeof(CacheBaseHeader) + 4;	×
65	}
66
67	SafeCast(zCentral.dirSize, dirSize);
68
69	if (forcex64) {	×
70	ZIP64CentralDir zCentral64{};	×
71	zCentral64.id = ZIP64CentralDir::ID;	×
72	zCentral64.madeBy = 10;	×
73	zCentral64.extractVersion = 10;	×
74	zCentral64.dirRecord = 0x2C;	×
75	zCentral64.numDiskEntries = numEntries;	×
76	zCentral64.numDirEntries = numEntries;	×
77	zCentral64.dirSize = dirSize;	×
78	zCentral64.dirOffset = dirOffset;	×
79
80	const size_t centralOffset = records.Tell();
81	records.Write(zCentral64);	×
82
83	ZIP64CentralDirLocator zLoca{};	×
84	zLoca.id = ZIP64CentralDirLocator::ID;	×
85	zLoca.centralDirOffset = centralOffset;	×
86	records.Write(zLoca);
87	}
88
89	records.Write(zCentral);
90
91	if (cache) {	×
92	cacheBeginCB();	×
93	cache->meta.zipSize = records.Tell();	×
94	BinWritter cacheWr(outputFile + ".cache");	×
95	cache->WaitAndWrite(cacheWr);	×
96	records.Seek(cache->meta.zipCheckupOffset);	×
97	records.Write(cache->meta);	×
98	}
99	}
100
101	void ZIPExtactContext::FinishFile(bool final) {	×
102	auto SafeCast = [&](auto &where, auto &&what) {
103	const uint64 limit =
104	std::numeric_limits<std::decay_t<decltype(where)>>::max();
105	bool forcex64 = false;
106
107	if (what >= limit) {	×
108	forcex64 = true;
109	where = limit;	×
110	} else {
111	where = what;	×
112	}
113
114	return forcex64;
115	};
116
117	const bool useLocalExtendedData =
118	SafeCast(zLocalFile.uncompressedSize, curFileSize);
119	zLocalFile.compressedSize = zLocalFile.uncompressedSize;	×
120
121	if (useLocalExtendedData) {	×
122	ZIP64Extra extra;
123	extra.compressedSize = curFileSize;	×
124	extra.uncompressedSize = curFileSize;	×
125	records.Write(extra);	×
126	zLocalFile.extraFieldSize = 20;	×
127	}
128
129	numEntries++;	×
130	records.Push();
131	records.Seek(curLocalFileOffset);	×
132	records.Write(zLocalFile);	×
133	const size_t fileDataBegin =
134	records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;	×
135
136	if (cache) {	×
137	cache->AddFile(curFileName, fileDataBegin, curFileSize);	×
138	cache->meta.zipCRC = crc32b(	×
139	cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);	×
140	} else {
141	fileOffsets.push_back(fileDataBegin);	×
142	}
143
144	records.Pop();
145
146	ZIPFile zFile{};	×
147	zFile.id = ZIPFile::ID;	×
148	zFile.madeBy = 10;	×
149	zFile.extractVersion = 10;	×
150	zFile.lastModFileDate = zLocalFile.lastModFileDate;	×
151	zFile.lastModFileTime = zLocalFile.lastModFileTime;	×
152	zFile.compression = ZIPCompressionMethod::Store;
153	zFile.compressedSize = zLocalFile.compressedSize;	×
154	zFile.uncompressedSize = zLocalFile.uncompressedSize;	×
155	zFile.fileNameSize = zLocalFile.fileNameSize;	×
156	zFile.crc = zLocalFile.crc;	×
157	const bool useFileExtendedData =
158	SafeCast(zFile.localHeaderOffset, curLocalFileOffset);
159
160	const bool useFileExtra = useFileExtendedData \|\| useLocalExtendedData;	×
161	ZIP64Extra extra;
162
163	if (useFileExtra) {	×
164	zFile.extraFieldSize = 4;	×
165
166	if (useLocalExtendedData) {	×
167	extra.uncompressedSize = curFileSize;	×
168	extra.compressedSize = curFileSize;	×
169	zFile.extraFieldSize += 16;	×
170	}
171
172	if (useFileExtendedData) {	×
173	extra.localHeaderOffset = curLocalFileOffset;	×
174	zFile.extraFieldSize += 8;	×
175	}
176	}
177
178	if (final && cache) {	×
179	zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;	×
180	}
181
182	entries.Write(zFile);	×
183	entries.WriteContainer(prefixPath);
184	entries.WriteContainer(curFileName);
185
186	if (useFileExtra) {	×
187	entries.Write(extra);
188	}
189	}
190
191	inline std::tm localtime(std::time_t t) {
192	#ifdef _MSC_VER
193	return *std::localtime(&t);
194	#else
195	std::tm temp;
196	localtime_r(&t, &temp);	×
197	return temp;
198	#endif
199	}
200
201	void ZIPExtactContext::NewFile(const std::string &path) {	×
202	if (path.empty()) [[unlikely]] {	×
203	throw std::runtime_error("NewFile path is empty");	×
204	}
205	AFileInfo pathInfo(path);	×
206	auto pathSv = pathInfo.GetFullPath();
207	if (!curFileName.empty()) {	×
208	FinishFile();	×
209	}
210
211	time_t curTime =
212	std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());	×
213	std::tm ts = localtime(curTime);	×
214
215	struct {
216	uint16 day : 5, month : 4, year : 7;
217	} dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};	×
218
219	struct {
220	uint16 second : 5, minute : 6, hour : 5;
221	} dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};	×
222
223	zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);	×
224	zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);	×
225	zLocalFile.compression = ZIPCompressionMethod::Store;	×
226	zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();	×
227	zLocalFile.crc = 0;	×
228	curFileSize = 0;	×
229
230	curFileName = pathSv;	×
231	curLocalFileOffset = records.Tell();	×
232	records.Write(zLocalFile);	×
233	records.WriteContainer(prefixPath);
234	records.WriteContainer(pathSv);
235
236	if (forEachFile) {	×
237	forEachFile();
238	}
239	}
240
241	void ZIPExtactContext::SendData(std::string_view data) {	×
242	curFileSize += data.size();	×
243	zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());	×
244	records.WriteContainer(data);
245	}
246
247	bool ZIPExtactContext::RequiresFolders() const { return false; }	×
248
249	void ZIPExtactContext::AddFolderPath(const std::string &) {	×
250	throw std::logic_error(	×
251	"AddFolderPath not supported, use RequiresFolders to check.");	×
252	}
253
254	void ZIPExtactContext::GenerateFolders() {	×
255	throw std::logic_error(	×
256	"GenerateFolders not supported, use RequiresFolders to check.");	×
257	}
258
259	void IOExtractContext::NewFile(const std::string &path) {	×
260	Close_();	×
261	if (path.empty()) [[unlikely]] {	×
262	throw std::runtime_error("NewFile path is empty");	×
263	}
264	AFileInfo cfleWrap(path);	×
265	std::string cfle(cfleWrap.GetFullPath());	×
266	Open(outDir + cfle);	×
267
268	if (forEachFile) {	×
269	forEachFile();
270	}
271	}
272
273	void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }	×
274
275	bool IOExtractContext::RequiresFolders() const { return true; }	×
276
277	void IOExtractContext::AddFolderPath(const std::string &path) {	×
278	AFileInfo cfleWrap(path);	×
279	auto cfle = cfleWrap.GetFullPath();
280
281	for (auto it = cfle.begin(); it != cfle.end(); it++) {	×
282	if (*it == '/') {	×
283	folderTree.emplace(cfle.begin(), it);	×
284	}
285	}
286
287	folderTree.emplace(path);
288	}
289
290	void IOExtractContext::GenerateFolders() {	×
291	for (auto &f : folderTree) {	×
292	auto genFolder = outDir + f;	×
293	es::mkdir(genFolder);
294	}
295
296	folderTree.clear();
297	}
298
299	static std::mutex ZIPLock;
300
301	void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {	×
302	if (!other.curFileName.empty()) {	×
303	other.FinishFile();	×
304	}
305
306	BinReaderRef localEntries(other.entriesStream);	×
307	char buffer[0x80000];
308	std::lock_guard<std::mutex> guard(ZIPLock);
309	const size_t filesSize = records.Tell();
310
311	numEntries += other.numEntries;	×
312
313	for (auto o : other.fileOffsets) {	×
314	ZIPFile zFile;
315	localEntries.Read(zFile);	×
316
317	const size_t localHeaderOffset = zFile.localHeaderOffset;	×
318	const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&	×
319	localHeaderOffset + filesSize >= 0xffffffff;	×
320	bool newExtra = false;
321
322	if (newExtraField) {
323	zFile.localHeaderOffset = 0xffffffff;	×
324
325	if (!zFile.extraFieldSize) {	×
326	zFile.extraFieldSize = 12;	×
327	newExtra = true;
328	} else {
329	zFile.extraFieldSize += 8;	×
330	}
331	}
332
333	if (zFile.localHeaderOffset < 0xffffffff) {	×
334	zFile.localHeaderOffset += filesSize;	×
335	}
336
337	entries.Write(zFile);	×
338	localEntries.ReadBuffer(buffer, zFile.fileNameSize);	×
339	cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,	×
340	zFile.uncompressedSize);	×
341	cache.meta.zipCRC = crc32b(cache.meta.zipCRC,	×
342	reinterpret_cast<const char *>(&zFile.crc), 4);
343	entries.WriteBuffer(buffer, zFile.fileNameSize);	×
344
345	if (newExtra) {	×
346	ZIP64Extra extra{};	×
347	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
348	entries.Write(extra);
349	} else if (zFile.extraFieldSize) {	×
350	ZIP64Extra extra{};	×
351	localEntries.Read(extra.id);
352	localEntries.Read(extra.size);
353
354	if (zFile.compressedSize == 0xffffffff) {	×
355	localEntries.Read(extra.compressedSize);
356	localEntries.Read(extra.uncompressedSize);
357	}
358
359	if (zFile.localHeaderOffset == 0xffffffff) {	×
360	if (newExtraField) {	×
361	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
362	} else {
363	localEntries.Read(extra.localHeaderOffset);
364	extra.localHeaderOffset += filesSize;	×
365	}
366	}
367
368	entries.Write(extra);
369	}
370	}
371
372	es::Dispose(other.entriesStream);	×
373
374	const size_t recordsSize = other.records.Tell();
375	es::Dispose(other.records);	×
376	BinReader rd(recordsFile);	×
377	const size_t numBlocks = recordsSize / sizeof(buffer);	×
378	const size_t restBytes = recordsSize % sizeof(buffer);	×
379
380	for (size_t b = 0; b < numBlocks; b++) {	×
381	rd.ReadBuffer(buffer, sizeof(buffer));
382	records.WriteBuffer(buffer, sizeof(buffer));
383	}
384
385	if (restBytes) {	×
386	rd.ReadBuffer(buffer, restBytes);
387	records.WriteBuffer(buffer, restBytes);
388	}
389	}
390
391	void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {	×
392	size_t entriesSize = entries.Tell();	×
393	es::Dispose(entries);	×
394	char buffer[0x80000];
395	BinReader rd(entriesFile);	×
396	const size_t numBlocks = entriesSize / sizeof(buffer);	×
397	const size_t restBytes = entriesSize % sizeof(buffer);	×
398	bool forcex64 = false;	×
399	const size_t dirOffset = records.Tell();
400
401	auto SafeCast = [&](auto &where, auto &&what) {
402	const uint64 limit =
403	std::numeric_limits<std::decay_t<decltype(where)>>::max();
404	if (what >= limit) {	×
405	forcex64 = true;	×
406	where = limit;	×
407	} else {
408	where = what;	×
409	}
410	};
411
412	ZIPCentralDir zCentral{};	×
413	zCentral.id = ZIPCentralDir::ID;	×
414	SafeCast(zCentral.numDirEntries, numEntries);
415	SafeCast(zCentral.numDiskEntries, numEntries);
416	SafeCast(zCentral.dirOffset, dirOffset);
417
418	for (size_t b = 0; b < numBlocks; b++) {	×
419	rd.ReadBuffer(buffer, sizeof(buffer));
420	records.WriteBuffer(buffer, sizeof(buffer));
421	}
422
423	if (restBytes) {	×
424	rd.ReadBuffer(buffer, restBytes);
425	records.WriteBuffer(buffer, restBytes);
426	}
427
428	bool validCacheEntry = false;
429
430	{
431	// Find last zipfile entry and modify extraFieldSize
432	const size_t skipValue = std::min(entriesSize, size_t(0x11000));	×
433	rd.Skip(-skipValue);	×
434	rd.ReadBuffer(buffer, skipValue);
435	std::string_view sv(buffer, skipValue);
436	size_t foundLastEntry = sv.rfind("PK\x01\x02");
437	validCacheEntry = foundLastEntry != sv.npos;
438
439	if (validCacheEntry) {	×
440	foundLastEntry += offsetof(ZIPFile, extraFieldSize);	×
441	char *data = buffer + foundLastEntry;	×
442	uint16 extraFieldSize = reinterpret_cast<uint16 >(data);	×
443	records.Push();
444	records.Skip(-(skipValue - foundLastEntry));	×
445	records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));	×
446	records.Pop();
447
448	records.Write<uint16>(0x4353);	×
449	records.Write<uint16>(sizeof(CacheBaseHeader));	×
450	cache.meta.zipCheckupOffset = records.Tell();	×
451	records.Write(cache.meta);	×
452	entriesSize += sizeof(CacheBaseHeader) + 4;	×
453	}
454	}
455
456	SafeCast(zCentral.dirSize, entriesSize);
457
458	if (forcex64) {	×
459	ZIP64CentralDir zCentral64{};	×
460	zCentral64.id = ZIP64CentralDir::ID;	×
461	zCentral64.madeBy = 10;	×
462	zCentral64.extractVersion = 10;	×
463	zCentral64.numDiskEntries = numEntries;	×
464	zCentral64.numDirEntries = numEntries;	×
465	zCentral64.dirSize = entriesSize;	×
466	zCentral64.dirOffset = dirOffset;	×
467	zCentral64.dirRecord = 0x2C;	×
468
469	const size_t centralOffset = records.Tell();
470	records.Write(zCentral64);	×
471
472	ZIP64CentralDirLocator zLoca{};	×
473	zLoca.id = ZIP64CentralDirLocator::ID;	×
474	zLoca.centralDirOffset = centralOffset;	×
475	records.Write(zLoca);
476	}
477
478	records.Write(zCentral);
479
480	es::Dispose(rd);	×
481	es::RemoveFile(entriesFile);	×
482
483	if (validCacheEntry) {	×
484	cacheBeginCB();	×
485	cache.meta.zipSize = records.Tell();	×
486	BinWritter cacheWr(outFile + ".cache");	×
487	cache.WaitAndWrite(cacheWr);	×
488	records.Seek(cache.meta.zipCheckupOffset);	×
489	records.Write(cache.meta);	×
490	}
491	}

PredatorCZ / PreCore / 460

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous