PredatorCZ / PreCore / 461

pending completion

Build # 461

Build Type

push

github-actions-ci

Committed by

PredatorCZ

Commit Message

update readme

Run Details

3204 of 6096 relevant lines covered (52.56%)

354.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/app/out_context.cpp

/*  Spike is universal dedicated module handler
    This souce contains data output context

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "spike/app/out_context.hpp"
#include "spike/app/console.hpp"
#include "spike/crypto/crc32.hpp"
#include "spike/format/ZIP_istream.inl"
#include "spike/format/ZIP_ostream.inl"
#include "spike/io/binreader.hpp"
#include "spike/io/fileinfo.hpp"
#include "spike/io/stat.hpp"
#include <chrono>
#include <mutex>

void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {
  FinishFile(true);

  auto entriesStr = std::move(entriesStream).str();
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();
  size_t dirSize = entriesStr.size();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  records.WriteContainer(entriesStr);
  es::Dispose(entriesStr);

  if (cache) {
    records.Write<uint16>(0x4353);
    records.Write<uint16>(sizeof(CacheBaseHeader));
    cache->meta.zipCheckupOffset = records.Tell();
    records.Write(cache->meta);
    dirSize += sizeof(CacheBaseHeader) + 4;
  }

  SafeCast(zCentral.dirSize, dirSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.dirRecord = 0x2C;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = dirSize;
    zCentral64.dirOffset = dirOffset;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  if (cache) {
    cacheBeginCB();
    cache->meta.zipSize = records.Tell();
    BinWritter cacheWr(outputFile + ".cache");
    cache->WaitAndWrite(cacheWr);
    records.Seek(cache->meta.zipCheckupOffset);
    records.Write(cache->meta);
  }
}

void ZIPExtactContext::FinishFile(bool final) {
  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    bool forcex64 = false;

    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }

    return forcex64;
  };

  const bool useLocalExtendedData =
      SafeCast(zLocalFile.uncompressedSize, curFileSize);
  zLocalFile.compressedSize = zLocalFile.uncompressedSize;

  if (useLocalExtendedData) {
    ZIP64Extra extra;
    extra.compressedSize = curFileSize;
    extra.uncompressedSize = curFileSize;
    records.Write(extra);
    zLocalFile.extraFieldSize = 20;
  }

  numEntries++;
  records.Push();
  records.Seek(curLocalFileOffset);
  records.Write(zLocalFile);
  const size_t fileDataBegin =
      records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;

  if (cache) {
    cache->AddFile(curFileName, fileDataBegin, curFileSize);
    cache->meta.zipCRC = crc32b(
        cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);
  } else {
    fileOffsets.push_back(fileDataBegin);
  }

  records.Pop();

  ZIPFile zFile{};
  zFile.id = ZIPFile::ID;
  zFile.madeBy = 10;
  zFile.extractVersion = 10;
  zFile.lastModFileDate = zLocalFile.lastModFileDate;
  zFile.lastModFileTime = zLocalFile.lastModFileTime;
  zFile.compression = ZIPCompressionMethod::Store;
  zFile.compressedSize = zLocalFile.compressedSize;
  zFile.uncompressedSize = zLocalFile.uncompressedSize;
  zFile.fileNameSize = zLocalFile.fileNameSize;
  zFile.crc = zLocalFile.crc;
  const bool useFileExtendedData =
      SafeCast(zFile.localHeaderOffset, curLocalFileOffset);

  const bool useFileExtra = useFileExtendedData || useLocalExtendedData;
  ZIP64Extra extra;

  if (useFileExtra) {
    zFile.extraFieldSize = 4;

    if (useLocalExtendedData) {
      extra.uncompressedSize = curFileSize;
      extra.compressedSize = curFileSize;
      zFile.extraFieldSize += 16;
    }

    if (useFileExtendedData) {
      extra.localHeaderOffset = curLocalFileOffset;
      zFile.extraFieldSize += 8;
    }
  }

  if (final && cache) {
    zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;
  }

  entries.Write(zFile);
  entries.WriteContainer(prefixPath);
  entries.WriteContainer(curFileName);

  if (useFileExtra) {
    entries.Write(extra);
  }
}

inline std::tm localtime(std::time_t t) {
#ifdef _MSC_VER
  return *std::localtime(&t);
#else
  std::tm temp;
  localtime_r(&t, &temp);
  return temp;
#endif
}

void ZIPExtactContext::NewFile(const std::string &path) {
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo pathInfo(path);
  auto pathSv = pathInfo.GetFullPath();
  if (!curFileName.empty()) {
    FinishFile();
  }

  time_t curTime =
      std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
  std::tm ts = localtime(curTime);

  struct {
    uint16 day : 5, month : 4, year : 7;
  } dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};

  struct {
    uint16 second : 5, minute : 6, hour : 5;
  } dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};

  zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);
  zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);
  zLocalFile.compression = ZIPCompressionMethod::Store;
  zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();
  zLocalFile.crc = 0;
  curFileSize = 0;

  curFileName = pathSv;
  curLocalFileOffset = records.Tell();
  records.Write(zLocalFile);
  records.WriteContainer(prefixPath);
  records.WriteContainer(pathSv);

  if (forEachFile) {
    forEachFile();
  }
}

void ZIPExtactContext::SendData(std::string_view data) {
  curFileSize += data.size();
  zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());
  records.WriteContainer(data);
}

bool ZIPExtactContext::RequiresFolders() const { return false; }

void ZIPExtactContext::AddFolderPath(const std::string &) {
  throw std::logic_error(
      "AddFolderPath not supported, use RequiresFolders to check.");
}

void ZIPExtactContext::GenerateFolders() {
  throw std::logic_error(
      "GenerateFolders not supported, use RequiresFolders to check.");
}

void IOExtractContext::NewFile(const std::string &path) {
  Close_();
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo cfleWrap(path);
  std::string cfle(cfleWrap.GetFullPath());
  Open(outDir + cfle);

  if (forEachFile) {
    forEachFile();
  }
}

void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }

bool IOExtractContext::RequiresFolders() const { return true; }

void IOExtractContext::AddFolderPath(const std::string &path) {
  AFileInfo cfleWrap(path);
  auto cfle = cfleWrap.GetFullPath();

  for (auto it = cfle.begin(); it != cfle.end(); it++) {
    if (*it == '/') {
      folderTree.emplace(cfle.begin(), it);
    }
  }

  folderTree.emplace(path);
}

void IOExtractContext::GenerateFolders() {
  for (auto &f : folderTree) {
    auto genFolder = outDir + f;
    es::mkdir(genFolder);
  }

  folderTree.clear();
}

static std::mutex ZIPLock;

void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {
  if (!other.curFileName.empty()) {
    other.FinishFile();
  }

  BinReaderRef localEntries(other.entriesStream);
  char buffer[0x80000];
  std::lock_guard<std::mutex> guard(ZIPLock);
  const size_t filesSize = records.Tell();

  numEntries += other.numEntries;

  for (auto o : other.fileOffsets) {
    ZIPFile zFile;
    localEntries.Read(zFile);

    const size_t localHeaderOffset = zFile.localHeaderOffset;
    const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&
                               localHeaderOffset + filesSize >= 0xffffffff;
    bool newExtra = false;

    if (newExtraField) {
      zFile.localHeaderOffset = 0xffffffff;

      if (!zFile.extraFieldSize) {
        zFile.extraFieldSize = 12;
        newExtra = true;
      } else {
        zFile.extraFieldSize += 8;
      }
    }

    if (zFile.localHeaderOffset < 0xffffffff) {
      zFile.localHeaderOffset += filesSize;
    }

    entries.Write(zFile);
    localEntries.ReadBuffer(buffer, zFile.fileNameSize);
    cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,
                  zFile.uncompressedSize);
    cache.meta.zipCRC = crc32b(cache.meta.zipCRC,
                               reinterpret_cast<const char *>(&zFile.crc), 4);
    entries.WriteBuffer(buffer, zFile.fileNameSize);

    if (newExtra) {
      ZIP64Extra extra{};
      extra.localHeaderOffset = localHeaderOffset + filesSize;
      entries.Write(extra);
    } else if (zFile.extraFieldSize) {
      ZIP64Extra extra{};
      localEntries.Read(extra.id);
      localEntries.Read(extra.size);

      if (zFile.compressedSize == 0xffffffff) {
        localEntries.Read(extra.compressedSize);
        localEntries.Read(extra.uncompressedSize);
      }

      if (zFile.localHeaderOffset == 0xffffffff) {
        if (newExtraField) {
          extra.localHeaderOffset = localHeaderOffset + filesSize;
        } else {
          localEntries.Read(extra.localHeaderOffset);
          extra.localHeaderOffset += filesSize;
        }
      }

      entries.Write(extra);
    }
  }

  es::Dispose(other.entriesStream);

  const size_t recordsSize = other.records.Tell();
  es::Dispose(other.records);
  BinReader rd(recordsFile);
  const size_t numBlocks = recordsSize / sizeof(buffer);
  const size_t restBytes = recordsSize % sizeof(buffer);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }
}

void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {
  size_t entriesSize = entries.Tell();
  es::Dispose(entries);
  char buffer[0x80000];
  BinReader rd(entriesFile);
  const size_t numBlocks = entriesSize / sizeof(buffer);
  const size_t restBytes = entriesSize % sizeof(buffer);
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }

  bool validCacheEntry = false;

  {
    // Find last zipfile entry and modify extraFieldSize
    const size_t skipValue = std::min(entriesSize, size_t(0x11000));
    rd.Skip(-skipValue);
    rd.ReadBuffer(buffer, skipValue);
    std::string_view sv(buffer, skipValue);
    size_t foundLastEntry = sv.rfind("PK\x01\x02");
    validCacheEntry = foundLastEntry != sv.npos;

    if (validCacheEntry) {
      foundLastEntry += offsetof(ZIPFile, extraFieldSize);
      char *data = buffer + foundLastEntry;
      uint16 extraFieldSize = *reinterpret_cast<uint16 *>(data);
      records.Push();
      records.Skip(-(skipValue - foundLastEntry));
      records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));
      records.Pop();

      records.Write<uint16>(0x4353);
      records.Write<uint16>(sizeof(CacheBaseHeader));
      cache.meta.zipCheckupOffset = records.Tell();
      records.Write(cache.meta);
      entriesSize += sizeof(CacheBaseHeader) + 4;
    }
  }

  SafeCast(zCentral.dirSize, entriesSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = entriesSize;
    zCentral64.dirOffset = dirOffset;
    zCentral64.dirRecord = 0x2C;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  es::Dispose(rd);
  es::RemoveFile(entriesFile);

  if (validCacheEntry) {
    cacheBeginCB();
    cache.meta.zipSize = records.Tell();
    BinWritter cacheWr(outFile + ".cache");
    cache.WaitAndWrite(cacheWr);
    records.Seek(cache.meta.zipCheckupOffset);
    records.Write(cache.meta);
  }
}

1	/* Spike is universal dedicated module handler
2	This souce contains data output context
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "spike/app/out_context.hpp"
20	#include "spike/app/console.hpp"
21	#include "spike/crypto/crc32.hpp"
22	#include "spike/format/ZIP_istream.inl"
23	#include "spike/format/ZIP_ostream.inl"
24	#include "spike/io/binreader.hpp"
25	#include "spike/io/fileinfo.hpp"
26	#include "spike/io/stat.hpp"
27	#include <chrono>
28	#include <mutex>
29
30	void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {	×
31	FinishFile(true);	×
32
33	auto entriesStr = std::move(entriesStream).str();
34	bool forcex64 = false;	×
35	const size_t dirOffset = records.Tell();
36	size_t dirSize = entriesStr.size();
37
38	auto SafeCast = [&](auto &where, auto &&what) {
39	const uint64 limit =
40	std::numeric_limits<std::decay_t<decltype(where)>>::max();
41	if (what >= limit) {	×
42	forcex64 = true;	×
43	where = limit;	×
44	} else {
45	where = what;	×
46	}
47	};
48
49	ZIPCentralDir zCentral{};	×
50	zCentral.id = ZIPCentralDir::ID;	×
51	SafeCast(zCentral.numDirEntries, numEntries);
52	SafeCast(zCentral.numDiskEntries, numEntries);
53	SafeCast(zCentral.dirOffset, dirOffset);
54
55	records.WriteContainer(entriesStr);
56	es::Dispose(entriesStr);	×
57
58	if (cache) {	×
59	records.Write<uint16>(0x4353);	×
60	records.Write<uint16>(sizeof(CacheBaseHeader));	×
61	cache->meta.zipCheckupOffset = records.Tell();	×
62	records.Write(cache->meta);	×
63	dirSize += sizeof(CacheBaseHeader) + 4;	×
64	}
65
66	SafeCast(zCentral.dirSize, dirSize);
67
68	if (forcex64) {	×
69	ZIP64CentralDir zCentral64{};	×
70	zCentral64.id = ZIP64CentralDir::ID;	×
71	zCentral64.madeBy = 10;	×
72	zCentral64.extractVersion = 10;	×
73	zCentral64.dirRecord = 0x2C;	×
74	zCentral64.numDiskEntries = numEntries;	×
75	zCentral64.numDirEntries = numEntries;	×
76	zCentral64.dirSize = dirSize;	×
77	zCentral64.dirOffset = dirOffset;	×
78
79	const size_t centralOffset = records.Tell();
80	records.Write(zCentral64);	×
81
82	ZIP64CentralDirLocator zLoca{};	×
83	zLoca.id = ZIP64CentralDirLocator::ID;	×
84	zLoca.centralDirOffset = centralOffset;	×
85	records.Write(zLoca);
86	}
87
88	records.Write(zCentral);
89
90	if (cache) {	×
91	cacheBeginCB();	×
92	cache->meta.zipSize = records.Tell();	×
93	BinWritter cacheWr(outputFile + ".cache");	×
94	cache->WaitAndWrite(cacheWr);	×
95	records.Seek(cache->meta.zipCheckupOffset);	×
96	records.Write(cache->meta);	×
97	}
98	}
99
100	void ZIPExtactContext::FinishFile(bool final) {	×
101	auto SafeCast = [&](auto &where, auto &&what) {
102	const uint64 limit =
103	std::numeric_limits<std::decay_t<decltype(where)>>::max();
104	bool forcex64 = false;
105
106	if (what >= limit) {	×
107	forcex64 = true;
108	where = limit;	×
109	} else {
110	where = what;	×
111	}
112
113	return forcex64;
114	};
115
116	const bool useLocalExtendedData =
117	SafeCast(zLocalFile.uncompressedSize, curFileSize);
118	zLocalFile.compressedSize = zLocalFile.uncompressedSize;	×
119
120	if (useLocalExtendedData) {	×
121	ZIP64Extra extra;
122	extra.compressedSize = curFileSize;	×
123	extra.uncompressedSize = curFileSize;	×
124	records.Write(extra);	×
125	zLocalFile.extraFieldSize = 20;	×
126	}
127
128	numEntries++;	×
129	records.Push();
130	records.Seek(curLocalFileOffset);	×
131	records.Write(zLocalFile);	×
132	const size_t fileDataBegin =
133	records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;	×
134
135	if (cache) {	×
136	cache->AddFile(curFileName, fileDataBegin, curFileSize);	×
137	cache->meta.zipCRC = crc32b(	×
138	cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);	×
139	} else {
140	fileOffsets.push_back(fileDataBegin);	×
141	}
142
143	records.Pop();
144
145	ZIPFile zFile{};	×
146	zFile.id = ZIPFile::ID;	×
147	zFile.madeBy = 10;	×
148	zFile.extractVersion = 10;	×
149	zFile.lastModFileDate = zLocalFile.lastModFileDate;	×
150	zFile.lastModFileTime = zLocalFile.lastModFileTime;	×
151	zFile.compression = ZIPCompressionMethod::Store;
152	zFile.compressedSize = zLocalFile.compressedSize;	×
153	zFile.uncompressedSize = zLocalFile.uncompressedSize;	×
154	zFile.fileNameSize = zLocalFile.fileNameSize;	×
155	zFile.crc = zLocalFile.crc;	×
156	const bool useFileExtendedData =
157	SafeCast(zFile.localHeaderOffset, curLocalFileOffset);
158
159	const bool useFileExtra = useFileExtendedData \|\| useLocalExtendedData;	×
160	ZIP64Extra extra;
161
162	if (useFileExtra) {	×
163	zFile.extraFieldSize = 4;	×
164
165	if (useLocalExtendedData) {	×
166	extra.uncompressedSize = curFileSize;	×
167	extra.compressedSize = curFileSize;	×
168	zFile.extraFieldSize += 16;	×
169	}
170
171	if (useFileExtendedData) {	×
172	extra.localHeaderOffset = curLocalFileOffset;	×
173	zFile.extraFieldSize += 8;	×
174	}
175	}
176
177	if (final && cache) {	×
178	zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;	×
179	}
180
181	entries.Write(zFile);	×
182	entries.WriteContainer(prefixPath);
183	entries.WriteContainer(curFileName);
184
185	if (useFileExtra) {	×
186	entries.Write(extra);
187	}
188	}
189
190	inline std::tm localtime(std::time_t t) {
191	#ifdef _MSC_VER
192	return *std::localtime(&t);
193	#else
194	std::tm temp;
195	localtime_r(&t, &temp);	×
196	return temp;
197	#endif
198	}
199
200	void ZIPExtactContext::NewFile(const std::string &path) {	×
201	if (path.empty()) [[unlikely]] {	×
202	throw std::runtime_error("NewFile path is empty");	×
203	}
204	AFileInfo pathInfo(path);	×
205	auto pathSv = pathInfo.GetFullPath();
206	if (!curFileName.empty()) {	×
207	FinishFile();	×
208	}
209
210	time_t curTime =
211	std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());	×
212	std::tm ts = localtime(curTime);	×
213
214	struct {
215	uint16 day : 5, month : 4, year : 7;
216	} dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};	×
217
218	struct {
219	uint16 second : 5, minute : 6, hour : 5;
220	} dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};	×
221
222	zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);	×
223	zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);	×
224	zLocalFile.compression = ZIPCompressionMethod::Store;	×
225	zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();	×
226	zLocalFile.crc = 0;	×
227	curFileSize = 0;	×
228
229	curFileName = pathSv;	×
230	curLocalFileOffset = records.Tell();	×
231	records.Write(zLocalFile);	×
232	records.WriteContainer(prefixPath);
233	records.WriteContainer(pathSv);
234
235	if (forEachFile) {	×
236	forEachFile();
237	}
238	}
239
240	void ZIPExtactContext::SendData(std::string_view data) {	×
241	curFileSize += data.size();	×
242	zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());	×
243	records.WriteContainer(data);
244	}
245
246	bool ZIPExtactContext::RequiresFolders() const { return false; }	×
247
248	void ZIPExtactContext::AddFolderPath(const std::string &) {	×
249	throw std::logic_error(	×
250	"AddFolderPath not supported, use RequiresFolders to check.");	×
251	}
252
253	void ZIPExtactContext::GenerateFolders() {	×
254	throw std::logic_error(	×
255	"GenerateFolders not supported, use RequiresFolders to check.");	×
256	}
257
258	void IOExtractContext::NewFile(const std::string &path) {	×
259	Close_();	×
260	if (path.empty()) [[unlikely]] {	×
261	throw std::runtime_error("NewFile path is empty");	×
262	}
263	AFileInfo cfleWrap(path);	×
264	std::string cfle(cfleWrap.GetFullPath());	×
265	Open(outDir + cfle);	×
266
267	if (forEachFile) {	×
268	forEachFile();
269	}
270	}
271
272	void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }	×
273
274	bool IOExtractContext::RequiresFolders() const { return true; }	×
275
276	void IOExtractContext::AddFolderPath(const std::string &path) {	×
277	AFileInfo cfleWrap(path);	×
278	auto cfle = cfleWrap.GetFullPath();
279
280	for (auto it = cfle.begin(); it != cfle.end(); it++) {	×
281	if (*it == '/') {	×
282	folderTree.emplace(cfle.begin(), it);	×
283	}
284	}
285
286	folderTree.emplace(path);
287	}
288
289	void IOExtractContext::GenerateFolders() {	×
290	for (auto &f : folderTree) {	×
291	auto genFolder = outDir + f;	×
292	es::mkdir(genFolder);
293	}
294
295	folderTree.clear();
296	}
297
298	static std::mutex ZIPLock;
299
300	void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {	×
301	if (!other.curFileName.empty()) {	×
302	other.FinishFile();	×
303	}
304
305	BinReaderRef localEntries(other.entriesStream);	×
306	char buffer[0x80000];
307	std::lock_guard<std::mutex> guard(ZIPLock);
308	const size_t filesSize = records.Tell();
309
310	numEntries += other.numEntries;	×
311
312	for (auto o : other.fileOffsets) {	×
313	ZIPFile zFile;
314	localEntries.Read(zFile);	×
315
316	const size_t localHeaderOffset = zFile.localHeaderOffset;	×
317	const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&	×
318	localHeaderOffset + filesSize >= 0xffffffff;	×
319	bool newExtra = false;
320
321	if (newExtraField) {
322	zFile.localHeaderOffset = 0xffffffff;	×
323
324	if (!zFile.extraFieldSize) {	×
325	zFile.extraFieldSize = 12;	×
326	newExtra = true;
327	} else {
328	zFile.extraFieldSize += 8;	×
329	}
330	}
331
332	if (zFile.localHeaderOffset < 0xffffffff) {	×
333	zFile.localHeaderOffset += filesSize;	×
334	}
335
336	entries.Write(zFile);	×
337	localEntries.ReadBuffer(buffer, zFile.fileNameSize);	×
338	cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,	×
339	zFile.uncompressedSize);	×
340	cache.meta.zipCRC = crc32b(cache.meta.zipCRC,	×
341	reinterpret_cast<const char *>(&zFile.crc), 4);
342	entries.WriteBuffer(buffer, zFile.fileNameSize);	×
343
344	if (newExtra) {	×
345	ZIP64Extra extra{};	×
346	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
347	entries.Write(extra);
348	} else if (zFile.extraFieldSize) {	×
349	ZIP64Extra extra{};	×
350	localEntries.Read(extra.id);
351	localEntries.Read(extra.size);
352
353	if (zFile.compressedSize == 0xffffffff) {	×
354	localEntries.Read(extra.compressedSize);
355	localEntries.Read(extra.uncompressedSize);
356	}
357
358	if (zFile.localHeaderOffset == 0xffffffff) {	×
359	if (newExtraField) {	×
360	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
361	} else {
362	localEntries.Read(extra.localHeaderOffset);
363	extra.localHeaderOffset += filesSize;	×
364	}
365	}
366
367	entries.Write(extra);
368	}
369	}
370
371	es::Dispose(other.entriesStream);	×
372
373	const size_t recordsSize = other.records.Tell();
374	es::Dispose(other.records);	×
375	BinReader rd(recordsFile);	×
376	const size_t numBlocks = recordsSize / sizeof(buffer);	×
377	const size_t restBytes = recordsSize % sizeof(buffer);	×
378
379	for (size_t b = 0; b < numBlocks; b++) {	×
380	rd.ReadBuffer(buffer, sizeof(buffer));
381	records.WriteBuffer(buffer, sizeof(buffer));
382	}
383
384	if (restBytes) {	×
385	rd.ReadBuffer(buffer, restBytes);
386	records.WriteBuffer(buffer, restBytes);
387	}
388	}
389
390	void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {	×
391	size_t entriesSize = entries.Tell();	×
392	es::Dispose(entries);	×
393	char buffer[0x80000];
394	BinReader rd(entriesFile);	×
395	const size_t numBlocks = entriesSize / sizeof(buffer);	×
396	const size_t restBytes = entriesSize % sizeof(buffer);	×
397	bool forcex64 = false;	×
398	const size_t dirOffset = records.Tell();
399
400	auto SafeCast = [&](auto &where, auto &&what) {
401	const uint64 limit =
402	std::numeric_limits<std::decay_t<decltype(where)>>::max();
403	if (what >= limit) {	×
404	forcex64 = true;	×
405	where = limit;	×
406	} else {
407	where = what;	×
408	}
409	};
410
411	ZIPCentralDir zCentral{};	×
412	zCentral.id = ZIPCentralDir::ID;	×
413	SafeCast(zCentral.numDirEntries, numEntries);
414	SafeCast(zCentral.numDiskEntries, numEntries);
415	SafeCast(zCentral.dirOffset, dirOffset);
416
417	for (size_t b = 0; b < numBlocks; b++) {	×
418	rd.ReadBuffer(buffer, sizeof(buffer));
419	records.WriteBuffer(buffer, sizeof(buffer));
420	}
421
422	if (restBytes) {	×
423	rd.ReadBuffer(buffer, restBytes);
424	records.WriteBuffer(buffer, restBytes);
425	}
426
427	bool validCacheEntry = false;
428
429	{
430	// Find last zipfile entry and modify extraFieldSize
431	const size_t skipValue = std::min(entriesSize, size_t(0x11000));	×
432	rd.Skip(-skipValue);	×
433	rd.ReadBuffer(buffer, skipValue);
434	std::string_view sv(buffer, skipValue);
435	size_t foundLastEntry = sv.rfind("PK\x01\x02");
436	validCacheEntry = foundLastEntry != sv.npos;
437
438	if (validCacheEntry) {	×
439	foundLastEntry += offsetof(ZIPFile, extraFieldSize);	×
440	char *data = buffer + foundLastEntry;	×
441	uint16 extraFieldSize = reinterpret_cast<uint16 >(data);	×
442	records.Push();
443	records.Skip(-(skipValue - foundLastEntry));	×
444	records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));	×
445	records.Pop();
446
447	records.Write<uint16>(0x4353);	×
448	records.Write<uint16>(sizeof(CacheBaseHeader));	×
449	cache.meta.zipCheckupOffset = records.Tell();	×
450	records.Write(cache.meta);	×
451	entriesSize += sizeof(CacheBaseHeader) + 4;	×
452	}
453	}
454
455	SafeCast(zCentral.dirSize, entriesSize);
456
457	if (forcex64) {	×
458	ZIP64CentralDir zCentral64{};	×
459	zCentral64.id = ZIP64CentralDir::ID;	×
460	zCentral64.madeBy = 10;	×
461	zCentral64.extractVersion = 10;	×
462	zCentral64.numDiskEntries = numEntries;	×
463	zCentral64.numDirEntries = numEntries;	×
464	zCentral64.dirSize = entriesSize;	×
465	zCentral64.dirOffset = dirOffset;	×
466	zCentral64.dirRecord = 0x2C;	×
467
468	const size_t centralOffset = records.Tell();
469	records.Write(zCentral64);	×
470
471	ZIP64CentralDirLocator zLoca{};	×
472	zLoca.id = ZIP64CentralDirLocator::ID;	×
473	zLoca.centralDirOffset = centralOffset;	×
474	records.Write(zLoca);
475	}
476
477	records.Write(zCentral);
478
479	es::Dispose(rd);	×
480	es::RemoveFile(entriesFile);	×
481
482	if (validCacheEntry) {	×
483	cacheBeginCB();	×
484	cache.meta.zipSize = records.Tell();	×
485	BinWritter cacheWr(outFile + ".cache");	×
486	cache.WaitAndWrite(cacheWr);	×
487	records.Seek(cache.meta.zipCheckupOffset);	×
488	records.Write(cache.meta);	×
489	}
490	}

PredatorCZ / PreCore / 461

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous