PredatorCZ / PreCore / 536

Committed 22 Sep 2024 12:23PM UTC coverage: 52.238% (-3.0%) from 55.265%

Build # 536

Build Type

push

github

Committed by

PredatorCZ

Commit Message

remove old xml serializers

Run Details

4084 of 7818 relevant lines covered (52.24%)

10609.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/app/out_context.cpp

/*  Spike is universal dedicated module handler
    This souce contains data output context

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "spike/app/out_context.hpp"
#include "spike/app/console.hpp"
#include "spike/app/texel.hpp"
#include "spike/crypto/crc32.hpp"
#include "spike/format/ZIP_istream.inl"
#include "spike/format/ZIP_ostream.inl"
#include "spike/io/binreader.hpp"
#include "spike/io/fileinfo.hpp"
#include "spike/io/stat.hpp"
#include <chrono>
#include <mutex>

void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {
  FinishFile(true);

  auto entriesStr = std::move(entriesStream).str();
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();
  size_t dirSize = entriesStr.size();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  records.WriteContainer(entriesStr);
  es::Dispose(entriesStr);

  if (cache) {
    records.Write<uint16>(0x4353);
    records.Write<uint16>(sizeof(CacheBaseHeader));
    cache->meta.zipCheckupOffset = records.Tell();
    records.Write(cache->meta);
    dirSize += sizeof(CacheBaseHeader) + 4;
  }

  SafeCast(zCentral.dirSize, dirSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.dirRecord = 0x2C;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = dirSize;
    zCentral64.dirOffset = dirOffset;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);
  records.BaseStream().flush();

  if (cache) {
    cacheBeginCB();
    cache->meta.zipSize = records.Tell();
    BinWritter cacheWr(outputFile + ".cache");
    cache->WaitAndWrite(cacheWr);
    records.Seek(cache->meta.zipCheckupOffset);
    records.Write(cache->meta);
  }
}

void ZIPExtactContext::FinishFile(bool final) {
  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    bool forcex64 = false;

    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }

    return forcex64;
  };

  const bool useLocalExtendedData =
      SafeCast(zLocalFile.uncompressedSize, curFileSize);
  zLocalFile.compressedSize = zLocalFile.uncompressedSize;

  if (useLocalExtendedData) {
    ZIP64Extra extra;
    extra.compressedSize = curFileSize;
    extra.uncompressedSize = curFileSize;
    records.Write(extra);
    zLocalFile.extraFieldSize = 20;
  }

  numEntries++;
  records.Push();
  records.Seek(curLocalFileOffset);
  records.Write(zLocalFile);
  const size_t fileDataBegin =
      records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;

  if (cache) {
    cache->AddFile(curFileName, fileDataBegin, curFileSize);
    cache->meta.zipCRC = crc32b(
        cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);
  } else {
    fileOffsets.push_back(fileDataBegin);
  }

  records.Pop();

  ZIPFile zFile{};
  zFile.id = ZIPFile::ID;
  zFile.madeBy = 10;
  zFile.extractVersion = 10;
  zFile.lastModFileDate = zLocalFile.lastModFileDate;
  zFile.lastModFileTime = zLocalFile.lastModFileTime;
  zFile.compression = ZIPCompressionMethod::Store;
  zFile.compressedSize = zLocalFile.compressedSize;
  zFile.uncompressedSize = zLocalFile.uncompressedSize;
  zFile.fileNameSize = zLocalFile.fileNameSize;
  zFile.crc = zLocalFile.crc;
  const bool useFileExtendedData =
      SafeCast(zFile.localHeaderOffset, curLocalFileOffset);

  const bool useFileExtra = useFileExtendedData || useLocalExtendedData;
  ZIP64Extra extra;

  if (useFileExtra) {
    zFile.extraFieldSize = 4;

    if (useLocalExtendedData) {
      extra.uncompressedSize = curFileSize;
      extra.compressedSize = curFileSize;
      zFile.extraFieldSize += 16;
    }

    if (useFileExtendedData) {
      extra.localHeaderOffset = curLocalFileOffset;
      zFile.extraFieldSize += 8;
    }
  }

  if (final && cache) {
    zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;
  }

  entries.Write(zFile);
  entries.WriteContainer(prefixPath);
  entries.WriteContainer(curFileName);

  if (useFileExtra) {
    entries.Write(extra);
  }
}

inline std::tm localtime(std::time_t t) {
#ifdef _MSC_VER
  return *std::localtime(&t);
#else
  std::tm temp;
  localtime_r(&t, &temp);
  return temp;
#endif
}

void ZIPExtactContext::NewFile(const std::string &path) {
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo pathInfo(path);
  auto pathSv = pathInfo.GetFullPath();
  if (!curFileName.empty()) {
    FinishFile();
  }

  time_t curTime =
      std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
  std::tm ts = localtime(curTime);

  struct {
    uint16 day : 5, month : 4, year : 7;
  } dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};

  struct {
    uint16 second : 5, minute : 6, hour : 5;
  } dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};

  zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);
  zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);
  zLocalFile.compression = ZIPCompressionMethod::Store;
  zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();
  zLocalFile.crc = 0;
  curFileSize = 0;

  curFileName = pathSv;
  curLocalFileOffset = records.Tell();
  records.Write(zLocalFile);
  records.WriteContainer(prefixPath);
  records.WriteContainer(pathSv);

  if (forEachFile) {
    forEachFile();
  }
}

void ZIPExtactContext::SendData(std::string_view data) {
  curFileSize += data.size();
  zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());
  records.WriteContainer(data);
}

bool ZIPExtactContext::RequiresFolders() const { return false; }

void ZIPExtactContext::AddFolderPath(const std::string &) {
  throw std::logic_error(
      "AddFolderPath not supported, use RequiresFolders to check.");
}

void ZIPExtactContext::GenerateFolders() {
  throw std::logic_error(
      "GenerateFolders not supported, use RequiresFolders to check.");
}

NewTexelContext *ZIPExtactContext::NewImage(const std::string &path,
                                            NewTexelContextCreate ctx) {
  auto newCtx = CreateTexelContext(ctx, this, path);

  if (texelContext) {
    static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
  }

  texelContext = std::move(newCtx);

  return texelContext.get();
}

void IOExtractContext::NewFile(const std::string &path) {
  Close_();
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo cfleWrap(path);
  std::string cfle(cfleWrap.GetFullPath());
  try {
    Open(outDir + cfle);
  } catch (const es::FileInvalidAccessError &) {
    auto folders = cfleWrap.Explode();
    folders.pop_back();
    std::string cpath;
    for (auto &f : folders) {
      cpath.append(f);
      cpath.push_back('/');
      es::mkdir(outDir + cpath);
    }

    Open(outDir + cfle);
  }

  if (forEachFile) {
    forEachFile();
  }
}

void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }

bool IOExtractContext::RequiresFolders() const { return true; }

void IOExtractContext::AddFolderPath(const std::string &path) {
  AFileInfo cfleWrap(path);
  auto cfle = cfleWrap.GetFullPath();

  for (auto it = cfle.begin(); it != cfle.end(); it++) {
    if (*it == '/') {
      folderTree.emplace(cfle.begin(), it);
    }
  }

  folderTree.emplace(path);
}

void IOExtractContext::GenerateFolders() {
  for (auto &f : folderTree) {
    auto genFolder = outDir + f;
    es::mkdir(genFolder);
  }

  folderTree.clear();
}

NewTexelContext *IOExtractContext::NewImage(const std::string &path,
                                            NewTexelContextCreate ctx) {
  auto newCtx = CreateTexelContext(ctx, this, path);

  if (texelContext) {
    static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
  }

  texelContext = std::move(newCtx);

  return texelContext.get();
}

static std::mutex ZIPLock;

void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {
  if (!other.curFileName.empty()) {
    other.FinishFile();
  }

  BinReaderRef localEntries(other.entriesStream);
  char buffer[0x80000];
  std::lock_guard<std::mutex> guard(ZIPLock);
  const size_t filesSize = records.Tell();

  numEntries += other.numEntries;

  for (auto o : other.fileOffsets) {
    ZIPFile zFile;
    localEntries.Read(zFile);

    const size_t localHeaderOffset = zFile.localHeaderOffset;
    const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&
                               localHeaderOffset + filesSize >= 0xffffffff;
    bool newExtra = false;

    if (newExtraField) {
      zFile.localHeaderOffset = 0xffffffff;

      if (!zFile.extraFieldSize) {
        zFile.extraFieldSize = 12;
        newExtra = true;
      } else {
        zFile.extraFieldSize += 8;
      }
    }

    if (zFile.localHeaderOffset < 0xffffffff) {
      zFile.localHeaderOffset += filesSize;
    }

    entries.Write(zFile);
    localEntries.ReadBuffer(buffer, zFile.fileNameSize);
    cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,
                  zFile.uncompressedSize);
    cache.meta.zipCRC = crc32b(cache.meta.zipCRC,
                               reinterpret_cast<const char *>(&zFile.crc), 4);
    entries.WriteBuffer(buffer, zFile.fileNameSize);

    if (newExtra) {
      ZIP64Extra extra{};
      extra.localHeaderOffset = localHeaderOffset + filesSize;
      entries.Write(extra);
    } else if (zFile.extraFieldSize) {
      ZIP64Extra extra{};
      localEntries.Read(extra.id);
      localEntries.Read(extra.size);

      if (zFile.compressedSize == 0xffffffff) {
        localEntries.Read(extra.compressedSize);
        localEntries.Read(extra.uncompressedSize);
      }

      if (zFile.localHeaderOffset == 0xffffffff) {
        if (newExtraField) {
          extra.localHeaderOffset = localHeaderOffset + filesSize;
        } else {
          localEntries.Read(extra.localHeaderOffset);
          extra.localHeaderOffset += filesSize;
        }
      }

      entries.Write(extra);
    }
  }

  es::Dispose(other.entriesStream);

  const size_t recordsSize = other.records.Tell();
  es::Dispose(other.records);
  BinReader rd(recordsFile);
  const size_t numBlocks = recordsSize / sizeof(buffer);
  const size_t restBytes = recordsSize % sizeof(buffer);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }
}

void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {
  size_t entriesSize = entries.Tell();
  es::Dispose(entries);
  char buffer[0x80000];
  BinReader rd(entriesFile);
  const size_t numBlocks = entriesSize / sizeof(buffer);
  const size_t restBytes = entriesSize % sizeof(buffer);
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }

  bool validCacheEntry = false;

  {
    // Find last zipfile entry and modify extraFieldSize
    const size_t skipValue = std::min(entriesSize, size_t(0x11000));
    rd.Skip(-skipValue);
    rd.ReadBuffer(buffer, skipValue);
    std::string_view sv(buffer, skipValue);
    size_t foundLastEntry = sv.rfind("PK\x01\x02");
    validCacheEntry = foundLastEntry != sv.npos;

    if (validCacheEntry) {
      foundLastEntry += offsetof(ZIPFile, extraFieldSize);
      char *data = buffer + foundLastEntry;
      uint16 extraFieldSize = *reinterpret_cast<uint16 *>(data);
      records.Push();
      records.Skip(-(skipValue - foundLastEntry));
      records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));
      records.Pop();

      records.Write<uint16>(0x4353);
      records.Write<uint16>(sizeof(CacheBaseHeader));
      cache.meta.zipCheckupOffset = records.Tell();
      records.Write(cache.meta);
      entriesSize += sizeof(CacheBaseHeader) + 4;
    }
  }

  SafeCast(zCentral.dirSize, entriesSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = entriesSize;
    zCentral64.dirOffset = dirOffset;
    zCentral64.dirRecord = 0x2C;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  es::Dispose(rd);
  es::RemoveFile(entriesFile);

  if (validCacheEntry) {
    cacheBeginCB();
    cache.meta.zipSize = records.Tell();
    BinWritter cacheWr(outFile + ".cache");
    cache.WaitAndWrite(cacheWr);
    records.Seek(cache.meta.zipCheckupOffset);
    records.Write(cache.meta);
  }
}

1	/* Spike is universal dedicated module handler
2	This souce contains data output context
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "spike/app/out_context.hpp"
20	#include "spike/app/console.hpp"
21	#include "spike/app/texel.hpp"
22	#include "spike/crypto/crc32.hpp"
23	#include "spike/format/ZIP_istream.inl"
24	#include "spike/format/ZIP_ostream.inl"
25	#include "spike/io/binreader.hpp"
26	#include "spike/io/fileinfo.hpp"
27	#include "spike/io/stat.hpp"
28	#include <chrono>
29	#include <mutex>
30
31	void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {	×
32	FinishFile(true);	×
33
34	auto entriesStr = std::move(entriesStream).str();
35	bool forcex64 = false;	×
36	const size_t dirOffset = records.Tell();
37	size_t dirSize = entriesStr.size();
38
39	auto SafeCast = [&](auto &where, auto &&what) {
40	const uint64 limit =
41	std::numeric_limits<std::decay_t<decltype(where)>>::max();
42	if (what >= limit) {	×
43	forcex64 = true;	×
44	where = limit;	×
45	} else {
46	where = what;	×
47	}
48	};
49
50	ZIPCentralDir zCentral{};	×
51	zCentral.id = ZIPCentralDir::ID;	×
52	SafeCast(zCentral.numDirEntries, numEntries);
53	SafeCast(zCentral.numDiskEntries, numEntries);
54	SafeCast(zCentral.dirOffset, dirOffset);
55
56	records.WriteContainer(entriesStr);
57	es::Dispose(entriesStr);
58
59	if (cache) {	×
60	records.Write<uint16>(0x4353);	×
61	records.Write<uint16>(sizeof(CacheBaseHeader));	×
62	cache->meta.zipCheckupOffset = records.Tell();	×
63	records.Write(cache->meta);	×
64	dirSize += sizeof(CacheBaseHeader) + 4;	×
65	}
66
67	SafeCast(zCentral.dirSize, dirSize);
68
69	if (forcex64) {	×
70	ZIP64CentralDir zCentral64{};	×
71	zCentral64.id = ZIP64CentralDir::ID;	×
72	zCentral64.madeBy = 10;	×
73	zCentral64.extractVersion = 10;	×
74	zCentral64.dirRecord = 0x2C;	×
75	zCentral64.numDiskEntries = numEntries;	×
76	zCentral64.numDirEntries = numEntries;	×
77	zCentral64.dirSize = dirSize;	×
78	zCentral64.dirOffset = dirOffset;	×
79
80	const size_t centralOffset = records.Tell();
81	records.Write(zCentral64);	×
82
83	ZIP64CentralDirLocator zLoca{};	×
84	zLoca.id = ZIP64CentralDirLocator::ID;	×
85	zLoca.centralDirOffset = centralOffset;	×
86	records.Write(zLoca);
87	}
88
89	records.Write(zCentral);
90	records.BaseStream().flush();	×
91
92	if (cache) {	×
93	cacheBeginCB();	×
94	cache->meta.zipSize = records.Tell();	×
95	BinWritter cacheWr(outputFile + ".cache");	×
96	cache->WaitAndWrite(cacheWr);	×
97	records.Seek(cache->meta.zipCheckupOffset);	×
98	records.Write(cache->meta);	×
99	}
100	}
101
102	void ZIPExtactContext::FinishFile(bool final) {	×
103	auto SafeCast = [&](auto &where, auto &&what) {
104	const uint64 limit =
105	std::numeric_limits<std::decay_t<decltype(where)>>::max();
106	bool forcex64 = false;
107
108	if (what >= limit) {	×
109	forcex64 = true;
110	where = limit;	×
111	} else {
112	where = what;	×
113	}
114
115	return forcex64;
116	};
117
118	const bool useLocalExtendedData =
119	SafeCast(zLocalFile.uncompressedSize, curFileSize);
120	zLocalFile.compressedSize = zLocalFile.uncompressedSize;	×
121
122	if (useLocalExtendedData) {	×
123	ZIP64Extra extra;
124	extra.compressedSize = curFileSize;	×
125	extra.uncompressedSize = curFileSize;	×
126	records.Write(extra);	×
127	zLocalFile.extraFieldSize = 20;	×
128	}
129
130	numEntries++;	×
131	records.Push();
132	records.Seek(curLocalFileOffset);	×
133	records.Write(zLocalFile);	×
134	const size_t fileDataBegin =
135	records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;	×
136
137	if (cache) {	×
138	cache->AddFile(curFileName, fileDataBegin, curFileSize);	×
139	cache->meta.zipCRC = crc32b(	×
140	cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);	×
141	} else {
142	fileOffsets.push_back(fileDataBegin);	×
143	}
144
145	records.Pop();
146
147	ZIPFile zFile{};	×
148	zFile.id = ZIPFile::ID;	×
149	zFile.madeBy = 10;	×
150	zFile.extractVersion = 10;	×
151	zFile.lastModFileDate = zLocalFile.lastModFileDate;	×
152	zFile.lastModFileTime = zLocalFile.lastModFileTime;	×
153	zFile.compression = ZIPCompressionMethod::Store;
154	zFile.compressedSize = zLocalFile.compressedSize;	×
155	zFile.uncompressedSize = zLocalFile.uncompressedSize;	×
156	zFile.fileNameSize = zLocalFile.fileNameSize;	×
157	zFile.crc = zLocalFile.crc;	×
158	const bool useFileExtendedData =
159	SafeCast(zFile.localHeaderOffset, curLocalFileOffset);
160
161	const bool useFileExtra = useFileExtendedData \|\| useLocalExtendedData;	×
162	ZIP64Extra extra;
163
164	if (useFileExtra) {	×
165	zFile.extraFieldSize = 4;	×
166
167	if (useLocalExtendedData) {	×
168	extra.uncompressedSize = curFileSize;	×
169	extra.compressedSize = curFileSize;	×
170	zFile.extraFieldSize += 16;	×
171	}
172
173	if (useFileExtendedData) {	×
174	extra.localHeaderOffset = curLocalFileOffset;	×
175	zFile.extraFieldSize += 8;	×
176	}
177	}
178
179	if (final && cache) {	×
180	zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;	×
181	}
182
183	entries.Write(zFile);	×
184	entries.WriteContainer(prefixPath);
185	entries.WriteContainer(curFileName);
186
187	if (useFileExtra) {	×
188	entries.Write(extra);
189	}
190	}
191
192	inline std::tm localtime(std::time_t t) {
193	#ifdef _MSC_VER
194	return *std::localtime(&t);
195	#else
196	std::tm temp;
197	localtime_r(&t, &temp);	×
198	return temp;
199	#endif
200	}
201
202	void ZIPExtactContext::NewFile(const std::string &path) {	×
203	if (path.empty()) [[unlikely]] {	×
204	throw std::runtime_error("NewFile path is empty");	×
205	}
206	AFileInfo pathInfo(path);	×
207	auto pathSv = pathInfo.GetFullPath();
208	if (!curFileName.empty()) {	×
209	FinishFile();	×
210	}
211
212	time_t curTime =
213	std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());	×
214	std::tm ts = localtime(curTime);	×
215
216	struct {
217	uint16 day : 5, month : 4, year : 7;
218	} dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};	×
219
220	struct {
221	uint16 second : 5, minute : 6, hour : 5;
222	} dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};	×
223
224	zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);	×
225	zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);	×
226	zLocalFile.compression = ZIPCompressionMethod::Store;	×
227	zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();	×
228	zLocalFile.crc = 0;	×
229	curFileSize = 0;	×
230
231	curFileName = pathSv;	×
232	curLocalFileOffset = records.Tell();	×
233	records.Write(zLocalFile);	×
234	records.WriteContainer(prefixPath);
235	records.WriteContainer(pathSv);
236
237	if (forEachFile) {	×
238	forEachFile();
239	}
240	}
241
242	void ZIPExtactContext::SendData(std::string_view data) {	×
243	curFileSize += data.size();	×
244	zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());	×
245	records.WriteContainer(data);
246	}
247
248	bool ZIPExtactContext::RequiresFolders() const { return false; }	×
249
250	void ZIPExtactContext::AddFolderPath(const std::string &) {	×
251	throw std::logic_error(	×
252	"AddFolderPath not supported, use RequiresFolders to check.");	×
253	}
254
255	void ZIPExtactContext::GenerateFolders() {	×
256	throw std::logic_error(	×
257	"GenerateFolders not supported, use RequiresFolders to check.");	×
258	}
259
260	NewTexelContext *ZIPExtactContext::NewImage(const std::string &path,	×
261	NewTexelContextCreate ctx) {
262	auto newCtx = CreateTexelContext(ctx, this, path);	×
263
264	if (texelContext) {	×
265	static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
266	}
267
268	texelContext = std::move(newCtx);
269
270	return texelContext.get();	×
271	}
272
273	void IOExtractContext::NewFile(const std::string &path) {	×
274	Close_();	×
275	if (path.empty()) [[unlikely]] {	×
276	throw std::runtime_error("NewFile path is empty");	×
277	}
278	AFileInfo cfleWrap(path);	×
279	std::string cfle(cfleWrap.GetFullPath());	×
280	try {
281	Open(outDir + cfle);	×
282	} catch (const es::FileInvalidAccessError &) {	×
283	auto folders = cfleWrap.Explode();	×
284	folders.pop_back();
285	std::string cpath;
286	for (auto &f : folders) {	×
287	cpath.append(f);
288	cpath.push_back('/');	×
289	es::mkdir(outDir + cpath);	×
290	}
291
292	Open(outDir + cfle);	×
293	}	×
294
295	if (forEachFile) {	×
296	forEachFile();
297	}
298	}
299
300	void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }	×
301
302	bool IOExtractContext::RequiresFolders() const { return true; }	×
303
304	void IOExtractContext::AddFolderPath(const std::string &path) {	×
305	AFileInfo cfleWrap(path);	×
306	auto cfle = cfleWrap.GetFullPath();
307
308	for (auto it = cfle.begin(); it != cfle.end(); it++) {	×
309	if (*it == '/') {	×
310	folderTree.emplace(cfle.begin(), it);	×
311	}
312	}
313
314	folderTree.emplace(path);
315	}
316
317	void IOExtractContext::GenerateFolders() {	×
318	for (auto &f : folderTree) {	×
319	auto genFolder = outDir + f;	×
320	es::mkdir(genFolder);
321	}
322
323	folderTree.clear();
324	}
325
326	NewTexelContext *IOExtractContext::NewImage(const std::string &path,	×
327	NewTexelContextCreate ctx) {
328	auto newCtx = CreateTexelContext(ctx, this, path);	×
329
330	if (texelContext) {	×
331	static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
332	}
333
334	texelContext = std::move(newCtx);
335
336	return texelContext.get();	×
337	}
338
339	static std::mutex ZIPLock;
340
341	void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {	×
342	if (!other.curFileName.empty()) {	×
343	other.FinishFile();	×
344	}
345
346	BinReaderRef localEntries(other.entriesStream);	×
347	char buffer[0x80000];
348	std::lock_guard<std::mutex> guard(ZIPLock);
349	const size_t filesSize = records.Tell();
350
351	numEntries += other.numEntries;	×
352
353	for (auto o : other.fileOffsets) {	×
354	ZIPFile zFile;
355	localEntries.Read(zFile);	×
356
357	const size_t localHeaderOffset = zFile.localHeaderOffset;	×
358	const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&	×
359	localHeaderOffset + filesSize >= 0xffffffff;	×
360	bool newExtra = false;
361
362	if (newExtraField) {
363	zFile.localHeaderOffset = 0xffffffff;	×
364
365	if (!zFile.extraFieldSize) {	×
366	zFile.extraFieldSize = 12;	×
367	newExtra = true;
368	} else {
369	zFile.extraFieldSize += 8;	×
370	}
371	}
372
373	if (zFile.localHeaderOffset < 0xffffffff) {	×
374	zFile.localHeaderOffset += filesSize;	×
375	}
376
377	entries.Write(zFile);	×
378	localEntries.ReadBuffer(buffer, zFile.fileNameSize);	×
379	cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,	×
380	zFile.uncompressedSize);	×
381	cache.meta.zipCRC = crc32b(cache.meta.zipCRC,	×
382	reinterpret_cast<const char *>(&zFile.crc), 4);
383	entries.WriteBuffer(buffer, zFile.fileNameSize);	×
384
385	if (newExtra) {	×
386	ZIP64Extra extra{};	×
387	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
388	entries.Write(extra);
389	} else if (zFile.extraFieldSize) {	×
390	ZIP64Extra extra{};	×
391	localEntries.Read(extra.id);
392	localEntries.Read(extra.size);
393
394	if (zFile.compressedSize == 0xffffffff) {	×
395	localEntries.Read(extra.compressedSize);
396	localEntries.Read(extra.uncompressedSize);
397	}
398
399	if (zFile.localHeaderOffset == 0xffffffff) {	×
400	if (newExtraField) {	×
401	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
402	} else {
403	localEntries.Read(extra.localHeaderOffset);
404	extra.localHeaderOffset += filesSize;	×
405	}
406	}
407
408	entries.Write(extra);
409	}
410	}
411
412	es::Dispose(other.entriesStream);	×
413
414	const size_t recordsSize = other.records.Tell();
415	es::Dispose(other.records);	×
416	BinReader rd(recordsFile);	×
417	const size_t numBlocks = recordsSize / sizeof(buffer);	×
418	const size_t restBytes = recordsSize % sizeof(buffer);	×
419
420	for (size_t b = 0; b < numBlocks; b++) {	×
421	rd.ReadBuffer(buffer, sizeof(buffer));
422	records.WriteBuffer(buffer, sizeof(buffer));
423	}
424
425	if (restBytes) {	×
426	rd.ReadBuffer(buffer, restBytes);
427	records.WriteBuffer(buffer, restBytes);
428	}
429	}
430
431	void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {	×
432	size_t entriesSize = entries.Tell();	×
433	es::Dispose(entries);	×
434	char buffer[0x80000];
435	BinReader rd(entriesFile);	×
436	const size_t numBlocks = entriesSize / sizeof(buffer);	×
437	const size_t restBytes = entriesSize % sizeof(buffer);	×
438	bool forcex64 = false;	×
439	const size_t dirOffset = records.Tell();
440
441	auto SafeCast = [&](auto &where, auto &&what) {
442	const uint64 limit =
443	std::numeric_limits<std::decay_t<decltype(where)>>::max();
444	if (what >= limit) {	×
445	forcex64 = true;	×
446	where = limit;	×
447	} else {
448	where = what;	×
449	}
450	};
451
452	ZIPCentralDir zCentral{};	×
453	zCentral.id = ZIPCentralDir::ID;	×
454	SafeCast(zCentral.numDirEntries, numEntries);
455	SafeCast(zCentral.numDiskEntries, numEntries);
456	SafeCast(zCentral.dirOffset, dirOffset);
457
458	for (size_t b = 0; b < numBlocks; b++) {	×
459	rd.ReadBuffer(buffer, sizeof(buffer));
460	records.WriteBuffer(buffer, sizeof(buffer));
461	}
462
463	if (restBytes) {	×
464	rd.ReadBuffer(buffer, restBytes);
465	records.WriteBuffer(buffer, restBytes);
466	}
467
468	bool validCacheEntry = false;
469
470	{
471	// Find last zipfile entry and modify extraFieldSize
472	const size_t skipValue = std::min(entriesSize, size_t(0x11000));	×
473	rd.Skip(-skipValue);	×
474	rd.ReadBuffer(buffer, skipValue);
475	std::string_view sv(buffer, skipValue);
476	size_t foundLastEntry = sv.rfind("PK\x01\x02");
477	validCacheEntry = foundLastEntry != sv.npos;
478
479	if (validCacheEntry) {	×
480	foundLastEntry += offsetof(ZIPFile, extraFieldSize);	×
481	char *data = buffer + foundLastEntry;	×
482	uint16 extraFieldSize = reinterpret_cast<uint16 >(data);	×
483	records.Push();
484	records.Skip(-(skipValue - foundLastEntry));	×
485	records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));	×
486	records.Pop();
487
488	records.Write<uint16>(0x4353);	×
489	records.Write<uint16>(sizeof(CacheBaseHeader));	×
490	cache.meta.zipCheckupOffset = records.Tell();	×
491	records.Write(cache.meta);	×
492	entriesSize += sizeof(CacheBaseHeader) + 4;	×
493	}
494	}
495
496	SafeCast(zCentral.dirSize, entriesSize);
497
498	if (forcex64) {	×
499	ZIP64CentralDir zCentral64{};	×
500	zCentral64.id = ZIP64CentralDir::ID;	×
501	zCentral64.madeBy = 10;	×
502	zCentral64.extractVersion = 10;	×
503	zCentral64.numDiskEntries = numEntries;	×
504	zCentral64.numDirEntries = numEntries;	×
505	zCentral64.dirSize = entriesSize;	×
506	zCentral64.dirOffset = dirOffset;	×
507	zCentral64.dirRecord = 0x2C;	×
508
509	const size_t centralOffset = records.Tell();
510	records.Write(zCentral64);	×
511
512	ZIP64CentralDirLocator zLoca{};	×
513	zLoca.id = ZIP64CentralDirLocator::ID;	×
514	zLoca.centralDirOffset = centralOffset;	×
515	records.Write(zLoca);
516	}
517
518	records.Write(zCentral);
519
520	es::Dispose(rd);	×
521	es::RemoveFile(entriesFile);	×
522
523	if (validCacheEntry) {	×
524	cacheBeginCB();	×
525	cache.meta.zipSize = records.Tell();	×
526	BinWritter cacheWr(outFile + ".cache");	×
527	cache.WaitAndWrite(cacheWr);	×
528	records.Seek(cache.meta.zipCheckupOffset);	×
529	records.Write(cache.meta);	×
530	}
531	}

PredatorCZ / PreCore / 536

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous