PredatorCZ / PreCore / 476

Committed 25 Aug 2023 03:59PM UTC coverage: 55.317% (+2.7%) from 52.584%

Build # 476

Build Type

push

github-actions-ci

Committed by

PredatorCZ

Commit Message

add texel context

Run Details

1382 of 1382 new or added lines in 8 files covered. (100.0%)

4146 of 7495 relevant lines covered (55.32%)

8934.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/app/out_context.cpp

/*  Spike is universal dedicated module handler
    This souce contains data output context

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "spike/app/out_context.hpp"
#include "spike/app/console.hpp"
#include "spike/app/texel.hpp"
#include "spike/crypto/crc32.hpp"
#include "spike/format/ZIP_istream.inl"
#include "spike/format/ZIP_ostream.inl"
#include "spike/io/binreader.hpp"
#include "spike/io/fileinfo.hpp"
#include "spike/io/stat.hpp"
#include <chrono>
#include <mutex>

void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {
  FinishFile(true);

  auto entriesStr = std::move(entriesStream).str();
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();
  size_t dirSize = entriesStr.size();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  records.WriteContainer(entriesStr);
  es::Dispose(entriesStr);

  if (cache) {
    records.Write<uint16>(0x4353);
    records.Write<uint16>(sizeof(CacheBaseHeader));
    cache->meta.zipCheckupOffset = records.Tell();
    records.Write(cache->meta);
    dirSize += sizeof(CacheBaseHeader) + 4;
  }

  SafeCast(zCentral.dirSize, dirSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.dirRecord = 0x2C;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = dirSize;
    zCentral64.dirOffset = dirOffset;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  if (cache) {
    cacheBeginCB();
    cache->meta.zipSize = records.Tell();
    BinWritter cacheWr(outputFile + ".cache");
    cache->WaitAndWrite(cacheWr);
    records.Seek(cache->meta.zipCheckupOffset);
    records.Write(cache->meta);
  }
}

void ZIPExtactContext::FinishFile(bool final) {
  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    bool forcex64 = false;

    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }

    return forcex64;
  };

  const bool useLocalExtendedData =
      SafeCast(zLocalFile.uncompressedSize, curFileSize);
  zLocalFile.compressedSize = zLocalFile.uncompressedSize;

  if (useLocalExtendedData) {
    ZIP64Extra extra;
    extra.compressedSize = curFileSize;
    extra.uncompressedSize = curFileSize;
    records.Write(extra);
    zLocalFile.extraFieldSize = 20;
  }

  numEntries++;
  records.Push();
  records.Seek(curLocalFileOffset);
  records.Write(zLocalFile);
  const size_t fileDataBegin =
      records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;

  if (cache) {
    cache->AddFile(curFileName, fileDataBegin, curFileSize);
    cache->meta.zipCRC = crc32b(
        cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);
  } else {
    fileOffsets.push_back(fileDataBegin);
  }

  records.Pop();

  ZIPFile zFile{};
  zFile.id = ZIPFile::ID;
  zFile.madeBy = 10;
  zFile.extractVersion = 10;
  zFile.lastModFileDate = zLocalFile.lastModFileDate;
  zFile.lastModFileTime = zLocalFile.lastModFileTime;
  zFile.compression = ZIPCompressionMethod::Store;
  zFile.compressedSize = zLocalFile.compressedSize;
  zFile.uncompressedSize = zLocalFile.uncompressedSize;
  zFile.fileNameSize = zLocalFile.fileNameSize;
  zFile.crc = zLocalFile.crc;
  const bool useFileExtendedData =
      SafeCast(zFile.localHeaderOffset, curLocalFileOffset);

  const bool useFileExtra = useFileExtendedData || useLocalExtendedData;
  ZIP64Extra extra;

  if (useFileExtra) {
    zFile.extraFieldSize = 4;

    if (useLocalExtendedData) {
      extra.uncompressedSize = curFileSize;
      extra.compressedSize = curFileSize;
      zFile.extraFieldSize += 16;
    }

    if (useFileExtendedData) {
      extra.localHeaderOffset = curLocalFileOffset;
      zFile.extraFieldSize += 8;
    }
  }

  if (final && cache) {
    zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;
  }

  entries.Write(zFile);
  entries.WriteContainer(prefixPath);
  entries.WriteContainer(curFileName);

  if (useFileExtra) {
    entries.Write(extra);
  }
}

inline std::tm localtime(std::time_t t) {
#ifdef _MSC_VER
  return *std::localtime(&t);
#else
  std::tm temp;
  localtime_r(&t, &temp);
  return temp;
#endif
}

void ZIPExtactContext::NewFile(const std::string &path) {
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo pathInfo(path);
  auto pathSv = pathInfo.GetFullPath();
  if (!curFileName.empty()) {
    FinishFile();
  }

  time_t curTime =
      std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
  std::tm ts = localtime(curTime);

  struct {
    uint16 day : 5, month : 4, year : 7;
  } dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};

  struct {
    uint16 second : 5, minute : 6, hour : 5;
  } dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};

  zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);
  zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);
  zLocalFile.compression = ZIPCompressionMethod::Store;
  zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();
  zLocalFile.crc = 0;
  curFileSize = 0;

  curFileName = pathSv;
  curLocalFileOffset = records.Tell();
  records.Write(zLocalFile);
  records.WriteContainer(prefixPath);
  records.WriteContainer(pathSv);

  if (forEachFile) {
    forEachFile();
  }
}

void ZIPExtactContext::SendData(std::string_view data) {
  curFileSize += data.size();
  zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());
  records.WriteContainer(data);
}

bool ZIPExtactContext::RequiresFolders() const { return false; }

void ZIPExtactContext::AddFolderPath(const std::string &) {
  throw std::logic_error(
      "AddFolderPath not supported, use RequiresFolders to check.");
}

void ZIPExtactContext::GenerateFolders() {
  throw std::logic_error(
      "GenerateFolders not supported, use RequiresFolders to check.");
}

NewTexelContext *ZIPExtactContext::NewImage(const std::string &path,
                                            NewTexelContextCreate ctx) {
  auto newCtx = CreateTexelContext(ctx, this, path);

  if (texelContext) {
    static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
  }

  texelContext = std::move(newCtx);

  return texelContext.get();
}

void IOExtractContext::NewFile(const std::string &path) {
  Close_();
  if (path.empty()) [[unlikely]] {
    throw std::runtime_error("NewFile path is empty");
  }
  AFileInfo cfleWrap(path);
  std::string cfle(cfleWrap.GetFullPath());
  Open(outDir + cfle);

  if (forEachFile) {
    forEachFile();
  }
}

void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }

bool IOExtractContext::RequiresFolders() const { return true; }

void IOExtractContext::AddFolderPath(const std::string &path) {
  AFileInfo cfleWrap(path);
  auto cfle = cfleWrap.GetFullPath();

  for (auto it = cfle.begin(); it != cfle.end(); it++) {
    if (*it == '/') {
      folderTree.emplace(cfle.begin(), it);
    }
  }

  folderTree.emplace(path);
}

void IOExtractContext::GenerateFolders() {
  for (auto &f : folderTree) {
    auto genFolder = outDir + f;
    es::mkdir(genFolder);
  }

  folderTree.clear();
}

NewTexelContext *IOExtractContext::NewImage(const std::string &path,
                                            NewTexelContextCreate ctx) {
  auto newCtx = CreateTexelContext(ctx, this, path);

  if (texelContext) {
    static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
  }

  texelContext = std::move(newCtx);

  return texelContext.get();
}

static std::mutex ZIPLock;

void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {
  if (!other.curFileName.empty()) {
    other.FinishFile();
  }

  BinReaderRef localEntries(other.entriesStream);
  char buffer[0x80000];
  std::lock_guard<std::mutex> guard(ZIPLock);
  const size_t filesSize = records.Tell();

  numEntries += other.numEntries;

  for (auto o : other.fileOffsets) {
    ZIPFile zFile;
    localEntries.Read(zFile);

    const size_t localHeaderOffset = zFile.localHeaderOffset;
    const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&
                               localHeaderOffset + filesSize >= 0xffffffff;
    bool newExtra = false;

    if (newExtraField) {
      zFile.localHeaderOffset = 0xffffffff;

      if (!zFile.extraFieldSize) {
        zFile.extraFieldSize = 12;
        newExtra = true;
      } else {
        zFile.extraFieldSize += 8;
      }
    }

    if (zFile.localHeaderOffset < 0xffffffff) {
      zFile.localHeaderOffset += filesSize;
    }

    entries.Write(zFile);
    localEntries.ReadBuffer(buffer, zFile.fileNameSize);
    cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,
                  zFile.uncompressedSize);
    cache.meta.zipCRC = crc32b(cache.meta.zipCRC,
                               reinterpret_cast<const char *>(&zFile.crc), 4);
    entries.WriteBuffer(buffer, zFile.fileNameSize);

    if (newExtra) {
      ZIP64Extra extra{};
      extra.localHeaderOffset = localHeaderOffset + filesSize;
      entries.Write(extra);
    } else if (zFile.extraFieldSize) {
      ZIP64Extra extra{};
      localEntries.Read(extra.id);
      localEntries.Read(extra.size);

      if (zFile.compressedSize == 0xffffffff) {
        localEntries.Read(extra.compressedSize);
        localEntries.Read(extra.uncompressedSize);
      }

      if (zFile.localHeaderOffset == 0xffffffff) {
        if (newExtraField) {
          extra.localHeaderOffset = localHeaderOffset + filesSize;
        } else {
          localEntries.Read(extra.localHeaderOffset);
          extra.localHeaderOffset += filesSize;
        }
      }

      entries.Write(extra);
    }
  }

  es::Dispose(other.entriesStream);

  const size_t recordsSize = other.records.Tell();
  es::Dispose(other.records);
  BinReader rd(recordsFile);
  const size_t numBlocks = recordsSize / sizeof(buffer);
  const size_t restBytes = recordsSize % sizeof(buffer);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }
}

void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {
  size_t entriesSize = entries.Tell();
  es::Dispose(entries);
  char buffer[0x80000];
  BinReader rd(entriesFile);
  const size_t numBlocks = entriesSize / sizeof(buffer);
  const size_t restBytes = entriesSize % sizeof(buffer);
  bool forcex64 = false;
  const size_t dirOffset = records.Tell();

  auto SafeCast = [&](auto &where, auto &&what) {
    const uint64 limit =
        std::numeric_limits<std::decay_t<decltype(where)>>::max();
    if (what >= limit) {
      forcex64 = true;
      where = limit;
    } else {
      where = what;
    }
  };

  ZIPCentralDir zCentral{};
  zCentral.id = ZIPCentralDir::ID;
  SafeCast(zCentral.numDirEntries, numEntries);
  SafeCast(zCentral.numDiskEntries, numEntries);
  SafeCast(zCentral.dirOffset, dirOffset);

  for (size_t b = 0; b < numBlocks; b++) {
    rd.ReadBuffer(buffer, sizeof(buffer));
    records.WriteBuffer(buffer, sizeof(buffer));
  }

  if (restBytes) {
    rd.ReadBuffer(buffer, restBytes);
    records.WriteBuffer(buffer, restBytes);
  }

  bool validCacheEntry = false;

  {
    // Find last zipfile entry and modify extraFieldSize
    const size_t skipValue = std::min(entriesSize, size_t(0x11000));
    rd.Skip(-skipValue);
    rd.ReadBuffer(buffer, skipValue);
    std::string_view sv(buffer, skipValue);
    size_t foundLastEntry = sv.rfind("PK\x01\x02");
    validCacheEntry = foundLastEntry != sv.npos;

    if (validCacheEntry) {
      foundLastEntry += offsetof(ZIPFile, extraFieldSize);
      char *data = buffer + foundLastEntry;
      uint16 extraFieldSize = *reinterpret_cast<uint16 *>(data);
      records.Push();
      records.Skip(-(skipValue - foundLastEntry));
      records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));
      records.Pop();

      records.Write<uint16>(0x4353);
      records.Write<uint16>(sizeof(CacheBaseHeader));
      cache.meta.zipCheckupOffset = records.Tell();
      records.Write(cache.meta);
      entriesSize += sizeof(CacheBaseHeader) + 4;
    }
  }

  SafeCast(zCentral.dirSize, entriesSize);

  if (forcex64) {
    ZIP64CentralDir zCentral64{};
    zCentral64.id = ZIP64CentralDir::ID;
    zCentral64.madeBy = 10;
    zCentral64.extractVersion = 10;
    zCentral64.numDiskEntries = numEntries;
    zCentral64.numDirEntries = numEntries;
    zCentral64.dirSize = entriesSize;
    zCentral64.dirOffset = dirOffset;
    zCentral64.dirRecord = 0x2C;

    const size_t centralOffset = records.Tell();
    records.Write(zCentral64);

    ZIP64CentralDirLocator zLoca{};
    zLoca.id = ZIP64CentralDirLocator::ID;
    zLoca.centralDirOffset = centralOffset;
    records.Write(zLoca);
  }

  records.Write(zCentral);

  es::Dispose(rd);
  es::RemoveFile(entriesFile);

  if (validCacheEntry) {
    cacheBeginCB();
    cache.meta.zipSize = records.Tell();
    BinWritter cacheWr(outFile + ".cache");
    cache.WaitAndWrite(cacheWr);
    records.Seek(cache.meta.zipCheckupOffset);
    records.Write(cache.meta);
  }
}

1	/* Spike is universal dedicated module handler
2	This souce contains data output context
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "spike/app/out_context.hpp"
20	#include "spike/app/console.hpp"
21	#include "spike/app/texel.hpp"
22	#include "spike/crypto/crc32.hpp"
23	#include "spike/format/ZIP_istream.inl"
24	#include "spike/format/ZIP_ostream.inl"
25	#include "spike/io/binreader.hpp"
26	#include "spike/io/fileinfo.hpp"
27	#include "spike/io/stat.hpp"
28	#include <chrono>
29	#include <mutex>
30
31	void ZIPExtactContext::FinishZIP(cache_begin_cb cacheBeginCB) {	×
32	FinishFile(true);	×
33
34	auto entriesStr = std::move(entriesStream).str();
35	bool forcex64 = false;	×
36	const size_t dirOffset = records.Tell();
37	size_t dirSize = entriesStr.size();
38
39	auto SafeCast = [&](auto &where, auto &&what) {
40	const uint64 limit =
41	std::numeric_limits<std::decay_t<decltype(where)>>::max();
42	if (what >= limit) {	×
43	forcex64 = true;	×
44	where = limit;	×
45	} else {
46	where = what;	×
47	}
48	};
49
50	ZIPCentralDir zCentral{};	×
51	zCentral.id = ZIPCentralDir::ID;	×
52	SafeCast(zCentral.numDirEntries, numEntries);
53	SafeCast(zCentral.numDiskEntries, numEntries);
54	SafeCast(zCentral.dirOffset, dirOffset);
55
56	records.WriteContainer(entriesStr);
57	es::Dispose(entriesStr);
58
59	if (cache) {	×
60	records.Write<uint16>(0x4353);	×
61	records.Write<uint16>(sizeof(CacheBaseHeader));	×
62	cache->meta.zipCheckupOffset = records.Tell();	×
63	records.Write(cache->meta);	×
64	dirSize += sizeof(CacheBaseHeader) + 4;	×
65	}
66
67	SafeCast(zCentral.dirSize, dirSize);
68
69	if (forcex64) {	×
70	ZIP64CentralDir zCentral64{};	×
71	zCentral64.id = ZIP64CentralDir::ID;	×
72	zCentral64.madeBy = 10;	×
73	zCentral64.extractVersion = 10;	×
74	zCentral64.dirRecord = 0x2C;	×
75	zCentral64.numDiskEntries = numEntries;	×
76	zCentral64.numDirEntries = numEntries;	×
77	zCentral64.dirSize = dirSize;	×
78	zCentral64.dirOffset = dirOffset;	×
79
80	const size_t centralOffset = records.Tell();
81	records.Write(zCentral64);	×
82
83	ZIP64CentralDirLocator zLoca{};	×
84	zLoca.id = ZIP64CentralDirLocator::ID;	×
85	zLoca.centralDirOffset = centralOffset;	×
86	records.Write(zLoca);
87	}
88
89	records.Write(zCentral);
90
91	if (cache) {	×
92	cacheBeginCB();	×
93	cache->meta.zipSize = records.Tell();	×
94	BinWritter cacheWr(outputFile + ".cache");	×
95	cache->WaitAndWrite(cacheWr);	×
96	records.Seek(cache->meta.zipCheckupOffset);	×
97	records.Write(cache->meta);	×
98	}
99	}
100
101	void ZIPExtactContext::FinishFile(bool final) {	×
102	auto SafeCast = [&](auto &where, auto &&what) {
103	const uint64 limit =
104	std::numeric_limits<std::decay_t<decltype(where)>>::max();
105	bool forcex64 = false;
106
107	if (what >= limit) {	×
108	forcex64 = true;
109	where = limit;	×
110	} else {
111	where = what;	×
112	}
113
114	return forcex64;
115	};
116
117	const bool useLocalExtendedData =
118	SafeCast(zLocalFile.uncompressedSize, curFileSize);
119	zLocalFile.compressedSize = zLocalFile.uncompressedSize;	×
120
121	if (useLocalExtendedData) {	×
122	ZIP64Extra extra;
123	extra.compressedSize = curFileSize;	×
124	extra.uncompressedSize = curFileSize;	×
125	records.Write(extra);	×
126	zLocalFile.extraFieldSize = 20;	×
127	}
128
129	numEntries++;	×
130	records.Push();
131	records.Seek(curLocalFileOffset);	×
132	records.Write(zLocalFile);	×
133	const size_t fileDataBegin =
134	records.Tell() + zLocalFile.extraFieldSize + zLocalFile.fileNameSize;	×
135
136	if (cache) {	×
137	cache->AddFile(curFileName, fileDataBegin, curFileSize);	×
138	cache->meta.zipCRC = crc32b(	×
139	cache->meta.zipCRC, reinterpret_cast<const char *>(&zLocalFile.crc), 4);	×
140	} else {
141	fileOffsets.push_back(fileDataBegin);	×
142	}
143
144	records.Pop();
145
146	ZIPFile zFile{};	×
147	zFile.id = ZIPFile::ID;	×
148	zFile.madeBy = 10;	×
149	zFile.extractVersion = 10;	×
150	zFile.lastModFileDate = zLocalFile.lastModFileDate;	×
151	zFile.lastModFileTime = zLocalFile.lastModFileTime;	×
152	zFile.compression = ZIPCompressionMethod::Store;
153	zFile.compressedSize = zLocalFile.compressedSize;	×
154	zFile.uncompressedSize = zLocalFile.uncompressedSize;	×
155	zFile.fileNameSize = zLocalFile.fileNameSize;	×
156	zFile.crc = zLocalFile.crc;	×
157	const bool useFileExtendedData =
158	SafeCast(zFile.localHeaderOffset, curLocalFileOffset);
159
160	const bool useFileExtra = useFileExtendedData \|\| useLocalExtendedData;	×
161	ZIP64Extra extra;
162
163	if (useFileExtra) {	×
164	zFile.extraFieldSize = 4;	×
165
166	if (useLocalExtendedData) {	×
167	extra.uncompressedSize = curFileSize;	×
168	extra.compressedSize = curFileSize;	×
169	zFile.extraFieldSize += 16;	×
170	}
171
172	if (useFileExtendedData) {	×
173	extra.localHeaderOffset = curLocalFileOffset;	×
174	zFile.extraFieldSize += 8;	×
175	}
176	}
177
178	if (final && cache) {	×
179	zFile.extraFieldSize += sizeof(CacheBaseHeader) + 4;	×
180	}
181
182	entries.Write(zFile);	×
183	entries.WriteContainer(prefixPath);
184	entries.WriteContainer(curFileName);
185
186	if (useFileExtra) {	×
187	entries.Write(extra);
188	}
189	}
190
191	inline std::tm localtime(std::time_t t) {
192	#ifdef _MSC_VER
193	return *std::localtime(&t);
194	#else
195	std::tm temp;
196	localtime_r(&t, &temp);	×
197	return temp;
198	#endif
199	}
200
201	void ZIPExtactContext::NewFile(const std::string &path) {	×
202	if (path.empty()) [[unlikely]] {	×
203	throw std::runtime_error("NewFile path is empty");	×
204	}
205	AFileInfo pathInfo(path);	×
206	auto pathSv = pathInfo.GetFullPath();
207	if (!curFileName.empty()) {	×
208	FinishFile();	×
209	}
210
211	time_t curTime =
212	std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());	×
213	std::tm ts = localtime(curTime);	×
214
215	struct {
216	uint16 day : 5, month : 4, year : 7;
217	} dosDate{uint16(ts.tm_mday), uint16(ts.tm_mon + 1), uint16(ts.tm_year - 80)};	×
218
219	struct {
220	uint16 second : 5, minute : 6, hour : 5;
221	} dosTime{uint16(ts.tm_sec / 2), uint16(ts.tm_min), uint16(ts.tm_hour)};	×
222
223	zLocalFile.lastModFileDate = reinterpret_cast<uint16 &>(dosDate);	×
224	zLocalFile.lastModFileTime = reinterpret_cast<uint16 &>(dosTime);	×
225	zLocalFile.compression = ZIPCompressionMethod::Store;	×
226	zLocalFile.fileNameSize = prefixPath.size() + pathSv.size();	×
227	zLocalFile.crc = 0;	×
228	curFileSize = 0;	×
229
230	curFileName = pathSv;	×
231	curLocalFileOffset = records.Tell();	×
232	records.Write(zLocalFile);	×
233	records.WriteContainer(prefixPath);
234	records.WriteContainer(pathSv);
235
236	if (forEachFile) {	×
237	forEachFile();
238	}
239	}
240
241	void ZIPExtactContext::SendData(std::string_view data) {	×
242	curFileSize += data.size();	×
243	zLocalFile.crc = crc32b(zLocalFile.crc, data.data(), data.size());	×
244	records.WriteContainer(data);
245	}
246
247	bool ZIPExtactContext::RequiresFolders() const { return false; }	×
248
249	void ZIPExtactContext::AddFolderPath(const std::string &) {	×
250	throw std::logic_error(	×
251	"AddFolderPath not supported, use RequiresFolders to check.");	×
252	}
253
254	void ZIPExtactContext::GenerateFolders() {	×
255	throw std::logic_error(	×
256	"GenerateFolders not supported, use RequiresFolders to check.");	×
257	}
258
259	NewTexelContext *ZIPExtactContext::NewImage(const std::string &path,	×
260	NewTexelContextCreate ctx) {
261	auto newCtx = CreateTexelContext(ctx, this, path);	×
262
263	if (texelContext) {	×
264	static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
265	}
266
267	texelContext = std::move(newCtx);
268
269	return texelContext.get();	×
270	}
271
272	void IOExtractContext::NewFile(const std::string &path) {	×
273	Close_();	×
274	if (path.empty()) [[unlikely]] {	×
275	throw std::runtime_error("NewFile path is empty");	×
276	}
277	AFileInfo cfleWrap(path);	×
278	std::string cfle(cfleWrap.GetFullPath());	×
279	Open(outDir + cfle);	×
280
281	if (forEachFile) {	×
282	forEachFile();
283	}
284	}
285
286	void IOExtractContext::SendData(std::string_view data) { WriteContainer(data); }	×
287
288	bool IOExtractContext::RequiresFolders() const { return true; }	×
289
290	void IOExtractContext::AddFolderPath(const std::string &path) {	×
291	AFileInfo cfleWrap(path);	×
292	auto cfle = cfleWrap.GetFullPath();
293
294	for (auto it = cfle.begin(); it != cfle.end(); it++) {	×
295	if (*it == '/') {	×
296	folderTree.emplace(cfle.begin(), it);	×
297	}
298	}
299
300	folderTree.emplace(path);
301	}
302
303	void IOExtractContext::GenerateFolders() {	×
304	for (auto &f : folderTree) {	×
305	auto genFolder = outDir + f;	×
306	es::mkdir(genFolder);
307	}
308
309	folderTree.clear();
310	}
311
312	NewTexelContext *IOExtractContext::NewImage(const std::string &path,	×
313	NewTexelContextCreate ctx) {
314	auto newCtx = CreateTexelContext(ctx, this, path);	×
315
316	if (texelContext) {	×
317	static_cast<NewTexelContextImpl *>(texelContext.get())->Finish();
318	}
319
320	texelContext = std::move(newCtx);
321
322	return texelContext.get();	×
323	}
324
325	static std::mutex ZIPLock;
326
327	void ZIPMerger::Merge(ZIPExtactContext &other, const std::string &recordsFile) {	×
328	if (!other.curFileName.empty()) {	×
329	other.FinishFile();	×
330	}
331
332	BinReaderRef localEntries(other.entriesStream);	×
333	char buffer[0x80000];
334	std::lock_guard<std::mutex> guard(ZIPLock);
335	const size_t filesSize = records.Tell();
336
337	numEntries += other.numEntries;	×
338
339	for (auto o : other.fileOffsets) {	×
340	ZIPFile zFile;
341	localEntries.Read(zFile);	×
342
343	const size_t localHeaderOffset = zFile.localHeaderOffset;	×
344	const bool newExtraField = zFile.localHeaderOffset < 0xffffffff &&	×
345	localHeaderOffset + filesSize >= 0xffffffff;	×
346	bool newExtra = false;
347
348	if (newExtraField) {
349	zFile.localHeaderOffset = 0xffffffff;	×
350
351	if (!zFile.extraFieldSize) {	×
352	zFile.extraFieldSize = 12;	×
353	newExtra = true;
354	} else {
355	zFile.extraFieldSize += 8;	×
356	}
357	}
358
359	if (zFile.localHeaderOffset < 0xffffffff) {	×
360	zFile.localHeaderOffset += filesSize;	×
361	}
362
363	entries.Write(zFile);	×
364	localEntries.ReadBuffer(buffer, zFile.fileNameSize);	×
365	cache.AddFile({buffer, zFile.fileNameSize}, o + filesSize,	×
366	zFile.uncompressedSize);	×
367	cache.meta.zipCRC = crc32b(cache.meta.zipCRC,	×
368	reinterpret_cast<const char *>(&zFile.crc), 4);
369	entries.WriteBuffer(buffer, zFile.fileNameSize);	×
370
371	if (newExtra) {	×
372	ZIP64Extra extra{};	×
373	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
374	entries.Write(extra);
375	} else if (zFile.extraFieldSize) {	×
376	ZIP64Extra extra{};	×
377	localEntries.Read(extra.id);
378	localEntries.Read(extra.size);
379
380	if (zFile.compressedSize == 0xffffffff) {	×
381	localEntries.Read(extra.compressedSize);
382	localEntries.Read(extra.uncompressedSize);
383	}
384
385	if (zFile.localHeaderOffset == 0xffffffff) {	×
386	if (newExtraField) {	×
387	extra.localHeaderOffset = localHeaderOffset + filesSize;	×
388	} else {
389	localEntries.Read(extra.localHeaderOffset);
390	extra.localHeaderOffset += filesSize;	×
391	}
392	}
393
394	entries.Write(extra);
395	}
396	}
397
398	es::Dispose(other.entriesStream);	×
399
400	const size_t recordsSize = other.records.Tell();
401	es::Dispose(other.records);	×
402	BinReader rd(recordsFile);	×
403	const size_t numBlocks = recordsSize / sizeof(buffer);	×
404	const size_t restBytes = recordsSize % sizeof(buffer);	×
405
406	for (size_t b = 0; b < numBlocks; b++) {	×
407	rd.ReadBuffer(buffer, sizeof(buffer));
408	records.WriteBuffer(buffer, sizeof(buffer));
409	}
410
411	if (restBytes) {	×
412	rd.ReadBuffer(buffer, restBytes);
413	records.WriteBuffer(buffer, restBytes);
414	}
415	}
416
417	void ZIPMerger::FinishMerge(cache_begin_cb cacheBeginCB) {	×
418	size_t entriesSize = entries.Tell();	×
419	es::Dispose(entries);	×
420	char buffer[0x80000];
421	BinReader rd(entriesFile);	×
422	const size_t numBlocks = entriesSize / sizeof(buffer);	×
423	const size_t restBytes = entriesSize % sizeof(buffer);	×
424	bool forcex64 = false;	×
425	const size_t dirOffset = records.Tell();
426
427	auto SafeCast = [&](auto &where, auto &&what) {
428	const uint64 limit =
429	std::numeric_limits<std::decay_t<decltype(where)>>::max();
430	if (what >= limit) {	×
431	forcex64 = true;	×
432	where = limit;	×
433	} else {
434	where = what;	×
435	}
436	};
437
438	ZIPCentralDir zCentral{};	×
439	zCentral.id = ZIPCentralDir::ID;	×
440	SafeCast(zCentral.numDirEntries, numEntries);
441	SafeCast(zCentral.numDiskEntries, numEntries);
442	SafeCast(zCentral.dirOffset, dirOffset);
443
444	for (size_t b = 0; b < numBlocks; b++) {	×
445	rd.ReadBuffer(buffer, sizeof(buffer));
446	records.WriteBuffer(buffer, sizeof(buffer));
447	}
448
449	if (restBytes) {	×
450	rd.ReadBuffer(buffer, restBytes);
451	records.WriteBuffer(buffer, restBytes);
452	}
453
454	bool validCacheEntry = false;
455
456	{
457	// Find last zipfile entry and modify extraFieldSize
458	const size_t skipValue = std::min(entriesSize, size_t(0x11000));	×
459	rd.Skip(-skipValue);	×
460	rd.ReadBuffer(buffer, skipValue);
461	std::string_view sv(buffer, skipValue);
462	size_t foundLastEntry = sv.rfind("PK\x01\x02");
463	validCacheEntry = foundLastEntry != sv.npos;
464
465	if (validCacheEntry) {	×
466	foundLastEntry += offsetof(ZIPFile, extraFieldSize);	×
467	char *data = buffer + foundLastEntry;	×
468	uint16 extraFieldSize = reinterpret_cast<uint16 >(data);	×
469	records.Push();
470	records.Skip(-(skipValue - foundLastEntry));	×
471	records.Write<uint16>(extraFieldSize + 4 + sizeof(CacheBaseHeader));	×
472	records.Pop();
473
474	records.Write<uint16>(0x4353);	×
475	records.Write<uint16>(sizeof(CacheBaseHeader));	×
476	cache.meta.zipCheckupOffset = records.Tell();	×
477	records.Write(cache.meta);	×
478	entriesSize += sizeof(CacheBaseHeader) + 4;	×
479	}
480	}
481
482	SafeCast(zCentral.dirSize, entriesSize);
483
484	if (forcex64) {	×
485	ZIP64CentralDir zCentral64{};	×
486	zCentral64.id = ZIP64CentralDir::ID;	×
487	zCentral64.madeBy = 10;	×
488	zCentral64.extractVersion = 10;	×
489	zCentral64.numDiskEntries = numEntries;	×
490	zCentral64.numDirEntries = numEntries;	×
491	zCentral64.dirSize = entriesSize;	×
492	zCentral64.dirOffset = dirOffset;	×
493	zCentral64.dirRecord = 0x2C;	×
494
495	const size_t centralOffset = records.Tell();
496	records.Write(zCentral64);	×
497
498	ZIP64CentralDirLocator zLoca{};	×
499	zLoca.id = ZIP64CentralDirLocator::ID;	×
500	zLoca.centralDirOffset = centralOffset;	×
501	records.Write(zLoca);
502	}
503
504	records.Write(zCentral);
505
506	es::Dispose(rd);	×
507	es::RemoveFile(entriesFile);	×
508
509	if (validCacheEntry) {	×
510	cacheBeginCB();	×
511	cache.meta.zipSize = records.Tell();	×
512	BinWritter cacheWr(outFile + ".cache");	×
513	cache.WaitAndWrite(cacheWr);	×
514	records.Seek(cache.meta.zipCheckupOffset);	×
515	records.Write(cache.meta);	×
516	}
517	}

PredatorCZ / PreCore / 476

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous