PredatorCZ / RevilLib / 170

Committed 19 Nov 2025 09:43PM UTC coverage: 10.614% (-0.02%) from 10.635%

Build # 170

Build Type

push

github

Committed by

PredatorCZ

Commit Message

add more mtf texture support

Run Details

0 of 139 new or added lines in 2 files covered. (0.0%)

156 existing lines in 2 files now uncovered.

757 of 7132 relevant lines covered (10.61%)

6360.16 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/arc.cpp

/*  Revil Format Library
    Copyright(C) 2020-2023 Lukas Cone

    This program is free software : you can redistribute it and / or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.If not, see <https://www.gnu.org/licenses/>.
*/

#include "revil/arc.hpp"
#include "arc.hpp"
#include "hfs.hpp"
#include "revil/hashreg.hpp"
#include "spike/crypto/blowfish.h"
#include "spike/io/fileinfo.hpp"
#include "spike/master_printer.hpp"
#include <set>

#include "lzx.h"
#include "mspack.h"
#include "zlib.h"

#pragma region XMemDecompress

struct mspack_file {
  uint8 *buffer;
  uint32 bufferSize;
  uint32 position;
  uint32 rest;
};

// https://github.com/gildor2/UEViewer/blob/master/Unreal/UnCoreCompression.cpp#L90
static int mspack_read(mspack_file *file, void *buffer, int bytes) {
  if (!file->rest) {
    // read block header
    if (file->buffer[file->position] == 0xFF) {
      // [0]   = FF
      // [1,2] = uncompressed block size
      // [3,4] = compressed block size
      file->rest = (file->buffer[file->position + 3] << 8) |
                   file->buffer[file->position + 4];
      file->position += 5;
    } else {
      // [0,1] = compressed size
      file->rest = (file->buffer[file->position + 0] << 8) |
                   file->buffer[file->position + 1];
      file->position += 2;
    }

    if (file->rest > file->bufferSize - file->position) {
      file->rest = file->bufferSize - file->position;
    }
  }

  if (bytes > file->rest) {
    bytes = file->rest;
  }

  if (bytes <= 0) {
    return 0;
  }

  memcpy(buffer, file->buffer + file->position, bytes);
  file->position += bytes;
  file->rest -= bytes;

  return bytes;
}

static int mspack_write(mspack_file *file, void *buffer, int bytes) {
  if (bytes <= 0) {
    return 0;
  }

  memcpy(file->buffer + file->position, buffer, bytes);
  file->position += bytes;
  return bytes;
}

static mspack_system mspackSystem{
    nullptr,                                                     // open
    nullptr,                                                     // close
    mspack_read,                                                 // read
    mspack_write,                                                // write
    nullptr,                                                     // seek
    nullptr,                                                     // tell
    nullptr,                                                     // message
    [](mspack_system *, size_t bytes) { return malloc(bytes); }, // alloc
    free,                                                        // free
    [](void *src, void *dst, size_t bytes) { memcpy(dst, src, bytes); }, // copy
};

static void DecompressLZX(char *inBuffer, uint32 compressedSize,
                          char *outBuffer, uint32 uncompressedSize,
                          uint32 wBits) {
  mspack_file inStream{};
  mspack_file outStream{};
  inStream.buffer = reinterpret_cast<uint8 *>(inBuffer);
  inStream.bufferSize = compressedSize;
  outStream.buffer = reinterpret_cast<uint8 *>(outBuffer);
  outStream.bufferSize = uncompressedSize;

  lzxd_stream *lzxd = lzxd_init(&mspackSystem, &inStream, &outStream, wBits, 0,
                                1 << wBits, uncompressedSize, false);

  int retVal = lzxd_decompress(lzxd, uncompressedSize);

  if (retVal != MSPACK_ERR_OK) {
    throw std::runtime_error("LZX decompression error " +
                             std::to_string(retVal));
  }

  lzxd_free(lzxd);
}

#pragma endregion

auto ReadARCC(BinReaderRef_e rd, BlowfishEncoder &enc) {
  ARC hdr;
  rd.Read(hdr);
  rd.Skip(-4);

  if (hdr.id != ARCCID) {
    throw es::InvalidHeaderError(hdr.id);
  }

  ARCFiles files;
  rd.ReadContainer(files, hdr.numFiles);

  auto buffer = reinterpret_cast<char *>(files.data());
  size_t bufferSize = sizeof(ARCFile) * hdr.numFiles;

  enc.Decode(buffer, bufferSize);

  return std::make_tuple(hdr, files);
}

void revil::EnumerateArchive(BinReaderRef_e rd, Platform platform,
                             std::string_view title,
                             std::function<AppExtractContext *()> demandContext,
                             const std::set<uint32> &classFilter) {
  uint32 id;
  rd.Push();
  rd.Read(id);
  rd.Pop();
  ARC hdr;

  std::stringstream backup;
  if (id == SFHID) {
    backup = ProcessHFS(rd);
    rd = BinReaderRef_e(backup);
    rd.Push();
    rd.Read(id);
    rd.Pop();
  }

  Platform autoPlatform = id == CRAID ? Platform::PS3 : Platform::Win32;

  if (platform != Platform::Auto) {
    if (revil::IsPlatformBigEndian(autoPlatform) !=
        revil::IsPlatformBigEndian(platform)) {
      printwarning("Platform setting mistmatch, using fallback platform: "
                   << (id == CRAID ? "PS3" : "Win32"));
    }
  } else {
    platform = autoPlatform;
  }

  BlowfishEncoder enc;

  auto WriteFiles = [&](auto &files) {
    auto ectx = demandContext();
    if (ectx->RequiresFolders()) {
      for (auto &f : files) {
        if (classFilter.size() > 0 && !classFilter.contains(f.typeHash)) {
          continue;
        }

        AFileInfo inf(f.fileName);
        const std::string cFolder(inf.GetFolder());
        ectx->AddFolderPath(cFolder);
      }

      ectx->GenerateFolders();
    }

    std::string inBuffer;
    std::string outBuffer;
    [&inBuffer, &outBuffer, &files] {
      size_t maxSize = 0;
      size_t maxSizeUnc = 0;

      for (auto &f : files) {
        if (f.uncompressedSize > maxSizeUnc) {
          maxSizeUnc = f.uncompressedSize;
        }

        if (f.compressedSize > maxSize) {
          maxSize = f.compressedSize;
        }
      }

      if (maxSizeUnc < 0x8000) {
        maxSizeUnc = 0x8000;
      }

      inBuffer.resize(maxSize);
      outBuffer.resize(maxSizeUnc);
    }();

    for (auto &f : files) {
      if (!f.compressedSize) {
        continue;
      }

      if (classFilter.size() > 0 && !classFilter.contains(f.typeHash)) {
        continue;
      }

      rd.Seek(f.offset);

      if (platform == Platform::PS3 && f.compressedSize == f.uncompressedSize) {
        rd.ReadBuffer(&outBuffer[0], f.compressedSize);

        if (id == ARCCID) {
          enc.Decode(&outBuffer[0], f.compressedSize);
        }
      } else {
        rd.ReadBuffer(&inBuffer[0], f.compressedSize);

        if (id == ARCCID) {
          enc.Decode(&inBuffer[0], f.compressedSize);
        }

        if (hdr.IsLZX()) {
          DecompressLZX(&inBuffer[0], f.compressedSize, &outBuffer[0],
                        f.uncompressedSize, id == ARCID ? 17 : 15);
        } else {
          z_stream infstream;
          infstream.zalloc = Z_NULL;
          infstream.zfree = Z_NULL;
          infstream.opaque = Z_NULL;
          infstream.avail_in = f.compressedSize;
          infstream.next_in = reinterpret_cast<Bytef *>(&inBuffer[0]);
          infstream.avail_out = outBuffer.size();
          infstream.next_out = reinterpret_cast<Bytef *>(&outBuffer[0]);
          inflateInit(&infstream);
          int state = inflate(&infstream, Z_FINISH);
          inflateEnd(&infstream);

          if (state < 0) {
            throw std::runtime_error(infstream.msg);
          }
        }
      }

      auto ext = revil::GetExtension(f.typeHash, title, platform);
      std::string filePath = f.fileName;
      filePath.push_back('.');

      if (ext.empty()) {
        char buffer[0x10]{};
        snprintf(buffer, sizeof(buffer), "%.8" PRIX32, f.typeHash);
        filePath += buffer;
      } else {
        filePath.append(ext);
      }

      ectx->NewFile(filePath);
      ectx->SendData({outBuffer.data(), f.uncompressedSize});
    }
  };

  auto ts = revil::GetTitleSupport(title, platform);

  if (ts->arc.flags & revil::DbArc_ExtendedPath) {
    ARCExtendedFiles files;
    std::tie(hdr, files) = ReadExtendedARC(rd);
    WriteFiles(files);
  } else {
    ARCFiles files;
    if (id == ARCCID) {
      std::string_view key(ts->arc.key);

      if (key.empty()) {
        throw es::RuntimeError(
            "Encrypted archives not supported for this title");
      }
      enc.SetKey(key);
      std::tie(hdr, files) = ReadARCC(rd, enc);
    } else {
      std::tie(hdr, files) = ReadARC(rd);
    }
    WriteFiles(files);
  }
}

size_t revil::CompressZlib(std::string_view inBuffer, std::string &outBuffer,
                           int windowSize, int level) {
  z_stream infstream;
  infstream.zalloc = Z_NULL;
  infstream.zfree = Z_NULL;
  infstream.opaque = Z_NULL;
  infstream.avail_in = inBuffer.size();
  infstream.next_in =
      const_cast<Bytef *>(reinterpret_cast<const Bytef *>(inBuffer.data()));
  infstream.avail_out = outBuffer.size();
  infstream.next_out = reinterpret_cast<Bytef *>(outBuffer.data());

  deflateInit2(&infstream, level, Z_DEFLATED, windowSize, 8,
               Z_DEFAULT_STRATEGY);
  int state = deflate(&infstream, Z_FINISH);
  deflateEnd(&infstream);

  if (state != Z_STREAM_END) {
    throw es::RuntimeError("Compression Error!");
  }

  return infstream.total_out;
}

1	/* Revil Format Library
2	Copyright(C) 2020-2023 Lukas Cone
3
4	This program is free software : you can redistribute it and / or modify
5	it under the terms of the GNU General Public License as published by
6	the Free Software Foundation, either version 3 of the License, or
7	(at your option) any later version.
8
9	This program is distributed in the hope that it will be useful,
10	but WITHOUT ANY WARRANTY; without even the implied warranty of
11	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
12	GNU General Public License for more details.
13
14	You should have received a copy of the GNU General Public License
15	along with this program.If not, see <https://www.gnu.org/licenses/>.
16	*/
17
18	#include "revil/arc.hpp"
19	#include "arc.hpp"
20	#include "hfs.hpp"
21	#include "revil/hashreg.hpp"
22	#include "spike/crypto/blowfish.h"
23	#include "spike/io/fileinfo.hpp"
24	#include "spike/master_printer.hpp"
25	#include <set>
26
27	#include "lzx.h"
28	#include "mspack.h"
29	#include "zlib.h"
30
31	#pragma region XMemDecompress
32
33	struct mspack_file {
34	uint8 *buffer;
35	uint32 bufferSize;
36	uint32 position;
37	uint32 rest;
38	};
39
40	// https://github.com/gildor2/UEViewer/blob/master/Unreal/UnCoreCompression.cpp#L90
41	static int mspack_read(mspack_file file, void buffer, int bytes) {
42	if (!file->rest) {	×
43	// read block header
44	if (file->buffer[file->position] == 0xFF) {	×
45	// [0] = FF
46	// [1,2] = uncompressed block size
47	// [3,4] = compressed block size
48	file->rest = (file->buffer[file->position + 3] << 8) \|	×
49	file->buffer[file->position + 4];	×
50	file->position += 5;	×
51	} else {
52	// [0,1] = compressed size
53	file->rest = (file->buffer[file->position + 0] << 8) \|	×
54	file->buffer[file->position + 1];	×
55	file->position += 2;	×
56	}
57
58	if (file->rest > file->bufferSize - file->position) {	×
59	file->rest = file->bufferSize - file->position;	×
60	}
61	}
62
63	if (bytes > file->rest) {	×
64	bytes = file->rest;	×
65	}
66
67	if (bytes <= 0) {	×
68	return 0;
69	}
70
71	memcpy(buffer, file->buffer + file->position, bytes);	×
72	file->position += bytes;	×
73	file->rest -= bytes;	×
74
75	return bytes;	×
76	}
77
78	static int mspack_write(mspack_file file, void buffer, int bytes) {
79	if (bytes <= 0) {	×
80	return 0;
81	}
82
83	memcpy(file->buffer + file->position, buffer, bytes);	×
84	file->position += bytes;	×
85	return bytes;	×
86	}
87
88	static mspack_system mspackSystem{
89	nullptr, // open
90	nullptr, // close
91	mspack_read, // read
92	mspack_write, // write
93	nullptr, // seek
94	nullptr, // tell
95	nullptr, // message
96	[](mspack_system *, size_t bytes) { return malloc(bytes); }, // alloc
97	free, // free
98	[](void src, void dst, size_t bytes) { memcpy(dst, src, bytes); }, // copy
99	};
100
101	static void DecompressLZX(char *inBuffer, uint32 compressedSize,
102	char *outBuffer, uint32 uncompressedSize,
103	uint32 wBits) {
104	mspack_file inStream{};	×
105	mspack_file outStream{};	×
106	inStream.buffer = reinterpret_cast<uint8 *>(inBuffer);	×
107	inStream.bufferSize = compressedSize;	×
108	outStream.buffer = reinterpret_cast<uint8 *>(outBuffer);	×
109	outStream.bufferSize = uncompressedSize;	×
110
111	lzxd_stream *lzxd = lzxd_init(&mspackSystem, &inStream, &outStream, wBits, 0,	×
112	1 << wBits, uncompressedSize, false);
113
114	int retVal = lzxd_decompress(lzxd, uncompressedSize);	×
115
116	if (retVal != MSPACK_ERR_OK) {	×
117	throw std::runtime_error("LZX decompression error " +	×
118	std::to_string(retVal));	×
119	}
120
121	lzxd_free(lzxd);	×
122	}
123
124	#pragma endregion
125
126	auto ReadARCC(BinReaderRef_e rd, BlowfishEncoder &enc) {	×
127	ARC hdr;
128	rd.Read(hdr);	×
129	rd.Skip(-4);	×
130
131	if (hdr.id != ARCCID) {	×
132	throw es::InvalidHeaderError(hdr.id);	×
133	}
134
135	ARCFiles files;	×
136	rd.ReadContainer(files, hdr.numFiles);	×
137
138	auto buffer = reinterpret_cast<char *>(files.data());
139	size_t bufferSize = sizeof(ARCFile) * hdr.numFiles;	×
140
141	enc.Decode(buffer, bufferSize);	×
142
143	return std::make_tuple(hdr, files);	×
144	}
145
146	void revil::EnumerateArchive(BinReaderRef_e rd, Platform platform,	×
147	std::string_view title,
148	std::function<AppExtractContext *()> demandContext,
149	const std::set<uint32> &classFilter) {
150	uint32 id;
151	rd.Push();
152	rd.Read(id);	×
153	rd.Pop();
154	ARC hdr;
155
156	std::stringstream backup;	×
157	if (id == SFHID) {	×
158	backup = ProcessHFS(rd);	×
159	rd = BinReaderRef_e(backup);	×
160	rd.Push();
161	rd.Read(id);	×
162	rd.Pop();
163	}
164
165	Platform autoPlatform = id == CRAID ? Platform::PS3 : Platform::Win32;	×
166
167	if (platform != Platform::Auto) {	×
168	if (revil::IsPlatformBigEndian(autoPlatform) !=	×
169	revil::IsPlatformBigEndian(platform)) {
170	printwarning("Platform setting mistmatch, using fallback platform: "	×
171	<< (id == CRAID ? "PS3" : "Win32"));
172	}
173	} else {
174	platform = autoPlatform;	×
175	}
176
177	BlowfishEncoder enc;	×
178
179	auto WriteFiles = [&](auto &files) {	×
180	auto ectx = demandContext();	×
181	if (ectx->RequiresFolders()) {	×
182	for (auto &f : files) {	×
183	if (classFilter.size() > 0 && !classFilter.contains(f.typeHash)) {	×
184	continue;	×
185	}
186
187	AFileInfo inf(f.fileName);	×
188	const std::string cFolder(inf.GetFolder());	×
189	ectx->AddFolderPath(cFolder);	×
190	}
191
192	ectx->GenerateFolders();	×
193	}
194
195	std::string inBuffer;
196	std::string outBuffer;
197	[&inBuffer, &outBuffer, &files] {	×
198	size_t maxSize = 0;
199	size_t maxSizeUnc = 0;
200
201	for (auto &f : files) {	×
202	if (f.uncompressedSize > maxSizeUnc) {	×
203	maxSizeUnc = f.uncompressedSize;
204	}
205
206	if (f.compressedSize > maxSize) {	×
207	maxSize = f.compressedSize;
208	}
209	}
210
211	if (maxSizeUnc < 0x8000) {	×
212	maxSizeUnc = 0x8000;
213	}
214
215	inBuffer.resize(maxSize);
216	outBuffer.resize(maxSizeUnc);
217	}();	×
218
219	for (auto &f : files) {	×
220	if (!f.compressedSize) {	×
221	continue;	×
222	}
223
224	if (classFilter.size() > 0 && !classFilter.contains(f.typeHash)) {	×
225	continue;	×
226	}
227
228	rd.Seek(f.offset);	×
229
230	if (platform == Platform::PS3 && f.compressedSize == f.uncompressedSize) {	×
231	rd.ReadBuffer(&outBuffer[0], f.compressedSize);
232
233	if (id == ARCCID) {	×
234	enc.Decode(&outBuffer[0], f.compressedSize);	×
235	}
236	} else {
237	rd.ReadBuffer(&inBuffer[0], f.compressedSize);	×
238
239	if (id == ARCCID) {	×
240	enc.Decode(&inBuffer[0], f.compressedSize);	×
241	}
242
243	if (hdr.IsLZX()) {	×
244	DecompressLZX(&inBuffer[0], f.compressedSize, &outBuffer[0],	×
245	f.uncompressedSize, id == ARCID ? 17 : 15);	×
246	} else {
247	z_stream infstream;
248	infstream.zalloc = Z_NULL;	×
249	infstream.zfree = Z_NULL;	×
250	infstream.opaque = Z_NULL;	×
251	infstream.avail_in = f.compressedSize;	×
252	infstream.next_in = reinterpret_cast<Bytef *>(&inBuffer[0]);	×
253	infstream.avail_out = outBuffer.size();	×
254	infstream.next_out = reinterpret_cast<Bytef *>(&outBuffer[0]);	×
255	inflateInit(&infstream);	×
256	int state = inflate(&infstream, Z_FINISH);	×
257	inflateEnd(&infstream);	×
258
259	if (state < 0) {	×
260	throw std::runtime_error(infstream.msg);	×
261	}
262	}
263	}
264
265	auto ext = revil::GetExtension(f.typeHash, title, platform);	×
266	std::string filePath = f.fileName;	×
267	filePath.push_back('.');	×
268
269	if (ext.empty()) {	×
270	char buffer[0x10]{};	×
271	snprintf(buffer, sizeof(buffer), "%.8" PRIX32, f.typeHash);	×
272	filePath += buffer;
273	} else {
274	filePath.append(ext);
275	}
276
277	ectx->NewFile(filePath);	×
278	ectx->SendData({outBuffer.data(), f.uncompressedSize});	×
279	}
280	};
281		×
282	auto ts = revil::GetTitleSupport(title, platform);	×
283		×
284	if (ts->arc.flags & revil::DbArc_ExtendedPath) {	×
285	ARCExtendedFiles files;	×
286	std::tie(hdr, files) = ReadExtendedARC(rd);	×
287	WriteFiles(files);
288	} else {
289	ARCFiles files;	×
290	if (id == ARCCID) {	×
291	std::string_view key(ts->arc.key);	×
292
293	if (key.empty()) {
294	throw es::RuntimeError(	×
295	"Encrypted archives not supported for this title");
296	}
297	enc.SetKey(key);
298	std::tie(hdr, files) = ReadARCC(rd, enc);
299	} else {	×
300	std::tie(hdr, files) = ReadARC(rd);
301	}
302	WriteFiles(files);
303	}
304	}
305
306	size_t revil::CompressZlib(std::string_view inBuffer, std::string &outBuffer,
307	int windowSize, int level) {
308	z_stream infstream;
309	infstream.zalloc = Z_NULL;
310	infstream.zfree = Z_NULL;
311	infstream.opaque = Z_NULL;
312	infstream.avail_in = inBuffer.size();
313	infstream.next_in =
314	const_cast<Bytef >(reinterpret_cast<const Bytef >(inBuffer.data()));
315	infstream.avail_out = outBuffer.size();
316	infstream.next_out = reinterpret_cast<Bytef *>(outBuffer.data());
317
318	deflateInit2(&infstream, level, Z_DEFLATED, windowSize, 8,
319	Z_DEFAULT_STRATEGY);	×
320	int state = deflate(&infstream, Z_FINISH);
321	deflateEnd(&infstream);	×
322		×
323	if (state != Z_STREAM_END) {	×
324	throw es::RuntimeError("Compression Error!");
325	}
326		×
327	return infstream.total_out;	×
328	}

PredatorCZ / RevilLib / 170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous