PredatorCZ / PreCore / 559

Committed 25 Jul 2025 03:19PM UTC coverage: 53.212% (+0.006%) from 53.206%

Build # 559

Build Type

push

github

Committed by

PredatorCZ

Commit Message

fix file creation exception for newfile

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

331 existing lines in 4 files now uncovered.

4241 of 7970 relevant lines covered (53.21%)

10412.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/cli/spike.cpp

/*  Spike is universal dedicated module handler
    This source contains code for CLI master app

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "nlohmann/json.hpp"
#include "project.h"
#include "spike/app/batch.hpp"
#include "spike/app/console.hpp"
#include "spike/app/tmp_storage.hpp"
#include "spike/io/binwritter.hpp"
#include "spike/io/stat.hpp"
#include "spike/master_printer.hpp"
#include "spike/type/tchar.hpp"
#include "spike/util/pugiex.hpp"
#include <thread>

static const char appHeader0[] =
    "Simply drag'n'drop files/folders onto application or "
    "use as ";
static const char appHeader1[] =
    " [options] path1 path2 ...\nTool can detect and scan folders and "
    "uncompressed zip archives.";

struct ProcessedFiles : LoadingBar, CounterLine {
  char buffer[128]{};
  CounterLine producedFiles;

  ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}
  void PrintLine() override {
    snprintf(buffer, sizeof(buffer),
             "Processed %4" PRIuMAX " files. Produced %4" PRIuMAX " files.",
             size_t(curitem), size_t(producedFiles));
    LoadingBar::PrintLine();
  }
};

struct ExtractStats {
  std::map<JenHash, size_t> archiveFiles;
  size_t totalFiles = 0;
};

struct UILines {
  ProgressBar *totalProgress{nullptr};
  CounterLine *totalCount{nullptr};
  CounterLine *totalOutCount{nullptr};
  std::map<uint32, ProgressBar *> bars;
  std::mutex barsMutex;

  auto ChooseBar() {
    if (bars.empty()) {
      return (ProgressBar *)(nullptr);
    }

    auto threadId = std::this_thread::get_id();
    auto id = reinterpret_cast<const uint32 &>(threadId);
    auto found = bars.find(id);

    if (es::IsEnd(bars, found)) {
      std::lock_guard<std::mutex> lg(barsMutex);
      auto retVal = bars.begin()->second;
      bars.emplace(id, retVal);
      bars.erase(bars.begin());

      return retVal;
    }

    return found->second;
  };

  UILines(const ExtractStats &stats) {
    ModifyElements([&](ElementAPI &api) {
      const size_t minThreads =
          std::min(size_t(std::thread::hardware_concurrency()),
                   stats.archiveFiles.size());

      if (minThreads < 2) {
        return;
      }

      for (size_t t = 0; t < minThreads; t++) {
        auto progBar = std::make_unique<ProgressBar>("Thread:");
        auto progBarRaw = progBar.get();
        bars.emplace(t, progBarRaw);
        api.Append(std::move(progBar));
      }
    });

    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(stats.totalFiles);
    totalCount = prog;
  }

  UILines(size_t totalInputFiles) {
    auto procFiles = AppendNewLogLine<ProcessedFiles>();
    totalCount = procFiles;
    totalOutCount = &procFiles->producedFiles;
    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(totalInputFiles);
    totalProgress = prog;
  }

  ~UILines() {
    ModifyElements([&](ElementAPI &api) {
      if (totalCount) {
        // Wait a little bit for internal queues to finish printing
        std::this_thread::sleep_for(std::chrono::milliseconds(100));
        if (totalProgress) {
          auto data = static_cast<ProcessedFiles *>(totalCount);
          data->Finish();
          api.Release(data);
        } else {
          auto data = static_cast<DetailedProgressBar *>(totalCount);
          api.Remove(data);
        }
      }
      api.Clean();
      std::this_thread::sleep_for(std::chrono::milliseconds(100));
    });
  }
};

bool ScanModules(const std::string &appFolder, const std::string &appName) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  bool isOkay = true;

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      APPContext ctx(moduleNameStr.data(), appFolder, appName);
      ctx.FromConfig();
    } catch (const std::runtime_error &e) {
      printerror(e.what());
      isOkay = false;
    }
  }

  return isOkay;
}

void GenerateDocumentation(const std::string &appFolder,
                           const std::string &appName,
                           const std::string &templatePath) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  std::set<std::string> modules;
  pugi::xml_document doc;

  if (!templatePath.empty()) {
    doc = XMLFromFile(templatePath);
  }

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      modules.emplace(moduleNameStr);
    } catch (const std::runtime_error &e) {
      printerror(e.what());
    }
  }

  BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");

  const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";

  if (auto child = doc.child("toolset_description"); child) {
    toolsetDescription = child.text().as_string();
  }

  wr.BaseStream() << toolsetDescription << "<h2>Module list</h2>\n<ul>\n";
  std::stringstream str;

  for (auto &m : modules) {
    pugi::xml_node node = doc.child(m.data());
    APPContext ctx(m.data(), appFolder, appName);
    ctx.GetMarkdownDoc(str, node);
    std::string className = ctx.GetClassName(node);
    std::string classNameLink = className;
    std::replace_if(
        classNameLink.begin(), classNameLink.end(),
        [](char c) { return c == ' '; }, '-');

    wr.BaseStream() << "<li><a href=\"#" << classNameLink << "\">" << className
                    << "</a></li>\n";
  }

  wr.BaseStream() << "</ul>\n\n" << str.str() << "\n\n";

  if (auto child = doc.child("toolset_footer"); child) {
    wr.BaseStream() << child.text().as_string();
  }
}

void PackModeBatch(Batch &batch) {
  struct PackData {
    size_t index = 0;
    std::unique_ptr<AppPackContext> archiveContext;
    std::string pbarLabel;
    DetailedProgressBar *progBar = nullptr;
    std::string folderPath;
  };

  auto payload = std::make_shared<PackData>();

  batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,
                                                   size_t numFiles) {
    payload->folderPath = path;
    payload->archiveContext.reset(ctx->NewArchive(path));
    payload->pbarLabel = "Folder id " + std::to_string(payload->index++);
    payload->progBar =
        AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);
    payload->progBar->ItemCount(numFiles);
    uint8 consoleDetail = 1 | uint8(ctx->info->multithreaded) << 1;
    ConsolePrintDetail(consoleDetail);
    printline("Processing: " << path);
  };

  batch.forEachFile = [payload](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {
      int notSlash = !payload->folderPath.ends_with('/');
      payload->archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +
                                                 notSlash),
          iCtx->GetStream());
    } else {
      payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                                        iCtx->GetStream());
    }
    (*payload->progBar)++;
  };

  batch.forEachFolderFinish = [payload] {
    ConsolePrintDetail(1);
    payload->archiveContext->Finish();
    payload->archiveContext.reset();
    RemoveLogLines(payload->progBar);
  };
}

void MergePackModeBatch(Batch &batch, const std::string &folderPath,
                        AppPackContext *archiveContext) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  printline("Processing: " << folderPath);

  batch.forEachFile = [=](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {
      int notSlash = !folderPath.ends_with('/');
      archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),
          iCtx->GetStream());
    } else {
      archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                               iCtx->GetStream());
    }
  };
}

auto ExtractStatBatch(Batch &batch) {
  struct ExtractStatsMaker : ExtractStats {
    std::mutex mtx;
    LoadingBar *scanBar;

    void Push(AppContextShare *ctx, size_t numFiles) {
      std::unique_lock<std::mutex> lg(mtx);
      archiveFiles.emplace(ctx->Hash(), numFiles);
      totalFiles += numFiles;
    }

    ~ExtractStatsMaker() { RemoveLogLines(scanBar); }
  };

  batch.keepFinishLines = false;
  auto sharedData = std::make_shared<ExtractStatsMaker>();
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  sharedData->scanBar =
      AppendNewLogLine<LoadingBar>("Processing extract stats.");

  batch.forEachFile = [payload = sharedData,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    payload->Push(iCtx, ctx->ExtractStat(std::bind(
                            [&](size_t offset, size_t size) {
                              return iCtx->GetBuffer(size, offset);
                            },
                            std::placeholders::_1, std::placeholders::_2)));
  };

  return sharedData;
}

void ProcessBatch(Batch &batch, ExtractStats *stats) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  batch.forEachFile = [payload = std::make_shared<UILines>(*stats),
                       archiveFiles =
                           std::make_shared<decltype(stats->archiveFiles)>(
                               std::move(stats->archiveFiles)),
                       ctx = batch.ctx](AppContextShare *iCtx) {
    auto currentBar = payload->ChooseBar();
    if (currentBar) {
      currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));
    }

    iCtx->forEachFile = [=] {
      if (currentBar) {
        (*currentBar)++;
      }

      if (payload->totalCount) {
        (*payload->totalCount)++;
      }
    };

    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
  };
}

void ProcessBatch(Batch &batch, size_t numFiles) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  auto payload = std::make_shared<UILines>(numFiles);
  batch.forEachFile = [payload = payload,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    iCtx->forEachFile = [=] {
      if (payload->totalOutCount) {
        (*payload->totalOutCount)++;
      }
    };
    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
    if (payload->totalCount) {
      (*payload->totalCount)++;
    }
  };

  auto totalFiles = std::make_shared<size_t>(numFiles);
  batch.updateFileCount = [payload = payload,
                           totalFiles = totalFiles](size_t addedFiles) {
    *totalFiles.get() += addedFiles;
    payload->totalProgress->ItemCount(*totalFiles);
  };
}

int CreateContent(const std::string &moduleName, const std::string &appFolder,
                  const std::string &appName, APPContext &ctx) {
  try {
    ctx = APPContext(moduleName.c_str(), appFolder, appName);
  } catch (const std::exception &e) {
    printerror(e.what());
    return 2;
  }

  ConsolePrintDetail(0);

  printline(ctx.info->header);
  printline(appHeader0 << appName << ' ' << moduleName << appHeader1);

  return 0;
}

int LoadProject(const std::string &path, const std::string &appFolder,
                const std::string &appName) {
  std::ifstream str(path);
  nlohmann::json project(nlohmann::json::parse(str));

  APPContext ctx;
  std::string moduleName = project["module"];

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  ConsolePrintDetail(1);

  std::string outputDir;

  if (!project["output_dir"].is_null()) {
    std::string outputDir = project["output_dir"];
    ctx.ApplySetting("out", outputDir);
  }

  bool noConfig = !project["no_config"].is_null() && project["no_config"];

  if (!noConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  nlohmann::json settings = project["settings"];

  for (auto &s : settings.items()) {
    std::string dumped = s.value().dump();
    if (dumped.front() == '"') {
      dumped.erase(0, 1);
    }
    if (dumped.back() == '"') {
      dumped.erase(dumped.size() - 1);
    }
    ctx.ApplySetting(s.key(), dumped);
  }

  nlohmann::json inputs = project["inputs"];

  InitTempStorage();
  ctx.SetupModule();
  std::unique_ptr<AppPackContext> archiveContext;

  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);
    AFileInfo batchPath(path);
    std::string batchBase(batchPath.GetFolder());

    if (ctx.NewArchive) {
      std::string folder(batchPath.GetFolder());
      std::string archive(batchPath.GetFullPathNoExt());
      archiveContext.reset(batch.ctx->NewArchive(archive));
      MergePackModeBatch(batch, folder, archiveContext.get());
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (std::string input : inputs) {
          batch.AddFile(batchBase + input);
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += inputs.size();
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, inputs.size());
      }
    }

    if (ctx.info->batchControlFilters.size() > 0) {
      std::string pathDir(AFileInfo(path).GetFolder());
      for (nlohmann::json input : inputs) {
        if (input.is_array()) {
          batch.AddBatch(input, pathDir);
        } else {
          printwarning("Expected group, got " << input.type_name()
                                              << " instead. Skipping input.");
        }
      }
    } else {
      for (nlohmann::json input : inputs) {
        if (input.is_string()) {
          batch.AddFile(batchBase + std::string(input));
        } else {
          printwarning("Expected path string, got "
                       << input.type_name() << " instead. Skipping input.");
        }
      }
    }

    batch.FinishBatch();
  }

  if (archiveContext) {
    ConsolePrintDetail(1);
    archiveContext->Finish();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int Main(int argc, TCHAR *argv[]) {
  ConsolePrintDetail(1);
  AFileInfo appLocation(std::to_string(*argv));
  std::string appFolder(appLocation.GetFolder());
  std::string appName(appLocation.GetFilename());
  es::SetDllRunPath(appFolder + "lib");

  if (argc < 2) {
    printwarning(
        "No parameters provided, entering scan mode and generating config.");
    return !ScanModules(appFolder, appName);
  }

  auto moduleName = std::to_string(argv[1]);

  if (moduleName == "--make-doc") {
    std::string templatePath;

    if (argc < 3) {
      printwarning("Expexted template path!");
    } else {
      templatePath = std::to_string(argv[2]);
    }

    GenerateDocumentation(appFolder, appName, templatePath);
    return 0;
  } else if (moduleName.ends_with(".json")) {
    return LoadProject(moduleName, appFolder, appName);
  }

  if (argc < 3) {
    printerror("Insufficient argument count, expected parameters.");
    return 1;
  }

  APPContext ctx;

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  if (IsHelp(argv[2])) {
    ctx.PrintCLIHelp();
    return 0;
  }

  ConsolePrintDetail(1);
  bool dontLoadConfig = false;
  std::vector<bool> markedFiles(size_t(argc), false);
  size_t totalFiles = 0;

  // Handle cli options and switches
  for (int a = 2; a < argc; a++) {
    auto opt = argv[a];

    if (opt[0] == '-') {
      auto optStr = std::to_string(opt);
      std::string_view optsw(optStr);

      if (optsw != "--out") {
        // We won't use config file, reset all booleans to false,
        // so we can properly use cli switches
        [&] {
          if (dontLoadConfig) {
            return;
          }

          printinfo("CLI option detected, config won't be loaded, all booleans "
                    "set to false!");
          ctx.ResetSwitchSettings();
        }();
        dontLoadConfig = true;
      }
      optsw.remove_prefix(1);

      if (opt[0] == '-') {
        optsw.remove_prefix(1);
      }

      auto valStr = std::to_string(argv[a + 1]);

      if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {
        a++;
      }

    } else {
      markedFiles[a] = true;
      totalFiles++;
    }
  }

  if (!dontLoadConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  InitTempStorage();
  ctx.SetupModule();
  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);

    if (ctx.NewArchive) {
      PackModeBatch(batch);
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (int a = 2; a < argc; a++) {
          if (!markedFiles.at(a)) {
            continue;
          }
          batch.AddFile(std::to_string(argv[a]));
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += totalFiles;
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, totalFiles);
      }
    }

    for (int a = 2; a < argc; a++) {
      if (!markedFiles.at(a)) {
        continue;
      }
      batch.AddFile(std::to_string(argv[a]));
    }

    batch.FinishBatch();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int _tmain(int argc, TCHAR *argv[]) {
  es::SetupWinApiConsole();
  InitConsole();
  CleanTempStorages();

  int retVal = Main(argc, argv);

  CleanCurrentTempStorage();

#ifndef NDEBUG
  auto cacheStats = CacheGenerator::GlobalMetrics();
  PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
            " search misses: ", cacheStats.numSearchMisses);
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
#endif
  TerminateConsole();
  return retVal;
}

1	/* Spike is universal dedicated module handler
2	This source contains code for CLI master app
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "nlohmann/json.hpp"
20	#include "project.h"
21	#include "spike/app/batch.hpp"
22	#include "spike/app/console.hpp"
23	#include "spike/app/tmp_storage.hpp"
24	#include "spike/io/binwritter.hpp"
25	#include "spike/io/stat.hpp"
26	#include "spike/master_printer.hpp"
27	#include "spike/type/tchar.hpp"
28	#include "spike/util/pugiex.hpp"
29	#include <thread>
30
31	static const char appHeader0[] =
32	"Simply drag'n'drop files/folders onto application or "
33	"use as ";
34	static const char appHeader1[] =
35	" [options] path1 path2 ...\nTool can detect and scan folders and "
36	"uncompressed zip archives.";
37
38	struct ProcessedFiles : LoadingBar, CounterLine {
39	char buffer[128]{};
40	CounterLine producedFiles;
41
42	ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}	×
43	void PrintLine() override {	×
UNCOV 44	snprintf(buffer, sizeof(buffer),	×
45	"Processed %4" PRIuMAX " files. Produced %4" PRIuMAX " files.",
46	size_t(curitem), size_t(producedFiles));
UNCOV 47	LoadingBar::PrintLine();	×
48	}
49	};
50
51	struct ExtractStats {
52	std::map<JenHash, size_t> archiveFiles;
53	size_t totalFiles = 0;
54	};
55
56	struct UILines {
57	ProgressBar *totalProgress{nullptr};
58	CounterLine *totalCount{nullptr};
59	CounterLine *totalOutCount{nullptr};
60	std::map<uint32, ProgressBar *> bars;
61	std::mutex barsMutex;
62
UNCOV 63	auto ChooseBar() {	×
UNCOV 64	if (bars.empty()) {	×
65	return (ProgressBar *)(nullptr);
66	}
67
68	auto threadId = std::this_thread::get_id();
69	auto id = reinterpret_cast<const uint32 &>(threadId);	×
70	auto found = bars.find(id);
71
72	if (es::IsEnd(bars, found)) {	×
UNCOV 73	std::lock_guard<std::mutex> lg(barsMutex);	×
UNCOV 74	auto retVal = bars.begin()->second;	×
UNCOV 75	bars.emplace(id, retVal);	×
76	bars.erase(bars.begin());
77
78	return retVal;
79	}
80
81	return found->second;	×
82	};
83
84	UILines(const ExtractStats &stats) {	×
85	ModifyElements([&](ElementAPI &api) {	×
86	const size_t minThreads =
87	std::min(size_t(std::thread::hardware_concurrency()),	×
UNCOV 88	stats.archiveFiles.size());	×
89
UNCOV 90	if (minThreads < 2) {	×
91	return;
92	}
93
94	for (size_t t = 0; t < minThreads; t++) {	×
95	auto progBar = std::make_unique<ProgressBar>("Thread:");	×
UNCOV 96	auto progBarRaw = progBar.get();	×
UNCOV 97	bars.emplace(t, progBarRaw);	×
UNCOV 98	api.Append(std::move(progBar));	×
99	}
100	});
101
UNCOV 102	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
UNCOV 103	prog->ItemCount(stats.totalFiles);	×
104	totalCount = prog;	×
105	}
106
107	UILines(size_t totalInputFiles) {	×
108	auto procFiles = AppendNewLogLine<ProcessedFiles>();	×
UNCOV 109	totalCount = procFiles;	×
UNCOV 110	totalOutCount = &procFiles->producedFiles;	×
111	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
112	prog->ItemCount(totalInputFiles);	×
113	totalProgress = prog;	×
114	}
115
116	~UILines() {	×
117	ModifyElements([&](ElementAPI &api) {	×
UNCOV 118	if (totalCount) {	×
119	// Wait a little bit for internal queues to finish printing
UNCOV 120	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
121	if (totalProgress) {	×
122	auto data = static_cast<ProcessedFiles *>(totalCount);	×
123	data->Finish();
UNCOV 124	api.Release(data);	×
125	} else {
126	auto data = static_cast<DetailedProgressBar *>(totalCount);	×
127	api.Remove(data);	×
128	}
129	}
UNCOV 130	api.Clean();	×
131	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
132	});	×
133	}
134	};
135
UNCOV 136	bool ScanModules(const std::string &appFolder, const std::string &appName) {	×
137	DirectoryScanner sc;	×
UNCOV 138	sc.AddFilter(std::string_view(".spk$"));	×
139	sc.Scan(appFolder);	×
140	bool isOkay = true;
141
142	for (auto &m : sc) {	×
143	try {
144	AFileInfo modulePath(m);	×
145	auto moduleName = modulePath.GetFilename();	×
146	const size_t firstDotPos = moduleName.find_first_of('.');
UNCOV 147	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
148	APPContext ctx(moduleNameStr.data(), appFolder, appName);	×
UNCOV 149	ctx.FromConfig();	×
UNCOV 150	} catch (const std::runtime_error &e) {	×
151	printerror(e.what());	×
152	isOkay = false;
UNCOV 153	}	×
154	}
155
UNCOV 156	return isOkay;	×
157	}
158
159	void GenerateDocumentation(const std::string &appFolder,	×
160	const std::string &appName,
161	const std::string &templatePath) {
UNCOV 162	DirectoryScanner sc;	×
163	sc.AddFilter(std::string_view(".spk$"));	×
164	sc.Scan(appFolder);	×
165	std::set<std::string> modules;
UNCOV 166	pugi::xml_document doc;	×
167
UNCOV 168	if (!templatePath.empty()) {	×
169	doc = XMLFromFile(templatePath);	×
170	}
171
172	for (auto &m : sc) {	×
173	try {
174	AFileInfo modulePath(m);	×
175	auto moduleName = modulePath.GetFilename();	×
176	const size_t firstDotPos = moduleName.find_first_of('.');
UNCOV 177	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
178	modules.emplace(moduleNameStr);
179	} catch (const std::runtime_error &e) {	×
UNCOV 180	printerror(e.what());	×
UNCOV 181	}	×
182	}
183
184	BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");	×
185
186	const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";
187
188	if (auto child = doc.child("toolset_description"); child) {	×
UNCOV 189	toolsetDescription = child.text().as_string();	×
190	}
191
192	wr.BaseStream() << toolsetDescription << "<h2>Module list</h2>\n<ul>\n";	×
193	std::stringstream str;	×
194
UNCOV 195	for (auto &m : modules) {	×
UNCOV 196	pugi::xml_node node = doc.child(m.data());	×
UNCOV 197	APPContext ctx(m.data(), appFolder, appName);	×
UNCOV 198	ctx.GetMarkdownDoc(str, node);	×
UNCOV 199	std::string className = ctx.GetClassName(node);	×
200	std::string classNameLink = className;
201	std::replace_if(
202	classNameLink.begin(), classNameLink.end(),
203	[](char c) { return c == ' '; }, '-');
204
205	wr.BaseStream() << "<li><a href=\"#" << classNameLink << "\">" << className
206	<< "</a></li>\n";	×
207	}
208
UNCOV 209	wr.BaseStream() << "</ul>\n\n" << str.str() << "\n\n";	×
210
211	if (auto child = doc.child("toolset_footer"); child) {	×
UNCOV 212	wr.BaseStream() << child.text().as_string();	×
213	}
214	}
215
UNCOV 216	void PackModeBatch(Batch &batch) {	×
217	struct PackData {
218	size_t index = 0;
219	std::unique_ptr<AppPackContext> archiveContext;
220	std::string pbarLabel;
221	DetailedProgressBar *progBar = nullptr;
222	std::string folderPath;
223	};
224
225	auto payload = std::make_shared<PackData>();
226
227	batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,	×
228	size_t numFiles) {	×
229	payload->folderPath = path;	×
230	payload->archiveContext.reset(ctx->NewArchive(path));
231	payload->pbarLabel = "Folder id " + std::to_string(payload->index++);	×
232	payload->progBar =	×
UNCOV 233	AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);	×
UNCOV 234	payload->progBar->ItemCount(numFiles);	×
235	uint8 consoleDetail = 1 \| uint8(ctx->info->multithreaded) << 1;	×
236	ConsolePrintDetail(consoleDetail);	×
237	printline("Processing: " << path);	×
238	};
239
240	batch.forEachFile = [payload](AppContextShare *iCtx) {	×
241	if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {	×
UNCOV 242	int notSlash = !payload->folderPath.ends_with('/');	×
243	payload->archiveContext->SendFile(	×
244	iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +	×
UNCOV 245	notSlash),	×
246	iCtx->GetStream());	×
247	} else {
UNCOV 248	payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
249	iCtx->GetStream());	×
250	}
251	(*payload->progBar)++;	×
252	};
253
UNCOV 254	batch.forEachFolderFinish = [payload] {	×
UNCOV 255	ConsolePrintDetail(1);	×
UNCOV 256	payload->archiveContext->Finish();	×
257	payload->archiveContext.reset();
UNCOV 258	RemoveLogLines(payload->progBar);	×
259	};
260	}
261
UNCOV 262	void MergePackModeBatch(Batch &batch, const std::string &folderPath,	×
263	AppPackContext *archiveContext) {
264	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
265	ConsolePrintDetail(consoleDetail);	×
266	printline("Processing: " << folderPath);	×
267
268	batch.forEachFile = [=](AppContextShare *iCtx) {	×
UNCOV 269	if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {	×
270	int notSlash = !folderPath.ends_with('/');	×
271	archiveContext->SendFile(	×
UNCOV 272	iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),	×
UNCOV 273	iCtx->GetStream());	×
274	} else {
UNCOV 275	archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
276	iCtx->GetStream());	×
277	}
278	};
279	}
280
281	auto ExtractStatBatch(Batch &batch) {	×
282	struct ExtractStatsMaker : ExtractStats {
283	std::mutex mtx;
284	LoadingBar *scanBar;
285
UNCOV 286	void Push(AppContextShare *ctx, size_t numFiles) {	×
287	std::unique_lock<std::mutex> lg(mtx);	×
UNCOV 288	archiveFiles.emplace(ctx->Hash(), numFiles);	×
UNCOV 289	totalFiles += numFiles;	×
290	}
291
292	~ExtractStatsMaker() { RemoveLogLines(scanBar); }	×
293	};
294
295	batch.keepFinishLines = false;	×
296	auto sharedData = std::make_shared<ExtractStatsMaker>();
297	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
298	ConsolePrintDetail(consoleDetail);	×
299	sharedData->scanBar =	×
UNCOV 300	AppendNewLogLine<LoadingBar>("Processing extract stats.");	×
301
UNCOV 302	batch.forEachFile = [payload = sharedData,	×
UNCOV 303	ctx = batch.ctx](AppContextShare *iCtx) {	×
UNCOV 304	payload->Push(iCtx, ctx->ExtractStat(std::bind(	×
305	[&](size_t offset, size_t size) {
306	return iCtx->GetBuffer(size, offset);	×
307	},
308	std::placeholders::_1, std::placeholders::_2)));
309	};
310
311	return sharedData;	×
312	}
313
UNCOV 314	void ProcessBatch(Batch &batch, ExtractStats *stats) {	×
315	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
316	ConsolePrintDetail(consoleDetail);	×
317	batch.forEachFile = [payload = std::make_shared<UILines>(*stats),	×
318	archiveFiles =
319	std::make_shared<decltype(stats->archiveFiles)>(
UNCOV 320	std::move(stats->archiveFiles)),	×
UNCOV 321	ctx = batch.ctx](AppContextShare *iCtx) {	×
322	auto currentBar = payload->ChooseBar();	×
323	if (currentBar) {	×
UNCOV 324	currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));	×
325	}
326
327	iCtx->forEachFile = [=] {	×
UNCOV 328	if (currentBar) {	×
329	(*currentBar)++;
330	}
331
332	if (payload->totalCount) {	×
333	(*payload->totalCount)++;
334	}
335	};
336
UNCOV 337	printline("Processing: " << iCtx->FullPath());	×
338	ctx->ProcessFile(iCtx);
UNCOV 339	if (payload->totalProgress) {	×
340	(*payload->totalProgress)++;
341	}
342	};
343	}
344
345	void ProcessBatch(Batch &batch, size_t numFiles) {	×
346	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
UNCOV 347	ConsolePrintDetail(consoleDetail);	×
348	auto payload = std::make_shared<UILines>(numFiles);
UNCOV 349	batch.forEachFile = [payload = payload,	×
UNCOV 350	ctx = batch.ctx](AppContextShare *iCtx) {	×
351	iCtx->forEachFile = [=] {	×
UNCOV 352	if (payload->totalOutCount) {	×
353	(*payload->totalOutCount)++;
354	}
355	};
356	printline("Processing: " << iCtx->FullPath());	×
357	ctx->ProcessFile(iCtx);
358	if (payload->totalProgress) {	×
359	(*payload->totalProgress)++;
360	}
UNCOV 361	if (payload->totalCount) {	×
362	(*payload->totalCount)++;
363	}
364	};
365
UNCOV 366	auto totalFiles = std::make_shared<size_t>(numFiles);	×
367	batch.updateFileCount = [payload = payload,	×
368	totalFiles = totalFiles](size_t addedFiles) {	×
369	*totalFiles.get() += addedFiles;	×
UNCOV 370	payload->totalProgress->ItemCount(*totalFiles);	×
371	};
372	}
373
UNCOV 374	int CreateContent(const std::string &moduleName, const std::string &appFolder,	×
375	const std::string &appName, APPContext &ctx) {
376	try {
UNCOV 377	ctx = APPContext(moduleName.c_str(), appFolder, appName);	×
378	} catch (const std::exception &e) {	×
UNCOV 379	printerror(e.what());	×
380	return 2;
381	}	×
382
383	ConsolePrintDetail(0);	×
384
UNCOV 385	printline(ctx.info->header);	×
UNCOV 386	printline(appHeader0 << appName << ' ' << moduleName << appHeader1);	×
387
UNCOV 388	return 0;	×
389	}
390
UNCOV 391	int LoadProject(const std::string &path, const std::string &appFolder,	×
392	const std::string &appName) {
393	std::ifstream str(path);	×
UNCOV 394	nlohmann::json project(nlohmann::json::parse(str));	×
395
396	APPContext ctx;
397	std::string moduleName = project["module"];	×
398
399	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
400	return ret;
401	}
402
UNCOV 403	ConsolePrintDetail(1);	×
404
405	std::string outputDir;
406
UNCOV 407	if (!project["output_dir"].is_null()) {	×
UNCOV 408	std::string outputDir = project["output_dir"];	×
409	ctx.ApplySetting("out", outputDir);	×
410	}
411
412	bool noConfig = !project["no_config"].is_null() && project["no_config"];	×
413
414	if (!noConfig) {
UNCOV 415	printinfo("Loading config: " << appName << ".config");	×
416	ctx.FromConfig();	×
417	}
418
419	nlohmann::json settings = project["settings"];	×
420
UNCOV 421	for (auto &s : settings.items()) {	×
422	std::string dumped = s.value().dump();	×
UNCOV 423	if (dumped.front() == '"') {	×
424	dumped.erase(0, 1);	×
425	}
UNCOV 426	if (dumped.back() == '"') {	×
UNCOV 427	dumped.erase(dumped.size() - 1);	×
428	}
429	ctx.ApplySetting(s.key(), dumped);	×
430	}
431
UNCOV 432	nlohmann::json inputs = project["inputs"];	×
433
434	InitTempStorage();	×
435	ctx.SetupModule();	×
436	std::unique_ptr<AppPackContext> archiveContext;
437
438	{
439	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
440	AFileInfo batchPath(path);	×
441	std::string batchBase(batchPath.GetFolder());	×
442
UNCOV 443	if (ctx.NewArchive) {	×
UNCOV 444	std::string folder(batchPath.GetFolder());	×
445	std::string archive(batchPath.GetFullPathNoExt());	×
446	archiveContext.reset(batch.ctx->NewArchive(archive));	×
447	MergePackModeBatch(batch, folder, archiveContext.get());	×
448	} else {
UNCOV 449	if (ctx.ExtractStat) {	×
450	auto stats = ExtractStatBatch(batch);	×
UNCOV 451	for (std::string input : inputs) {	×
UNCOV 452	batch.AddFile(batchBase + input);	×
453	}
454
455	batch.FinishBatch();	×
456	batch.Clean();	×
457	stats.get()->totalFiles += inputs.size();	×
458	ProcessBatch(batch, stats.get());	×
459	} else {
460	ProcessBatch(batch, inputs.size());	×
461	}
462	}
463
UNCOV 464	if (ctx.info->batchControlFilters.size() > 0) {	×
465	std::string pathDir(AFileInfo(path).GetFolder());	×
466	for (nlohmann::json input : inputs) {	×
467	if (input.is_array()) {	×
UNCOV 468	batch.AddBatch(input, pathDir);	×
469	} else {
UNCOV 470	printwarning("Expected group, got " << input.type_name()	×
471	<< " instead. Skipping input.");
472	}
473	}
474	} else {
475	for (nlohmann::json input : inputs) {	×
UNCOV 476	if (input.is_string()) {	×
UNCOV 477	batch.AddFile(batchBase + std::string(input));	×
478	} else {
479	printwarning("Expected path string, got "	×
480	<< input.type_name() << " instead. Skipping input.");
481	}
482	}
483	}
484
UNCOV 485	batch.FinishBatch();	×
486	}
487
UNCOV 488	if (archiveContext) {	×
UNCOV 489	ConsolePrintDetail(1);	×
490	archiveContext->Finish();	×
491	}
492
493	if (ctx.FinishContext) {	×
494	ctx.FinishContext();
495	}
496
497	return 0;
498	}
499
500	int Main(int argc, TCHAR *argv[]) {	×
UNCOV 501	ConsolePrintDetail(1);	×
UNCOV 502	AFileInfo appLocation(std::to_string(*argv));	×
503	std::string appFolder(appLocation.GetFolder());	×
UNCOV 504	std::string appName(appLocation.GetFilename());	×
505	es::SetDllRunPath(appFolder + "lib");	×
506
UNCOV 507	if (argc < 2) {	×
508	printwarning(	×
509	"No parameters provided, entering scan mode and generating config.");
UNCOV 510	return !ScanModules(appFolder, appName);	×
511	}
512
UNCOV 513	auto moduleName = std::to_string(argv[1]);	×
514
UNCOV 515	if (moduleName == "--make-doc") {	×
516	std::string templatePath;
517
UNCOV 518	if (argc < 3) {	×
UNCOV 519	printwarning("Expexted template path!");	×
520	} else {
521	templatePath = std::to_string(argv[2]);	×
522	}
523
UNCOV 524	GenerateDocumentation(appFolder, appName, templatePath);	×
525	return 0;
UNCOV 526	} else if (moduleName.ends_with(".json")) {	×
527	return LoadProject(moduleName, appFolder, appName);	×
528	}
529
UNCOV 530	if (argc < 3) {	×
531	printerror("Insufficient argument count, expected parameters.");	×
532	return 1;
533	}
534
535	APPContext ctx;
536
537	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
538	return ret;
539	}
540
UNCOV 541	if (IsHelp(argv[2])) {	×
542	ctx.PrintCLIHelp();	×
543	return 0;
544	}
545
UNCOV 546	ConsolePrintDetail(1);	×
UNCOV 547	bool dontLoadConfig = false;	×
UNCOV 548	std::vector<bool> markedFiles(size_t(argc), false);	×
549	size_t totalFiles = 0;
550
551	// Handle cli options and switches
552	for (int a = 2; a < argc; a++) {	×
553	auto opt = argv[a];	×
554
UNCOV 555	if (opt[0] == '-') {	×
556	auto optStr = std::to_string(opt);
557	std::string_view optsw(optStr);
558
559	if (optsw != "--out") {	×
560	// We won't use config file, reset all booleans to false,
561	// so we can properly use cli switches
UNCOV 562	[&] {	×
UNCOV 563	if (dontLoadConfig) {	×
564	return;
565	}
566
UNCOV 567	printinfo("CLI option detected, config won't be loaded, all booleans "	×
568	"set to false!");
569	ctx.ResetSwitchSettings();	×
UNCOV 570	}();	×
571	dontLoadConfig = true;	×
572	}
573	optsw.remove_prefix(1);
574
UNCOV 575	if (opt[0] == '-') {	×
576	optsw.remove_prefix(1);
577	}
578
UNCOV 579	auto valStr = std::to_string(argv[a + 1]);	×
580
581	if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {	×
582	a++;	×
583	}
584
585	} else {
586	markedFiles[a] = true;
587	totalFiles++;	×
588	}
589	}
590
591	if (!dontLoadConfig) {	×
592	printinfo("Loading config: " << appName << ".config");	×
UNCOV 593	ctx.FromConfig();	×
594	}
595
596	InitTempStorage();	×
597	ctx.SetupModule();	×
598	{
UNCOV 599	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
600
UNCOV 601	if (ctx.NewArchive) {	×
UNCOV 602	PackModeBatch(batch);	×
603	} else {
604	if (ctx.ExtractStat) {	×
605	auto stats = ExtractStatBatch(batch);	×
606	for (int a = 2; a < argc; a++) {	×
UNCOV 607	if (!markedFiles.at(a)) {	×
608	continue;	×
609	}
UNCOV 610	batch.AddFile(std::to_string(argv[a]));	×
611	}
612
613	batch.FinishBatch();	×
614	batch.Clean();	×
UNCOV 615	stats.get()->totalFiles += totalFiles;	×
616	ProcessBatch(batch, stats.get());	×
617	} else {
UNCOV 618	ProcessBatch(batch, totalFiles);	×
619	}
620	}
621
622	for (int a = 2; a < argc; a++) {	×
UNCOV 623	if (!markedFiles.at(a)) {	×
UNCOV 624	continue;	×
625	}
UNCOV 626	batch.AddFile(std::to_string(argv[a]));	×
627	}
628
629	batch.FinishBatch();	×
630	}
631
632	if (ctx.FinishContext) {	×
633	ctx.FinishContext();
634	}
635
636	return 0;
637	}
638
UNCOV 639	int _tmain(int argc, TCHAR *argv[]) {	×
640	es::SetupWinApiConsole();
UNCOV 641	InitConsole();	×
UNCOV 642	CleanTempStorages();	×
643
644	int retVal = Main(argc, argv);	×
645
UNCOV 646	CleanCurrentTempStorage();	×
647
648	#ifndef NDEBUG
649	auto cacheStats = CacheGenerator::GlobalMetrics();
650	PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
651	" search misses: ", cacheStats.numSearchMisses);
652	std::this_thread::sleep_for(std::chrono::milliseconds(100));
653	#endif
UNCOV 654	TerminateConsole();	×
655	return retVal;
656	}

PredatorCZ / PreCore / 559

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous