PredatorCZ / PreCore / 563

Committed 15 Oct 2025 05:00PM UTC coverage: 52.313% (-0.06%) from 52.372%

Build # 563

Build Type

push

github

Committed by

PredatorCZ

Commit Message

use table for module list in docgen

Run Details

0 of 13 new or added lines in 1 file covered. (0.0%)

1 existing line in 1 file now uncovered.

4128 of 7891 relevant lines covered (52.31%)

10512.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/cli/spike.cpp

/*  Spike is universal dedicated module handler
    This source contains code for CLI master app

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "nlohmann/json.hpp"
#include "project.h"
#include "spike/app/batch.hpp"
#include "spike/app/console.hpp"
#include "spike/app/tmp_storage.hpp"
#include "spike/io/binwritter.hpp"
#include "spike/io/stat.hpp"
#include "spike/master_printer.hpp"
#include "spike/type/tchar.hpp"
#include "spike/util/pugiex.hpp"
#include <thread>

static const char appHeader0[] =
    "Simply drag'n'drop files/folders onto application or "
    "use as ";
static const char appHeader1[] =
    " [options] path1 path2 ...\nTool can detect and scan folders and "
    "uncompressed zip archives.";

struct ProcessedFiles : LoadingBar, CounterLine {
  char buffer[128]{};
  CounterLine producedFiles;

  ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}
  void PrintLine() override {
    snprintf(buffer, sizeof(buffer),
             "Processed %4" PRIuMAX " files. Produced %4" PRIuMAX " files.",
             size_t(curitem), size_t(producedFiles));
    LoadingBar::PrintLine();
  }
};

struct ExtractStats {
  std::map<JenHash, size_t> archiveFiles;
  size_t totalFiles = 0;
};

struct UILines {
  ProgressBar *totalProgress{nullptr};
  CounterLine *totalCount{nullptr};
  CounterLine *totalOutCount{nullptr};
  std::map<uint32, ProgressBar *> bars;
  std::mutex barsMutex;

  auto ChooseBar() {
    if (bars.empty()) {
      return (ProgressBar *)(nullptr);
    }

    auto threadId = std::this_thread::get_id();
    auto id = reinterpret_cast<const uint32 &>(threadId);
    auto found = bars.find(id);

    if (es::IsEnd(bars, found)) {
      std::lock_guard<std::mutex> lg(barsMutex);
      auto retVal = bars.begin()->second;
      bars.emplace(id, retVal);
      bars.erase(bars.begin());

      return retVal;
    }

    return found->second;
  };

  UILines(const ExtractStats &stats) {
    ModifyElements([&](ElementAPI &api) {
      const size_t minThreads =
          std::min(size_t(std::thread::hardware_concurrency()),
                   stats.archiveFiles.size());

      if (minThreads < 2) {
        return;
      }

      for (size_t t = 0; t < minThreads; t++) {
        auto progBar = std::make_unique<ProgressBar>("Thread:");
        auto progBarRaw = progBar.get();
        bars.emplace(t, progBarRaw);
        api.Append(std::move(progBar));
      }
    });

    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(stats.totalFiles);
    totalCount = prog;
  }

  UILines(size_t totalInputFiles) {
    auto procFiles = AppendNewLogLine<ProcessedFiles>();
    totalCount = procFiles;
    totalOutCount = &procFiles->producedFiles;
    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(totalInputFiles);
    totalProgress = prog;
  }

  ~UILines() {
    ModifyElements([&](ElementAPI &api) {
      if (totalCount) {
        // Wait a little bit for internal queues to finish printing
        std::this_thread::sleep_for(std::chrono::milliseconds(100));
        if (totalProgress) {
          auto data = static_cast<ProcessedFiles *>(totalCount);
          data->Finish();
          api.Release(data);
        } else {
          auto data = static_cast<DetailedProgressBar *>(totalCount);
          api.Remove(data);
        }
      }
      api.Clean();
      std::this_thread::sleep_for(std::chrono::milliseconds(100));
    });
  }
};

bool ScanModules(const std::string &appFolder, const std::string &appName) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  bool isOkay = true;

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      APPContext ctx(moduleNameStr.data(), appFolder, appName);
      ctx.FromConfig();
    } catch (const std::runtime_error &e) {
      printerror(e.what());
      isOkay = false;
    }
  }

  return isOkay;
}

void GenerateDocumentation(const std::string &appFolder,
                           const std::string &appName,
                           const std::string &templatePath, bool tableModList) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  std::set<std::string> modules;
  pugi::xml_document doc;

  if (!templatePath.empty()) {
    doc = XMLFromFile(templatePath);
  }

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      modules.emplace(moduleNameStr);
    } catch (const std::runtime_error &e) {
      printerror(e.what());
    }
  }

  BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");

  const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";

  if (auto child = doc.child("toolset_description"); child) {
    toolsetDescription = child.text().as_string();
  }

  const char *tag = tableModList ? "table" : "ul";

  wr.BaseStream() << toolsetDescription << "<h2>Module list</h2>\n<" << tag
                  << ">\n";
  std::stringstream str;

  for (auto &m : modules) {
    pugi::xml_node node = doc.child(m.data());
    APPContext ctx(m.data(), appFolder, appName);
    ctx.GetMarkdownDoc(str, node);
    std::string className = ctx.GetClassName(node);
    std::string classNameLink = className;
    std::replace_if(
        classNameLink.begin(), classNameLink.end(),
        [](char c) { return c == ' '; }, '-');

    if (tableModList) {
      std::string_view desc = ctx.info->header;

      if (size_t foundv = desc.find(" v"); foundv != desc.npos &&
                                           desc.at(foundv + 2) >= '0' &&
                                           desc.at(foundv + 2) <= '9') {
        desc = desc.substr(0, foundv);
      }

      wr.BaseStream() << "<tr><td><a href=\"#" << classNameLink << "\">"
                      << className << "</a></td><td>" << desc << "</td></tr>\n";
    } else {
      wr.BaseStream() << "<li><a href=\"#" << classNameLink << "\">"
                      << className << "</a></li>\n";
    }
  }

  wr.BaseStream() << "</" << tag << ">\n\n" << str.str() << "\n\n";

  if (auto child = doc.child("toolset_footer"); child) {
    wr.BaseStream() << child.text().as_string();
  }
}

void PackModeBatch(Batch &batch) {
  struct PackData {
    size_t index = 0;
    std::unique_ptr<AppPackContext> archiveContext;
    std::string pbarLabel;
    DetailedProgressBar *progBar = nullptr;
    std::string folderPath;
  };

  auto payload = std::make_shared<PackData>();

  batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,
                                                   size_t numFiles) {
    payload->folderPath = path;
    payload->archiveContext.reset(ctx->NewArchive(path));
    payload->pbarLabel = "Folder id " + std::to_string(payload->index++);
    payload->progBar =
        AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);
    payload->progBar->ItemCount(numFiles);
    uint8 consoleDetail = 1 | uint8(ctx->info->multithreaded) << 1;
    ConsolePrintDetail(consoleDetail);
    printline("Processing: " << path);
  };

  batch.forEachFile = [payload](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {
      int notSlash = !payload->folderPath.ends_with('/');
      payload->archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +
                                                 notSlash),
          iCtx->GetStream());
    } else {
      payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                                        iCtx->GetStream());
    }
    (*payload->progBar)++;
  };

  batch.forEachFolderFinish = [payload] {
    ConsolePrintDetail(1);
    payload->archiveContext->Finish();
    payload->archiveContext.reset();
    RemoveLogLines(payload->progBar);
  };
}

void MergePackModeBatch(Batch &batch, const std::string &folderPath,
                        AppPackContext *archiveContext) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  printline("Processing: " << folderPath);

  batch.forEachFile = [=](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {
      int notSlash = !folderPath.ends_with('/');
      archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),
          iCtx->GetStream());
    } else {
      archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                               iCtx->GetStream());
    }
  };
}

auto ExtractStatBatch(Batch &batch) {
  struct ExtractStatsMaker : ExtractStats {
    std::mutex mtx;
    LoadingBar *scanBar;

    void Push(AppContextShare *ctx, size_t numFiles) {
      std::unique_lock<std::mutex> lg(mtx);
      archiveFiles.emplace(ctx->Hash(), numFiles);
      totalFiles += numFiles;
    }

    ~ExtractStatsMaker() { RemoveLogLines(scanBar); }
  };

  batch.keepFinishLines = false;
  auto sharedData = std::make_shared<ExtractStatsMaker>();
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  sharedData->scanBar =
      AppendNewLogLine<LoadingBar>("Processing extract stats.");

  batch.forEachFile = [payload = sharedData,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    payload->Push(iCtx, ctx->ExtractStat(std::bind(
                            [&](size_t offset, size_t size) {
                              return iCtx->GetBuffer(size, offset);
                            },
                            std::placeholders::_1, std::placeholders::_2)));
  };

  return sharedData;
}

void ProcessBatch(Batch &batch, ExtractStats *stats) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  batch.forEachFile = [payload = std::make_shared<UILines>(*stats),
                       archiveFiles =
                           std::make_shared<decltype(stats->archiveFiles)>(
                               std::move(stats->archiveFiles)),
                       ctx = batch.ctx](AppContextShare *iCtx) {
    auto currentBar = payload->ChooseBar();
    if (currentBar) {
      currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));
    }

    iCtx->forEachFile = [=] {
      if (currentBar) {
        (*currentBar)++;
      }

      if (payload->totalCount) {
        (*payload->totalCount)++;
      }
    };

    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
  };
}

void ProcessBatch(Batch &batch, size_t numFiles) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  auto payload = std::make_shared<UILines>(numFiles);
  batch.forEachFile = [payload = payload,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    iCtx->forEachFile = [=] {
      if (payload->totalOutCount) {
        (*payload->totalOutCount)++;
      }
    };
    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
    if (payload->totalCount) {
      (*payload->totalCount)++;
    }
  };

  auto totalFiles = std::make_shared<size_t>(numFiles);
  batch.updateFileCount = [payload = payload,
                           totalFiles = totalFiles](size_t addedFiles) {
    *totalFiles.get() += addedFiles;
    payload->totalProgress->ItemCount(*totalFiles);
  };
}

int CreateContent(const std::string &moduleName, const std::string &appFolder,
                  const std::string &appName, APPContext &ctx) {
  try {
    ctx = APPContext(moduleName.c_str(), appFolder, appName);
  } catch (const std::exception &e) {
    printerror(e.what());
    return 2;
  }

  ConsolePrintDetail(0);

  printline(ctx.info->header);
  printline(appHeader0 << appName << ' ' << moduleName << appHeader1);

  return 0;
}

int LoadProject(const std::string &path, const std::string &appFolder,
                const std::string &appName) {
  std::ifstream str(path);
  nlohmann::json project(nlohmann::json::parse(str));

  APPContext ctx;
  std::string moduleName = project["module"];

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  ConsolePrintDetail(1);

  std::string outputDir;

  if (!project["output_dir"].is_null()) {
    std::string outputDir = project["output_dir"];
    ctx.ApplySetting("out", outputDir);
  }

  bool noConfig = !project["no_config"].is_null() && project["no_config"];

  if (!noConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  nlohmann::json settings = project["settings"];

  for (auto &s : settings.items()) {
    std::string dumped = s.value().dump();
    if (dumped.front() == '"') {
      dumped.erase(0, 1);
    }
    if (dumped.back() == '"') {
      dumped.erase(dumped.size() - 1);
    }
    ctx.ApplySetting(s.key(), dumped);
  }

  nlohmann::json inputs = project["inputs"];

  InitTempStorage();
  ctx.SetupModule();
  std::unique_ptr<AppPackContext> archiveContext;

  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);
    AFileInfo batchPath(path);
    std::string batchBase(batchPath.GetFolder());

    if (ctx.NewArchive) {
      std::string folder(batchPath.GetFolder());
      std::string archive(batchPath.GetFullPathNoExt());
      archiveContext.reset(batch.ctx->NewArchive(archive));
      MergePackModeBatch(batch, folder, archiveContext.get());
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (std::string input : inputs) {
          batch.AddFile(batchBase + input);
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += inputs.size();
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, inputs.size());
      }
    }

    if (ctx.info->batchControlFilters.size() > 0) {
      std::string pathDir(AFileInfo(path).GetFolder());
      for (nlohmann::json input : inputs) {
        if (input.is_array()) {
          batch.AddBatch(input, pathDir);
        } else {
          printwarning("Expected group, got " << input.type_name()
                                              << " instead. Skipping input.");
        }
      }
    } else {
      for (nlohmann::json input : inputs) {
        if (input.is_string()) {
          batch.AddFile(batchBase + std::string(input));
        } else {
          printwarning("Expected path string, got "
                       << input.type_name() << " instead. Skipping input.");
        }
      }
    }

    batch.FinishBatch();
  }

  if (archiveContext) {
    ConsolePrintDetail(1);
    archiveContext->Finish();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int Main(int argc, TCHAR *argv[]) {
  ConsolePrintDetail(1);
  AFileInfo appLocation(std::to_string(*argv));
  std::string appFolder(appLocation.GetFolder());
  std::string appName(appLocation.GetFilename());
  es::SetDllRunPath(appFolder + "lib");

  if (argc < 2) {
    printwarning(
        "No parameters provided, entering scan mode and generating config.");
    return !ScanModules(appFolder, appName);
  }

  auto moduleName = std::to_string(argv[1]);

  if (moduleName == "--make-doc") {
    std::string templatePath;

    if (argc < 3) {
      printwarning("Expexted template path!");
    } else {
      templatePath = std::to_string(argv[2]);
    }

    GenerateDocumentation(appFolder, appName, templatePath,
                          argc > 3 &&
                              std::string_view(argv[3]) == "use_table");
    return 0;
  } else if (moduleName.ends_with(".json")) {
    return LoadProject(moduleName, appFolder, appName);
  }

  if (argc < 3) {
    printerror("Insufficient argument count, expected parameters.");
    return 1;
  }

  APPContext ctx;

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  if (IsHelp(argv[2])) {
    ctx.PrintCLIHelp();
    return 0;
  }

  ConsolePrintDetail(1);
  bool dontLoadConfig = false;
  std::vector<bool> markedFiles(size_t(argc), false);
  size_t totalFiles = 0;

  // Handle cli options and switches
  for (int a = 2; a < argc; a++) {
    auto opt = argv[a];

    if (opt[0] == '-') {
      auto optStr = std::to_string(opt);
      std::string_view optsw(optStr);

      if (optsw != "--out") {
        // We won't use config file, reset all booleans to false,
        // so we can properly use cli switches
        [&] {
          if (dontLoadConfig) {
            return;
          }

          printinfo("CLI option detected, config won't be loaded, all booleans "
                    "set to false!");
          ctx.ResetSwitchSettings();
        }();
        dontLoadConfig = true;
      }
      optsw.remove_prefix(1);

      if (opt[0] == '-') {
        optsw.remove_prefix(1);
      }

      auto valStr = std::to_string(argv[a + 1]);

      if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {
        a++;
      }

    } else {
      markedFiles[a] = true;
      totalFiles++;
    }
  }

  if (!dontLoadConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  InitTempStorage();
  ctx.SetupModule();
  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);

    if (ctx.NewArchive) {
      PackModeBatch(batch);
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (int a = 2; a < argc; a++) {
          if (!markedFiles.at(a)) {
            continue;
          }
          batch.AddFile(std::to_string(argv[a]));
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += totalFiles;
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, totalFiles);
      }
    }

    for (int a = 2; a < argc; a++) {
      if (!markedFiles.at(a)) {
        continue;
      }
      batch.AddFile(std::to_string(argv[a]));
    }

    batch.FinishBatch();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int _tmain(int argc, TCHAR *argv[]) {
  es::SetupWinApiConsole();
  InitConsole();
  CleanTempStorages();

  int retVal = Main(argc, argv);

  CleanCurrentTempStorage();

#ifndef NDEBUG
  auto cacheStats = CacheGenerator::GlobalMetrics();
  PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
            " search misses: ", cacheStats.numSearchMisses);
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
#endif
  TerminateConsole();
  return retVal;
}

1	/* Spike is universal dedicated module handler
2	This source contains code for CLI master app
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "nlohmann/json.hpp"
20	#include "project.h"
21	#include "spike/app/batch.hpp"
22	#include "spike/app/console.hpp"
23	#include "spike/app/tmp_storage.hpp"
24	#include "spike/io/binwritter.hpp"
25	#include "spike/io/stat.hpp"
26	#include "spike/master_printer.hpp"
27	#include "spike/type/tchar.hpp"
28	#include "spike/util/pugiex.hpp"
29	#include <thread>
30
31	static const char appHeader0[] =
32	"Simply drag'n'drop files/folders onto application or "
33	"use as ";
34	static const char appHeader1[] =
35	" [options] path1 path2 ...\nTool can detect and scan folders and "
36	"uncompressed zip archives.";
37
38	struct ProcessedFiles : LoadingBar, CounterLine {
39	char buffer[128]{};
40	CounterLine producedFiles;
41
42	ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}	×
43	void PrintLine() override {	×
44	snprintf(buffer, sizeof(buffer),	×
45	"Processed %4" PRIuMAX " files. Produced %4" PRIuMAX " files.",
46	size_t(curitem), size_t(producedFiles));
47	LoadingBar::PrintLine();	×
48	}
49	};
50
51	struct ExtractStats {
52	std::map<JenHash, size_t> archiveFiles;
53	size_t totalFiles = 0;
54	};
55
56	struct UILines {
57	ProgressBar *totalProgress{nullptr};
58	CounterLine *totalCount{nullptr};
59	CounterLine *totalOutCount{nullptr};
60	std::map<uint32, ProgressBar *> bars;
61	std::mutex barsMutex;
62
63	auto ChooseBar() {	×
64	if (bars.empty()) {	×
65	return (ProgressBar *)(nullptr);
66	}
67
68	auto threadId = std::this_thread::get_id();
69	auto id = reinterpret_cast<const uint32 &>(threadId);	×
70	auto found = bars.find(id);
71
72	if (es::IsEnd(bars, found)) {	×
73	std::lock_guard<std::mutex> lg(barsMutex);	×
74	auto retVal = bars.begin()->second;	×
75	bars.emplace(id, retVal);	×
76	bars.erase(bars.begin());
77
78	return retVal;
79	}
80
81	return found->second;	×
82	};
83
84	UILines(const ExtractStats &stats) {	×
85	ModifyElements([&](ElementAPI &api) {	×
86	const size_t minThreads =
87	std::min(size_t(std::thread::hardware_concurrency()),	×
88	stats.archiveFiles.size());	×
89
90	if (minThreads < 2) {	×
91	return;
92	}
93
94	for (size_t t = 0; t < minThreads; t++) {	×
95	auto progBar = std::make_unique<ProgressBar>("Thread:");	×
96	auto progBarRaw = progBar.get();	×
97	bars.emplace(t, progBarRaw);	×
98	api.Append(std::move(progBar));	×
99	}
100	});
101
102	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
103	prog->ItemCount(stats.totalFiles);	×
104	totalCount = prog;	×
105	}
106
107	UILines(size_t totalInputFiles) {	×
108	auto procFiles = AppendNewLogLine<ProcessedFiles>();	×
109	totalCount = procFiles;	×
110	totalOutCount = &procFiles->producedFiles;	×
111	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
112	prog->ItemCount(totalInputFiles);	×
113	totalProgress = prog;	×
114	}
115
116	~UILines() {	×
117	ModifyElements([&](ElementAPI &api) {	×
118	if (totalCount) {	×
119	// Wait a little bit for internal queues to finish printing
120	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
121	if (totalProgress) {	×
122	auto data = static_cast<ProcessedFiles *>(totalCount);	×
123	data->Finish();
124	api.Release(data);	×
125	} else {
126	auto data = static_cast<DetailedProgressBar *>(totalCount);	×
127	api.Remove(data);	×
128	}
129	}
130	api.Clean();	×
131	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
132	});	×
133	}
134	};
135
136	bool ScanModules(const std::string &appFolder, const std::string &appName) {	×
137	DirectoryScanner sc;	×
138	sc.AddFilter(std::string_view(".spk$"));	×
139	sc.Scan(appFolder);	×
140	bool isOkay = true;
141
142	for (auto &m : sc) {	×
143	try {
144	AFileInfo modulePath(m);	×
145	auto moduleName = modulePath.GetFilename();	×
146	const size_t firstDotPos = moduleName.find_first_of('.');
147	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
148	APPContext ctx(moduleNameStr.data(), appFolder, appName);	×
149	ctx.FromConfig();	×
150	} catch (const std::runtime_error &e) {	×
151	printerror(e.what());	×
152	isOkay = false;
153	}	×
154	}
155
156	return isOkay;	×
157	}
158
159	void GenerateDocumentation(const std::string &appFolder,	×
160	const std::string &appName,
161	const std::string &templatePath, bool tableModList) {
162	DirectoryScanner sc;	×
163	sc.AddFilter(std::string_view(".spk$"));	×
164	sc.Scan(appFolder);	×
165	std::set<std::string> modules;
166	pugi::xml_document doc;	×
167
168	if (!templatePath.empty()) {	×
169	doc = XMLFromFile(templatePath);	×
170	}
171
172	for (auto &m : sc) {	×
173	try {
174	AFileInfo modulePath(m);	×
175	auto moduleName = modulePath.GetFilename();	×
176	const size_t firstDotPos = moduleName.find_first_of('.');
177	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
178	modules.emplace(moduleNameStr);
179	} catch (const std::runtime_error &e) {	×
180	printerror(e.what());	×
181	}	×
182	}
183
184	BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");	×
185
186	const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";
187
188	if (auto child = doc.child("toolset_description"); child) {	×
189	toolsetDescription = child.text().as_string();	×
190	}
191
NEW 192	const char *tag = tableModList ? "table" : "ul";	×
193
194	wr.BaseStream() << toolsetDescription << "<h2>Module list</h2>\n<" << tag
NEW 195	<< ">\n";	×
UNCOV 196	std::stringstream str;	×
197
198	for (auto &m : modules) {	×
199	pugi::xml_node node = doc.child(m.data());	×
200	APPContext ctx(m.data(), appFolder, appName);	×
201	ctx.GetMarkdownDoc(str, node);	×
202	std::string className = ctx.GetClassName(node);	×
203	std::string classNameLink = className;
204	std::replace_if(
205	classNameLink.begin(), classNameLink.end(),
206	[](char c) { return c == ' '; }, '-');
207
NEW 208	if (tableModList) {	×
NEW 209	std::string_view desc = ctx.info->header;	×
210
NEW 211	if (size_t foundv = desc.find(" v"); foundv != desc.npos &&	×
NEW 212	desc.at(foundv + 2) >= '0' &&	×
213	desc.at(foundv + 2) <= '9') {
NEW 214	desc = desc.substr(0, foundv);	×
215	}
216
217	wr.BaseStream() << "<tr><td><a href=\"#" << classNameLink << "\">"
NEW 218	<< className << "</a></td><td>" << desc << "</td></tr>\n";	×
219	} else {
220	wr.BaseStream() << "<li><a href=\"#" << classNameLink << "\">"
NEW 221	<< className << "</a></li>\n";	×
222	}
223	}
224
NEW 225	wr.BaseStream() << "</" << tag << ">\n\n" << str.str() << "\n\n";	×
226
227	if (auto child = doc.child("toolset_footer"); child) {	×
228	wr.BaseStream() << child.text().as_string();	×
229	}
230	}
231
232	void PackModeBatch(Batch &batch) {	×
233	struct PackData {
234	size_t index = 0;
235	std::unique_ptr<AppPackContext> archiveContext;
236	std::string pbarLabel;
237	DetailedProgressBar *progBar = nullptr;
238	std::string folderPath;
239	};
240
241	auto payload = std::make_shared<PackData>();
242
243	batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,	×
244	size_t numFiles) {	×
245	payload->folderPath = path;	×
246	payload->archiveContext.reset(ctx->NewArchive(path));
247	payload->pbarLabel = "Folder id " + std::to_string(payload->index++);	×
248	payload->progBar =	×
249	AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);	×
250	payload->progBar->ItemCount(numFiles);	×
251	uint8 consoleDetail = 1 \| uint8(ctx->info->multithreaded) << 1;	×
252	ConsolePrintDetail(consoleDetail);	×
253	printline("Processing: " << path);	×
254	};
255
256	batch.forEachFile = [payload](AppContextShare *iCtx) {	×
257	if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {	×
258	int notSlash = !payload->folderPath.ends_with('/');	×
259	payload->archiveContext->SendFile(	×
260	iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +	×
261	notSlash),	×
262	iCtx->GetStream());	×
263	} else {
264	payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
265	iCtx->GetStream());	×
266	}
267	(*payload->progBar)++;	×
268	};
269
270	batch.forEachFolderFinish = [payload] {	×
271	ConsolePrintDetail(1);	×
272	payload->archiveContext->Finish();	×
273	payload->archiveContext.reset();
274	RemoveLogLines(payload->progBar);	×
275	};
276	}
277
278	void MergePackModeBatch(Batch &batch, const std::string &folderPath,	×
279	AppPackContext *archiveContext) {
280	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
281	ConsolePrintDetail(consoleDetail);	×
282	printline("Processing: " << folderPath);	×
283
284	batch.forEachFile = [=](AppContextShare *iCtx) {	×
285	if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {	×
286	int notSlash = !folderPath.ends_with('/');	×
287	archiveContext->SendFile(	×
288	iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),	×
289	iCtx->GetStream());	×
290	} else {
291	archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
292	iCtx->GetStream());	×
293	}
294	};
295	}
296
297	auto ExtractStatBatch(Batch &batch) {	×
298	struct ExtractStatsMaker : ExtractStats {
299	std::mutex mtx;
300	LoadingBar *scanBar;
301
302	void Push(AppContextShare *ctx, size_t numFiles) {	×
303	std::unique_lock<std::mutex> lg(mtx);	×
304	archiveFiles.emplace(ctx->Hash(), numFiles);	×
305	totalFiles += numFiles;	×
306	}
307
308	~ExtractStatsMaker() { RemoveLogLines(scanBar); }	×
309	};
310
311	batch.keepFinishLines = false;	×
312	auto sharedData = std::make_shared<ExtractStatsMaker>();
313	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
314	ConsolePrintDetail(consoleDetail);	×
315	sharedData->scanBar =	×
316	AppendNewLogLine<LoadingBar>("Processing extract stats.");	×
317
318	batch.forEachFile = [payload = sharedData,	×
319	ctx = batch.ctx](AppContextShare *iCtx) {	×
320	payload->Push(iCtx, ctx->ExtractStat(std::bind(	×
321	[&](size_t offset, size_t size) {
322	return iCtx->GetBuffer(size, offset);	×
323	},
324	std::placeholders::_1, std::placeholders::_2)));
325	};
326
327	return sharedData;	×
328	}
329
330	void ProcessBatch(Batch &batch, ExtractStats *stats) {	×
331	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
332	ConsolePrintDetail(consoleDetail);	×
333	batch.forEachFile = [payload = std::make_shared<UILines>(*stats),	×
334	archiveFiles =
335	std::make_shared<decltype(stats->archiveFiles)>(
336	std::move(stats->archiveFiles)),	×
337	ctx = batch.ctx](AppContextShare *iCtx) {	×
338	auto currentBar = payload->ChooseBar();	×
339	if (currentBar) {	×
340	currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));	×
341	}
342
343	iCtx->forEachFile = [=] {	×
344	if (currentBar) {	×
345	(*currentBar)++;
346	}
347
348	if (payload->totalCount) {	×
349	(*payload->totalCount)++;
350	}
351	};
352
353	printline("Processing: " << iCtx->FullPath());	×
354	ctx->ProcessFile(iCtx);
355	if (payload->totalProgress) {	×
356	(*payload->totalProgress)++;
357	}
358	};
359	}
360
361	void ProcessBatch(Batch &batch, size_t numFiles) {	×
362	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
363	ConsolePrintDetail(consoleDetail);	×
364	auto payload = std::make_shared<UILines>(numFiles);
365	batch.forEachFile = [payload = payload,	×
366	ctx = batch.ctx](AppContextShare *iCtx) {	×
367	iCtx->forEachFile = [=] {	×
368	if (payload->totalOutCount) {	×
369	(*payload->totalOutCount)++;
370	}
371	};
372	printline("Processing: " << iCtx->FullPath());	×
373	ctx->ProcessFile(iCtx);
374	if (payload->totalProgress) {	×
375	(*payload->totalProgress)++;
376	}
377	if (payload->totalCount) {	×
378	(*payload->totalCount)++;
379	}
380	};
381
382	auto totalFiles = std::make_shared<size_t>(numFiles);	×
383	batch.updateFileCount = [payload = payload,	×
384	totalFiles = totalFiles](size_t addedFiles) {	×
385	*totalFiles.get() += addedFiles;	×
386	payload->totalProgress->ItemCount(*totalFiles);	×
387	};
388	}
389
390	int CreateContent(const std::string &moduleName, const std::string &appFolder,	×
391	const std::string &appName, APPContext &ctx) {
392	try {
393	ctx = APPContext(moduleName.c_str(), appFolder, appName);	×
394	} catch (const std::exception &e) {	×
395	printerror(e.what());	×
396	return 2;
397	}	×
398
399	ConsolePrintDetail(0);	×
400
401	printline(ctx.info->header);	×
402	printline(appHeader0 << appName << ' ' << moduleName << appHeader1);	×
403
404	return 0;	×
405	}
406
407	int LoadProject(const std::string &path, const std::string &appFolder,	×
408	const std::string &appName) {
409	std::ifstream str(path);	×
410	nlohmann::json project(nlohmann::json::parse(str));	×
411
412	APPContext ctx;
413	std::string moduleName = project["module"];	×
414
415	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
416	return ret;
417	}
418
419	ConsolePrintDetail(1);	×
420
421	std::string outputDir;
422
423	if (!project["output_dir"].is_null()) {	×
424	std::string outputDir = project["output_dir"];	×
425	ctx.ApplySetting("out", outputDir);	×
426	}
427
428	bool noConfig = !project["no_config"].is_null() && project["no_config"];	×
429
430	if (!noConfig) {
431	printinfo("Loading config: " << appName << ".config");	×
432	ctx.FromConfig();	×
433	}
434
435	nlohmann::json settings = project["settings"];	×
436
437	for (auto &s : settings.items()) {	×
438	std::string dumped = s.value().dump();	×
439	if (dumped.front() == '"') {	×
440	dumped.erase(0, 1);	×
441	}
442	if (dumped.back() == '"') {	×
443	dumped.erase(dumped.size() - 1);	×
444	}
445	ctx.ApplySetting(s.key(), dumped);	×
446	}
447
448	nlohmann::json inputs = project["inputs"];	×
449
450	InitTempStorage();	×
451	ctx.SetupModule();	×
452	std::unique_ptr<AppPackContext> archiveContext;
453
454	{
455	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
456	AFileInfo batchPath(path);	×
457	std::string batchBase(batchPath.GetFolder());	×
458
459	if (ctx.NewArchive) {	×
460	std::string folder(batchPath.GetFolder());	×
461	std::string archive(batchPath.GetFullPathNoExt());	×
462	archiveContext.reset(batch.ctx->NewArchive(archive));	×
463	MergePackModeBatch(batch, folder, archiveContext.get());	×
464	} else {
465	if (ctx.ExtractStat) {	×
466	auto stats = ExtractStatBatch(batch);	×
467	for (std::string input : inputs) {	×
468	batch.AddFile(batchBase + input);	×
469	}
470
471	batch.FinishBatch();	×
472	batch.Clean();	×
473	stats.get()->totalFiles += inputs.size();	×
474	ProcessBatch(batch, stats.get());	×
475	} else {
476	ProcessBatch(batch, inputs.size());	×
477	}
478	}
479
480	if (ctx.info->batchControlFilters.size() > 0) {	×
481	std::string pathDir(AFileInfo(path).GetFolder());	×
482	for (nlohmann::json input : inputs) {	×
483	if (input.is_array()) {	×
484	batch.AddBatch(input, pathDir);	×
485	} else {
486	printwarning("Expected group, got " << input.type_name()	×
487	<< " instead. Skipping input.");
488	}
489	}
490	} else {
491	for (nlohmann::json input : inputs) {	×
492	if (input.is_string()) {	×
493	batch.AddFile(batchBase + std::string(input));	×
494	} else {
495	printwarning("Expected path string, got "	×
496	<< input.type_name() << " instead. Skipping input.");
497	}
498	}
499	}
500
501	batch.FinishBatch();	×
502	}
503
504	if (archiveContext) {	×
505	ConsolePrintDetail(1);	×
506	archiveContext->Finish();	×
507	}
508
509	if (ctx.FinishContext) {	×
510	ctx.FinishContext();
511	}
512
513	return 0;
514	}
515
516	int Main(int argc, TCHAR *argv[]) {	×
517	ConsolePrintDetail(1);	×
518	AFileInfo appLocation(std::to_string(*argv));	×
519	std::string appFolder(appLocation.GetFolder());	×
520	std::string appName(appLocation.GetFilename());	×
521	es::SetDllRunPath(appFolder + "lib");	×
522
523	if (argc < 2) {	×
524	printwarning(	×
525	"No parameters provided, entering scan mode and generating config.");
526	return !ScanModules(appFolder, appName);	×
527	}
528
529	auto moduleName = std::to_string(argv[1]);	×
530
531	if (moduleName == "--make-doc") {	×
532	std::string templatePath;
533
534	if (argc < 3) {	×
535	printwarning("Expexted template path!");	×
536	} else {
537	templatePath = std::to_string(argv[2]);	×
538	}
539
NEW 540	GenerateDocumentation(appFolder, appName, templatePath,	×
NEW 541	argc > 3 &&	×
NEW 542	std::string_view(argv[3]) == "use_table");	×
543	return 0;
544	} else if (moduleName.ends_with(".json")) {	×
545	return LoadProject(moduleName, appFolder, appName);	×
546	}
547
548	if (argc < 3) {	×
549	printerror("Insufficient argument count, expected parameters.");	×
550	return 1;
551	}
552
553	APPContext ctx;
554
555	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
556	return ret;
557	}
558
559	if (IsHelp(argv[2])) {	×
560	ctx.PrintCLIHelp();	×
561	return 0;
562	}
563
564	ConsolePrintDetail(1);	×
565	bool dontLoadConfig = false;	×
566	std::vector<bool> markedFiles(size_t(argc), false);	×
567	size_t totalFiles = 0;
568
569	// Handle cli options and switches
570	for (int a = 2; a < argc; a++) {	×
571	auto opt = argv[a];	×
572
573	if (opt[0] == '-') {	×
574	auto optStr = std::to_string(opt);
575	std::string_view optsw(optStr);
576
577	if (optsw != "--out") {	×
578	// We won't use config file, reset all booleans to false,
579	// so we can properly use cli switches
580	[&] {	×
581	if (dontLoadConfig) {	×
582	return;
583	}
584
585	printinfo("CLI option detected, config won't be loaded, all booleans "	×
586	"set to false!");
587	ctx.ResetSwitchSettings();	×
588	}();	×
589	dontLoadConfig = true;	×
590	}
591	optsw.remove_prefix(1);
592
593	if (opt[0] == '-') {	×
594	optsw.remove_prefix(1);
595	}
596
597	auto valStr = std::to_string(argv[a + 1]);	×
598
599	if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {	×
600	a++;	×
601	}
602
603	} else {
604	markedFiles[a] = true;
605	totalFiles++;	×
606	}
607	}
608
609	if (!dontLoadConfig) {	×
610	printinfo("Loading config: " << appName << ".config");	×
611	ctx.FromConfig();	×
612	}
613
614	InitTempStorage();	×
615	ctx.SetupModule();	×
616	{
617	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
618
619	if (ctx.NewArchive) {	×
620	PackModeBatch(batch);	×
621	} else {
622	if (ctx.ExtractStat) {	×
623	auto stats = ExtractStatBatch(batch);	×
624	for (int a = 2; a < argc; a++) {	×
625	if (!markedFiles.at(a)) {	×
626	continue;	×
627	}
628	batch.AddFile(std::to_string(argv[a]));	×
629	}
630
631	batch.FinishBatch();	×
632	batch.Clean();	×
633	stats.get()->totalFiles += totalFiles;	×
634	ProcessBatch(batch, stats.get());	×
635	} else {
636	ProcessBatch(batch, totalFiles);	×
637	}
638	}
639
640	for (int a = 2; a < argc; a++) {	×
641	if (!markedFiles.at(a)) {	×
642	continue;	×
643	}
644	batch.AddFile(std::to_string(argv[a]));	×
645	}
646
647	batch.FinishBatch();	×
648	}
649
650	if (ctx.FinishContext) {	×
651	ctx.FinishContext();
652	}
653
654	return 0;
655	}
656
657	int _tmain(int argc, TCHAR *argv[]) {	×
658	es::SetupWinApiConsole();
659	InitConsole();	×
660	CleanTempStorages();	×
661
662	int retVal = Main(argc, argv);	×
663
664	CleanCurrentTempStorage();	×
665
666	#ifndef NDEBUG
667	auto cacheStats = CacheGenerator::GlobalMetrics();
668	PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
669	" search misses: ", cacheStats.numSearchMisses);
670	std::this_thread::sleep_for(std::chrono::milliseconds(100));
671	#endif
672	TerminateConsole();	×
673	return retVal;
674	}

PredatorCZ / PreCore / 563

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous