PredatorCZ / PreCore / 505

Committed 24 Mar 2024 11:59AM UTC coverage: 54.243% (-0.8%) from 55.056%

Build # 505

Build Type

push

github

Committed by

PredatorCZ

Commit Message

fix more build fails

Run Details

4142 of 7636 relevant lines covered (54.24%)

8779.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/cli/spike.cpp

/*  Spike is universal dedicated module handler
    This source contains code for CLI master app

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "nlohmann/json.hpp"
#include "project.h"
#include "spike/app/batch.hpp"
#include "spike/app/console.hpp"
#include "spike/app/tmp_storage.hpp"
#include "spike/io/binwritter.hpp"
#include "spike/io/stat.hpp"
#include "spike/master_printer.hpp"
#include "spike/type/tchar.hpp"
#include "spike/util/pugiex.hpp"
#include <thread>

static const char appHeader0[] =
    "Simply drag'n'drop files/folders onto application or "
    "use as ";
static const char appHeader1[] =
    " [options] path1 path2 ...\nTool can detect and scan folders and "
    "uncompressed zip archives.";

struct ProcessedFiles : LoadingBar, CounterLine {
  char buffer[128]{};

  ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}
  void PrintLine() override {
    snprintf(buffer, sizeof(buffer), "Processed %4" PRIuMAX " files.",
             curitem.load(std::memory_order_relaxed));
    LoadingBar::PrintLine();
  }
};

struct ExtractStats {
  std::map<JenHash, size_t> archiveFiles;
  size_t totalFiles = 0;
};

struct UILines {
  ProgressBar *totalProgress{nullptr};
  CounterLine *totalCount{nullptr};
  std::map<uint32, ProgressBar *> bars;
  std::mutex barsMutex;

  auto ChooseBar() {
    if (bars.empty()) {
      return (ProgressBar *)(nullptr);
    }

    auto threadId = std::this_thread::get_id();
    auto id = reinterpret_cast<const uint32 &>(threadId);
    auto found = bars.find(id);

    if (es::IsEnd(bars, found)) {
      std::lock_guard<std::mutex> lg(barsMutex);
      auto retVal = bars.begin()->second;
      bars.emplace(id, retVal);
      bars.erase(bars.begin());

      return retVal;
    }

    return found->second;
  };

  UILines(const ExtractStats &stats) {
    ModifyElements([&](ElementAPI &api) {
      const size_t minThreads =
          std::min(size_t(std::thread::hardware_concurrency()),
                   stats.archiveFiles.size());

      if (minThreads < 2) {
        return;
      }

      for (size_t t = 0; t < minThreads; t++) {
        auto progBar = std::make_unique<ProgressBar>("Thread:");
        auto progBarRaw = progBar.get();
        bars.emplace(t, progBarRaw);
        api.Append(std::move(progBar));
      }
    });

    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(stats.totalFiles);
    totalCount = prog;
  }

  UILines(size_t totalInputFiles) {
    totalCount = AppendNewLogLine<ProcessedFiles>();
    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(totalInputFiles);
    totalProgress = prog;
  }

  ~UILines() {
    ModifyElements([&](ElementAPI &api) {
      if (totalCount) {
        // Wait a little bit for internal queues to finish printing
        std::this_thread::sleep_for(std::chrono::milliseconds(100));
        if (totalProgress) {
          auto data = static_cast<ProcessedFiles *>(totalCount);
          data->Finish();
          api.Release(data);
        } else {
          auto data = static_cast<DetailedProgressBar *>(totalCount);
          api.Remove(data);
        }
      }
      api.Clean();
      std::this_thread::sleep_for(std::chrono::milliseconds(100));
    });
  }
};

bool ScanModules(const std::string &appFolder, const std::string &appName) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  bool isOkay = true;

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      APPContext ctx(moduleNameStr.data(), appFolder, appName);
      ctx.FromConfig();
    } catch (const std::runtime_error &e) {
      printerror(e.what());
      isOkay = false;
    }
  }

  return isOkay;
}

void GenerateDocumentation(const std::string &appFolder,
                           const std::string &appName,
                           const std::string &templatePath) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  std::set<std::string> modules;
  pugi::xml_document doc;

  if (!templatePath.empty()) {
    doc = XMLFromFile(templatePath);
  }

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      modules.emplace(moduleNameStr);
    } catch (const std::runtime_error &e) {
      printerror(e.what());
    }
  }

  BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");
  const char *toolsetName = "[[TOOLSET NAME]]";

  if (auto child = doc.child("toolset_name"); child) {
    toolsetName = child.text().as_string();
  }

  const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";

  if (auto child = doc.child("toolset_description"); child) {
    toolsetDescription = child.text().as_string();
  }

  wr.BaseStream() << "# " << toolsetName << "\n\n"
                  << toolsetDescription << "\n\n";

  for (auto &m : modules) {
    pugi::xml_node node = doc.child(m.data());
    APPContext ctx(m.data(), appFolder, appName);
    ctx.GetMarkdownDoc(wr.BaseStream(), node);
  }

  if (auto child = doc.child("toolset_footer"); child) {
    wr.BaseStream() << child.text().as_string();
  }
}

void PackModeBatch(Batch &batch) {
  struct PackData {
    size_t index = 0;
    std::unique_ptr<AppPackContext> archiveContext;
    std::string pbarLabel;
    DetailedProgressBar *progBar = nullptr;
    std::string folderPath;
  };

  auto payload = std::make_shared<PackData>();

  batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,
                                                   size_t numFiles) {
    payload->folderPath = path;
    payload->archiveContext.reset(ctx->NewArchive(path));
    payload->pbarLabel = "Folder id " + std::to_string(payload->index++);
    payload->progBar =
        AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);
    payload->progBar->ItemCount(numFiles);
    uint8 consoleDetail = 1 | uint8(ctx->info->multithreaded) << 1;
    ConsolePrintDetail(consoleDetail);
    printline("Processing: " << path);
  };

  batch.forEachFile = [payload](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {
      int notSlash = !payload->folderPath.ends_with('/');
      payload->archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +
                                                 notSlash),
          iCtx->GetStream());
    } else {
      payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                                        iCtx->GetStream());
    }
    (*payload->progBar)++;
  };

  batch.forEachFolderFinish = [payload] {
    ConsolePrintDetail(1);
    payload->archiveContext->Finish();
    payload->archiveContext.reset();
    RemoveLogLines(payload->progBar);
  };
}

void MergePackModeBatch(Batch &batch, const std::string &folderPath,
                        AppPackContext *archiveContext) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  printline("Processing: " << folderPath);

  batch.forEachFile = [=](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {
      int notSlash = !folderPath.ends_with('/');
      archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),
          iCtx->GetStream());
    } else {
      archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                               iCtx->GetStream());
    }
  };
}

auto ExtractStatBatch(Batch &batch) {
  struct ExtractStatsMaker : ExtractStats {
    std::mutex mtx;
    LoadingBar *scanBar;

    void Push(AppContextShare *ctx, size_t numFiles) {
      std::unique_lock<std::mutex> lg(mtx);
      archiveFiles.emplace(ctx->Hash(), numFiles);
      totalFiles += numFiles;
    }

    ~ExtractStatsMaker() { RemoveLogLines(scanBar); }
  };

  batch.keepFinishLines = false;
  auto sharedData = std::make_shared<ExtractStatsMaker>();
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  sharedData->scanBar =
      AppendNewLogLine<LoadingBar>("Processing extract stats.");

  batch.forEachFile = [payload = sharedData,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    payload->Push(iCtx, ctx->ExtractStat(std::bind(
                            [&](size_t offset, size_t size) {
                              return iCtx->GetBuffer(size, offset);
                            },
                            std::placeholders::_1, std::placeholders::_2)));
  };

  return sharedData;
}

void ProcessBatch(Batch &batch, ExtractStats *stats) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  batch.forEachFile = [payload = std::make_shared<UILines>(*stats),
                       archiveFiles =
                           std::make_shared<decltype(stats->archiveFiles)>(
                               std::move(stats->archiveFiles)),
                       ctx = batch.ctx](AppContextShare *iCtx) {
    auto currentBar = payload->ChooseBar();
    if (currentBar) {
      currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));
    }

    iCtx->forEachFile = [=] {
      if (currentBar) {
        (*currentBar)++;
      }

      if (payload->totalCount) {
        (*payload->totalCount)++;
      }
    };

    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
  };
}

void ProcessBatch(Batch &batch, size_t numFiles) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  auto payload = std::make_shared<UILines>(numFiles);
  batch.forEachFile = [payload = payload,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
    if (payload->totalCount) {
      (*payload->totalCount)++;
    }
  };

  auto totalFiles = std::make_shared<size_t>(numFiles);
  batch.updateFileCount = [payload = payload,
                           totalFiles = totalFiles](size_t addedFiles) {
    *totalFiles.get() += addedFiles;
    payload->totalProgress->ItemCount(*totalFiles);
  };
}

int CreateContent(const std::string &moduleName, const std::string &appFolder,
                  const std::string &appName, APPContext &ctx) {
  try {
    ctx = APPContext(moduleName.c_str(), appFolder, appName);
  } catch (const std::exception &e) {
    printerror(e.what());
    return 2;
  }

  ConsolePrintDetail(0);

  printline(ctx.info->header);
  printline(appHeader0 << appName << ' ' << moduleName << appHeader1);

  return 0;
}

int LoadProject(const std::string &path, const std::string &appFolder,
                const std::string &appName) {
  std::ifstream str(path);
  nlohmann::json project(nlohmann::json::parse(str));

  APPContext ctx;
  std::string moduleName = project["module"];

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  ConsolePrintDetail(1);

  std::string outputDir;

  if (!project["output_dir"].is_null()) {
    std::string outputDir = project["output_dir"];
    ctx.ApplySetting("out", outputDir);
  }

  bool noConfig = !project["no_config"].is_null() && project["no_config"];

  if (!noConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  nlohmann::json settings = project["settings"];

  for (auto &s : settings.items()) {
    std::string dumped = s.value().dump();
    if (dumped.front() == '"') {
      dumped.erase(0, 1);
    }
    if (dumped.back() == '"') {
      dumped.erase(dumped.size() - 1);
    }
    ctx.ApplySetting(s.key(), dumped);
  }

  nlohmann::json inputs = project["inputs"];

  InitTempStorage();
  ctx.SetupModule();
  std::unique_ptr<AppPackContext> archiveContext;

  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);
    AFileInfo batchPath(path);
    std::string batchBase(batchPath.GetFolder());

    if (ctx.NewArchive) {
      std::string folder(batchPath.GetFolder());
      std::string archive(batchPath.GetFullPathNoExt());
      archiveContext.reset(batch.ctx->NewArchive(archive));
      MergePackModeBatch(batch, folder, archiveContext.get());
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (std::string input : inputs) {
          batch.AddFile(batchBase + input);
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += inputs.size();
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, inputs.size());
      }
    }

    if (ctx.info->batchControlFilters.size() > 0) {
      std::string pathDir(AFileInfo(path).GetFolder());
      for (nlohmann::json input : inputs) {
        if (input.is_array()) {
          batch.AddBatch(input, pathDir);
        } else {
          printwarning("Expected group, got " << input.type_name()
                                              << " instead. Skipping input.");
        }
      }
    } else {
      for (nlohmann::json input : inputs) {
        if (input.is_string()) {
          batch.AddFile(batchBase + std::string(input));
        } else {
          printwarning("Expected path string, got "
                       << input.type_name() << " instead. Skipping input.");
        }
      }
    }

    batch.FinishBatch();
  }

  if (archiveContext) {
    ConsolePrintDetail(1);
    archiveContext->Finish();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int Main(int argc, TCHAR *argv[]) {
  ConsolePrintDetail(1);
  AFileInfo appLocation(std::to_string(*argv));
  std::string appFolder(appLocation.GetFolder());
  std::string appName(appLocation.GetFilename());
  es::SetDllRunPath(appFolder + "lib");

  if (argc < 2) {
    printwarning(
        "No parameters provided, entering scan mode and generating config.");
    return !ScanModules(appFolder, appName);
  }

  auto moduleName = std::to_string(argv[1]);

  if (moduleName == "--make-doc") {
    std::string templatePath;

    if (argc < 3) {
      printwarning("Expexted template path!");
    } else {
      templatePath = std::to_string(argv[2]);
    }

    GenerateDocumentation(appFolder, appName, templatePath);
    return 0;
  } else if (moduleName.ends_with(".json")) {
    return LoadProject(moduleName, appFolder, appName);
  }

  if (argc < 3) {
    printerror("Insufficient argument count, expected parameters.");
    return 1;
  }

  APPContext ctx;

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  if (IsHelp(argv[2])) {
    ctx.PrintCLIHelp();
    return 0;
  }

  ConsolePrintDetail(1);
  bool dontLoadConfig = false;
  std::vector<bool> markedFiles(size_t(argc), false);
  size_t totalFiles = 0;

  // Handle cli options and switches
  for (int a = 2; a < argc; a++) {
    auto opt = argv[a];

    if (opt[0] == '-') {
      auto optStr = std::to_string(opt);
      std::string_view optsw(optStr);

      if (optsw != "--out") {
        // We won't use config file, reset all booleans to false,
        // so we can properly use cli switches
        [&] {
          if (dontLoadConfig) {
            return;
          }

          printinfo("CLI option detected, config won't be loaded, all booleans "
                    "set to false!");
          ctx.ResetSwitchSettings();
        }();
        dontLoadConfig = true;
      }
      optsw.remove_prefix(1);

      if (opt[0] == '-') {
        optsw.remove_prefix(1);
      }

      auto valStr = std::to_string(argv[a + 1]);

      if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {
        a++;
      }

    } else {
      markedFiles[a] = true;
      totalFiles++;
    }
  }

  if (!dontLoadConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  InitTempStorage();
  ctx.SetupModule();
  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);

    if (ctx.NewArchive) {
      PackModeBatch(batch);
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (int a = 2; a < argc; a++) {
          if (!markedFiles.at(a)) {
            continue;
          }
          batch.AddFile(std::to_string(argv[a]));
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += totalFiles;
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, totalFiles);
      }
    }

    for (int a = 2; a < argc; a++) {
      if (!markedFiles.at(a)) {
        continue;
      }
      batch.AddFile(std::to_string(argv[a]));
    }

    batch.FinishBatch();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int _tmain(int argc, TCHAR *argv[]) {
  es::SetupWinApiConsole();
  InitConsole();
  CleanTempStorages();

  int retVal = Main(argc, argv);

  CleanCurrentTempStorage();

#ifndef NDEBUG
  auto cacheStats = CacheGenerator::GlobalMetrics();
  PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
            " search misses: ", cacheStats.numSearchMisses);
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
#endif
  TerminateConsole();
  return retVal;
}

1	/* Spike is universal dedicated module handler
2	This source contains code for CLI master app
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "nlohmann/json.hpp"
20	#include "project.h"
21	#include "spike/app/batch.hpp"
22	#include "spike/app/console.hpp"
23	#include "spike/app/tmp_storage.hpp"
24	#include "spike/io/binwritter.hpp"
25	#include "spike/io/stat.hpp"
26	#include "spike/master_printer.hpp"
27	#include "spike/type/tchar.hpp"
28	#include "spike/util/pugiex.hpp"
29	#include <thread>
30
31	static const char appHeader0[] =
32	"Simply drag'n'drop files/folders onto application or "
33	"use as ";
34	static const char appHeader1[] =
35	" [options] path1 path2 ...\nTool can detect and scan folders and "
36	"uncompressed zip archives.";
37
38	struct ProcessedFiles : LoadingBar, CounterLine {
39	char buffer[128]{};
40
41	ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}	×
42	void PrintLine() override {	×
43	snprintf(buffer, sizeof(buffer), "Processed %4" PRIuMAX " files.",	×
44	curitem.load(std::memory_order_relaxed));
45	LoadingBar::PrintLine();	×
46	}
47	};
48
49	struct ExtractStats {
50	std::map<JenHash, size_t> archiveFiles;
51	size_t totalFiles = 0;
52	};
53
54	struct UILines {
55	ProgressBar *totalProgress{nullptr};
56	CounterLine *totalCount{nullptr};
57	std::map<uint32, ProgressBar *> bars;
58	std::mutex barsMutex;
59
60	auto ChooseBar() {	×
61	if (bars.empty()) {	×
62	return (ProgressBar *)(nullptr);
63	}
64
65	auto threadId = std::this_thread::get_id();
66	auto id = reinterpret_cast<const uint32 &>(threadId);	×
67	auto found = bars.find(id);
68
69	if (es::IsEnd(bars, found)) {	×
70	std::lock_guard<std::mutex> lg(barsMutex);	×
71	auto retVal = bars.begin()->second;	×
72	bars.emplace(id, retVal);	×
73	bars.erase(bars.begin());
74
75	return retVal;
76	}
77
78	return found->second;	×
79	};
80
81	UILines(const ExtractStats &stats) {	×
82	ModifyElements([&](ElementAPI &api) {	×
83	const size_t minThreads =
84	std::min(size_t(std::thread::hardware_concurrency()),	×
85	stats.archiveFiles.size());	×
86
87	if (minThreads < 2) {	×
88	return;
89	}
90
91	for (size_t t = 0; t < minThreads; t++) {	×
92	auto progBar = std::make_unique<ProgressBar>("Thread:");	×
93	auto progBarRaw = progBar.get();	×
94	bars.emplace(t, progBarRaw);	×
95	api.Append(std::move(progBar));	×
96	}
97	});
98
99	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
100	prog->ItemCount(stats.totalFiles);	×
101	totalCount = prog;	×
102	}
103
104	UILines(size_t totalInputFiles) {	×
105	totalCount = AppendNewLogLine<ProcessedFiles>();	×
106	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
107	prog->ItemCount(totalInputFiles);	×
108	totalProgress = prog;	×
109	}
110
111	~UILines() {	×
112	ModifyElements([&](ElementAPI &api) {	×
113	if (totalCount) {	×
114	// Wait a little bit for internal queues to finish printing
115	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
116	if (totalProgress) {	×
117	auto data = static_cast<ProcessedFiles *>(totalCount);	×
118	data->Finish();
119	api.Release(data);	×
120	} else {
121	auto data = static_cast<DetailedProgressBar *>(totalCount);	×
122	api.Remove(data);	×
123	}
124	}
125	api.Clean();	×
126	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
127	});	×
128	}
129	};
130
131	bool ScanModules(const std::string &appFolder, const std::string &appName) {	×
132	DirectoryScanner sc;	×
133	sc.AddFilter(std::string_view(".spk$"));	×
134	sc.Scan(appFolder);	×
135	bool isOkay = true;
136
137	for (auto &m : sc) {	×
138	try {
139	AFileInfo modulePath(m);	×
140	auto moduleName = modulePath.GetFilename();	×
141	const size_t firstDotPos = moduleName.find_first_of('.');
142	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
143	APPContext ctx(moduleNameStr.data(), appFolder, appName);	×
144	ctx.FromConfig();	×
145	} catch (const std::runtime_error &e) {	×
146	printerror(e.what());	×
147	isOkay = false;
148	}	×
149	}
150
151	return isOkay;	×
152	}
153
154	void GenerateDocumentation(const std::string &appFolder,	×
155	const std::string &appName,
156	const std::string &templatePath) {
157	DirectoryScanner sc;	×
158	sc.AddFilter(std::string_view(".spk$"));	×
159	sc.Scan(appFolder);	×
160	std::set<std::string> modules;
161	pugi::xml_document doc;	×
162
163	if (!templatePath.empty()) {	×
164	doc = XMLFromFile(templatePath);	×
165	}
166
167	for (auto &m : sc) {	×
168	try {
169	AFileInfo modulePath(m);	×
170	auto moduleName = modulePath.GetFilename();	×
171	const size_t firstDotPos = moduleName.find_first_of('.');
172	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
173	modules.emplace(moduleNameStr);
174	} catch (const std::runtime_error &e) {	×
175	printerror(e.what());	×
176	}	×
177	}
178
179	BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");	×
180	const char *toolsetName = "[[TOOLSET NAME]]";
181
182	if (auto child = doc.child("toolset_name"); child) {	×
183	toolsetName = child.text().as_string();	×
184	}
185
186	const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";
187
188	if (auto child = doc.child("toolset_description"); child) {	×
189	toolsetDescription = child.text().as_string();	×
190	}
191
192	wr.BaseStream() << "# " << toolsetName << "\n\n"
193	<< toolsetDescription << "\n\n";	×
194
195	for (auto &m : modules) {	×
196	pugi::xml_node node = doc.child(m.data());	×
197	APPContext ctx(m.data(), appFolder, appName);	×
198	ctx.GetMarkdownDoc(wr.BaseStream(), node);	×
199	}
200
201	if (auto child = doc.child("toolset_footer"); child) {	×
202	wr.BaseStream() << child.text().as_string();	×
203	}
204	}
205
206	void PackModeBatch(Batch &batch) {	×
207	struct PackData {
208	size_t index = 0;
209	std::unique_ptr<AppPackContext> archiveContext;
210	std::string pbarLabel;
211	DetailedProgressBar *progBar = nullptr;
212	std::string folderPath;
213	};
214
215	auto payload = std::make_shared<PackData>();
216
217	batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,	×
218	size_t numFiles) {	×
219	payload->folderPath = path;	×
220	payload->archiveContext.reset(ctx->NewArchive(path));
221	payload->pbarLabel = "Folder id " + std::to_string(payload->index++);	×
222	payload->progBar =	×
223	AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);	×
224	payload->progBar->ItemCount(numFiles);	×
225	uint8 consoleDetail = 1 \| uint8(ctx->info->multithreaded) << 1;	×
226	ConsolePrintDetail(consoleDetail);	×
227	printline("Processing: " << path);	×
228	};
229
230	batch.forEachFile = [payload](AppContextShare *iCtx) {	×
231	if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {	×
232	int notSlash = !payload->folderPath.ends_with('/');	×
233	payload->archiveContext->SendFile(	×
234	iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +	×
235	notSlash),	×
236	iCtx->GetStream());	×
237	} else {
238	payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
239	iCtx->GetStream());	×
240	}
241	(*payload->progBar)++;	×
242	};
243
244	batch.forEachFolderFinish = [payload] {	×
245	ConsolePrintDetail(1);	×
246	payload->archiveContext->Finish();	×
247	payload->archiveContext.reset();
248	RemoveLogLines(payload->progBar);	×
249	};
250	}
251
252	void MergePackModeBatch(Batch &batch, const std::string &folderPath,	×
253	AppPackContext *archiveContext) {
254	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
255	ConsolePrintDetail(consoleDetail);	×
256	printline("Processing: " << folderPath);	×
257
258	batch.forEachFile = [=](AppContextShare *iCtx) {	×
259	if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {	×
260	int notSlash = !folderPath.ends_with('/');	×
261	archiveContext->SendFile(	×
262	iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),	×
263	iCtx->GetStream());	×
264	} else {
265	archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
266	iCtx->GetStream());	×
267	}
268	};
269	}
270
271	auto ExtractStatBatch(Batch &batch) {	×
272	struct ExtractStatsMaker : ExtractStats {
273	std::mutex mtx;
274	LoadingBar *scanBar;
275
276	void Push(AppContextShare *ctx, size_t numFiles) {	×
277	std::unique_lock<std::mutex> lg(mtx);	×
278	archiveFiles.emplace(ctx->Hash(), numFiles);	×
279	totalFiles += numFiles;	×
280	}
281
282	~ExtractStatsMaker() { RemoveLogLines(scanBar); }	×
283	};
284
285	batch.keepFinishLines = false;	×
286	auto sharedData = std::make_shared<ExtractStatsMaker>();
287	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
288	ConsolePrintDetail(consoleDetail);	×
289	sharedData->scanBar =	×
290	AppendNewLogLine<LoadingBar>("Processing extract stats.");	×
291
292	batch.forEachFile = [payload = sharedData,	×
293	ctx = batch.ctx](AppContextShare *iCtx) {	×
294	payload->Push(iCtx, ctx->ExtractStat(std::bind(	×
295	[&](size_t offset, size_t size) {
296	return iCtx->GetBuffer(size, offset);	×
297	},
298	std::placeholders::_1, std::placeholders::_2)));
299	};
300
301	return sharedData;	×
302	}
303
304	void ProcessBatch(Batch &batch, ExtractStats *stats) {	×
305	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
306	ConsolePrintDetail(consoleDetail);	×
307	batch.forEachFile = [payload = std::make_shared<UILines>(*stats),	×
308	archiveFiles =
309	std::make_shared<decltype(stats->archiveFiles)>(
310	std::move(stats->archiveFiles)),	×
311	ctx = batch.ctx](AppContextShare *iCtx) {	×
312	auto currentBar = payload->ChooseBar();	×
313	if (currentBar) {	×
314	currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));	×
315	}
316
317	iCtx->forEachFile = [=] {	×
318	if (currentBar) {	×
319	(*currentBar)++;
320	}
321
322	if (payload->totalCount) {	×
323	(*payload->totalCount)++;
324	}
325	};
326
327	printline("Processing: " << iCtx->FullPath());	×
328	ctx->ProcessFile(iCtx);
329	if (payload->totalProgress) {	×
330	(*payload->totalProgress)++;
331	}
332	};
333	}
334
335	void ProcessBatch(Batch &batch, size_t numFiles) {	×
336	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
337	ConsolePrintDetail(consoleDetail);	×
338	auto payload = std::make_shared<UILines>(numFiles);
339	batch.forEachFile = [payload = payload,	×
340	ctx = batch.ctx](AppContextShare *iCtx) {	×
341	printline("Processing: " << iCtx->FullPath());	×
342	ctx->ProcessFile(iCtx);
343	if (payload->totalProgress) {	×
344	(*payload->totalProgress)++;
345	}
346	if (payload->totalCount) {	×
347	(*payload->totalCount)++;
348	}
349	};
350
351	auto totalFiles = std::make_shared<size_t>(numFiles);	×
352	batch.updateFileCount = [payload = payload,	×
353	totalFiles = totalFiles](size_t addedFiles) {	×
354	*totalFiles.get() += addedFiles;	×
355	payload->totalProgress->ItemCount(*totalFiles);	×
356	};
357	}
358
359	int CreateContent(const std::string &moduleName, const std::string &appFolder,	×
360	const std::string &appName, APPContext &ctx) {
361	try {
362	ctx = APPContext(moduleName.c_str(), appFolder, appName);	×
363	} catch (const std::exception &e) {	×
364	printerror(e.what());	×
365	return 2;
366	}	×
367
368	ConsolePrintDetail(0);	×
369
370	printline(ctx.info->header);	×
371	printline(appHeader0 << appName << ' ' << moduleName << appHeader1);	×
372
373	return 0;	×
374	}
375
376	int LoadProject(const std::string &path, const std::string &appFolder,	×
377	const std::string &appName) {
378	std::ifstream str(path);	×
379	nlohmann::json project(nlohmann::json::parse(str));	×
380
381	APPContext ctx;
382	std::string moduleName = project["module"];	×
383
384	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
385	return ret;
386	}
387
388	ConsolePrintDetail(1);	×
389
390	std::string outputDir;
391
392	if (!project["output_dir"].is_null()) {	×
393	std::string outputDir = project["output_dir"];	×
394	ctx.ApplySetting("out", outputDir);	×
395	}
396
397	bool noConfig = !project["no_config"].is_null() && project["no_config"];	×
398
399	if (!noConfig) {
400	printinfo("Loading config: " << appName << ".config");	×
401	ctx.FromConfig();	×
402	}
403
404	nlohmann::json settings = project["settings"];	×
405
406	for (auto &s : settings.items()) {	×
407	std::string dumped = s.value().dump();	×
408	if (dumped.front() == '"') {	×
409	dumped.erase(0, 1);	×
410	}
411	if (dumped.back() == '"') {	×
412	dumped.erase(dumped.size() - 1);	×
413	}
414	ctx.ApplySetting(s.key(), dumped);	×
415	}
416
417	nlohmann::json inputs = project["inputs"];	×
418
419	InitTempStorage();	×
420	ctx.SetupModule();	×
421	std::unique_ptr<AppPackContext> archiveContext;
422
423	{
424	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
425	AFileInfo batchPath(path);	×
426	std::string batchBase(batchPath.GetFolder());	×
427
428	if (ctx.NewArchive) {	×
429	std::string folder(batchPath.GetFolder());	×
430	std::string archive(batchPath.GetFullPathNoExt());	×
431	archiveContext.reset(batch.ctx->NewArchive(archive));	×
432	MergePackModeBatch(batch, folder, archiveContext.get());	×
433	} else {
434	if (ctx.ExtractStat) {	×
435	auto stats = ExtractStatBatch(batch);	×
436	for (std::string input : inputs) {	×
437	batch.AddFile(batchBase + input);	×
438	}
439
440	batch.FinishBatch();	×
441	batch.Clean();	×
442	stats.get()->totalFiles += inputs.size();	×
443	ProcessBatch(batch, stats.get());	×
444	} else {
445	ProcessBatch(batch, inputs.size());	×
446	}
447	}
448
449	if (ctx.info->batchControlFilters.size() > 0) {	×
450	std::string pathDir(AFileInfo(path).GetFolder());	×
451	for (nlohmann::json input : inputs) {	×
452	if (input.is_array()) {	×
453	batch.AddBatch(input, pathDir);	×
454	} else {
455	printwarning("Expected group, got " << input.type_name()	×
456	<< " instead. Skipping input.");
457	}
458	}
459	} else {
460	for (nlohmann::json input : inputs) {	×
461	if (input.is_string()) {	×
462	batch.AddFile(batchBase + std::string(input));	×
463	} else {
464	printwarning("Expected path string, got "	×
465	<< input.type_name() << " instead. Skipping input.");
466	}
467	}
468	}
469
470	batch.FinishBatch();	×
471	}
472
473	if (archiveContext) {	×
474	ConsolePrintDetail(1);	×
475	archiveContext->Finish();	×
476	}
477
478	if (ctx.FinishContext) {	×
479	ctx.FinishContext();
480	}
481
482	return 0;
483	}
484
485	int Main(int argc, TCHAR *argv[]) {	×
486	ConsolePrintDetail(1);	×
487	AFileInfo appLocation(std::to_string(*argv));	×
488	std::string appFolder(appLocation.GetFolder());	×
489	std::string appName(appLocation.GetFilename());	×
490	es::SetDllRunPath(appFolder + "lib");	×
491
492	if (argc < 2) {	×
493	printwarning(	×
494	"No parameters provided, entering scan mode and generating config.");
495	return !ScanModules(appFolder, appName);	×
496	}
497
498	auto moduleName = std::to_string(argv[1]);	×
499
500	if (moduleName == "--make-doc") {	×
501	std::string templatePath;
502
503	if (argc < 3) {	×
504	printwarning("Expexted template path!");	×
505	} else {
506	templatePath = std::to_string(argv[2]);	×
507	}
508
509	GenerateDocumentation(appFolder, appName, templatePath);	×
510	return 0;
511	} else if (moduleName.ends_with(".json")) {	×
512	return LoadProject(moduleName, appFolder, appName);	×
513	}
514
515	if (argc < 3) {	×
516	printerror("Insufficient argument count, expected parameters.");	×
517	return 1;
518	}
519
520	APPContext ctx;
521
522	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
523	return ret;
524	}
525
526	if (IsHelp(argv[2])) {	×
527	ctx.PrintCLIHelp();	×
528	return 0;
529	}
530
531	ConsolePrintDetail(1);	×
532	bool dontLoadConfig = false;	×
533	std::vector<bool> markedFiles(size_t(argc), false);	×
534	size_t totalFiles = 0;
535
536	// Handle cli options and switches
537	for (int a = 2; a < argc; a++) {	×
538	auto opt = argv[a];	×
539
540	if (opt[0] == '-') {	×
541	auto optStr = std::to_string(opt);
542	std::string_view optsw(optStr);
543
544	if (optsw != "--out") {	×
545	// We won't use config file, reset all booleans to false,
546	// so we can properly use cli switches
547	[&] {	×
548	if (dontLoadConfig) {	×
549	return;
550	}
551
552	printinfo("CLI option detected, config won't be loaded, all booleans "	×
553	"set to false!");
554	ctx.ResetSwitchSettings();	×
555	}();	×
556	dontLoadConfig = true;	×
557	}
558	optsw.remove_prefix(1);
559
560	if (opt[0] == '-') {	×
561	optsw.remove_prefix(1);
562	}
563
564	auto valStr = std::to_string(argv[a + 1]);	×
565
566	if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {	×
567	a++;	×
568	}
569
570	} else {
571	markedFiles[a] = true;
572	totalFiles++;	×
573	}
574	}
575
576	if (!dontLoadConfig) {	×
577	printinfo("Loading config: " << appName << ".config");	×
578	ctx.FromConfig();	×
579	}
580
581	InitTempStorage();	×
582	ctx.SetupModule();	×
583	{
584	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
585
586	if (ctx.NewArchive) {	×
587	PackModeBatch(batch);	×
588	} else {
589	if (ctx.ExtractStat) {	×
590	auto stats = ExtractStatBatch(batch);	×
591	for (int a = 2; a < argc; a++) {	×
592	if (!markedFiles.at(a)) {	×
593	continue;	×
594	}
595	batch.AddFile(std::to_string(argv[a]));	×
596	}
597
598	batch.FinishBatch();	×
599	batch.Clean();	×
600	stats.get()->totalFiles += totalFiles;	×
601	ProcessBatch(batch, stats.get());	×
602	} else {
603	ProcessBatch(batch, totalFiles);	×
604	}
605	}
606
607	for (int a = 2; a < argc; a++) {	×
608	if (!markedFiles.at(a)) {	×
609	continue;	×
610	}
611	batch.AddFile(std::to_string(argv[a]));	×
612	}
613
614	batch.FinishBatch();	×
615	}
616
617	if (ctx.FinishContext) {	×
618	ctx.FinishContext();
619	}
620
621	return 0;
622	}
623
624	int _tmain(int argc, TCHAR *argv[]) {	×
625	es::SetupWinApiConsole();
626	InitConsole();	×
627	CleanTempStorages();	×
628
629	int retVal = Main(argc, argv);	×
630
631	CleanCurrentTempStorage();	×
632
633	#ifndef NDEBUG
634	auto cacheStats = CacheGenerator::GlobalMetrics();
635	PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
636	" search misses: ", cacheStats.numSearchMisses);
637	std::this_thread::sleep_for(std::chrono::milliseconds(100));
638	#endif
639	TerminateConsole();	×
640	return retVal;
641	}

PredatorCZ / PreCore / 505

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous