PredatorCZ / PreCore / 514

Committed 17 Apr 2024 06:12PM UTC coverage: 54.165% (-0.04%) from 54.2%

Build # 514

Build Type

push

github

Committed by

PredatorCZ

Commit Message

update doc generator

Run Details

0 of 11 new or added lines in 2 files covered. (0.0%)

2 existing lines in 1 file now uncovered.

4142 of 7647 relevant lines covered (54.17%)

8767.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/cli/spike.cpp

/*  Spike is universal dedicated module handler
    This source contains code for CLI master app

    Copyright 2021-2023 Lukas Cone

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
*/

#include "nlohmann/json.hpp"
#include "project.h"
#include "spike/app/batch.hpp"
#include "spike/app/console.hpp"
#include "spike/app/tmp_storage.hpp"
#include "spike/io/binwritter.hpp"
#include "spike/io/stat.hpp"
#include "spike/master_printer.hpp"
#include "spike/type/tchar.hpp"
#include "spike/util/pugiex.hpp"
#include <thread>

static const char appHeader0[] =
    "Simply drag'n'drop files/folders onto application or "
    "use as ";
static const char appHeader1[] =
    " [options] path1 path2 ...\nTool can detect and scan folders and "
    "uncompressed zip archives.";

struct ProcessedFiles : LoadingBar, CounterLine {
  char buffer[128]{};

  ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}
  void PrintLine() override {
    snprintf(buffer, sizeof(buffer), "Processed %4" PRIuMAX " files.",
             curitem.load(std::memory_order_relaxed));
    LoadingBar::PrintLine();
  }
};

struct ExtractStats {
  std::map<JenHash, size_t> archiveFiles;
  size_t totalFiles = 0;
};

struct UILines {
  ProgressBar *totalProgress{nullptr};
  CounterLine *totalCount{nullptr};
  std::map<uint32, ProgressBar *> bars;
  std::mutex barsMutex;

  auto ChooseBar() {
    if (bars.empty()) {
      return (ProgressBar *)(nullptr);
    }

    auto threadId = std::this_thread::get_id();
    auto id = reinterpret_cast<const uint32 &>(threadId);
    auto found = bars.find(id);

    if (es::IsEnd(bars, found)) {
      std::lock_guard<std::mutex> lg(barsMutex);
      auto retVal = bars.begin()->second;
      bars.emplace(id, retVal);
      bars.erase(bars.begin());

      return retVal;
    }

    return found->second;
  };

  UILines(const ExtractStats &stats) {
    ModifyElements([&](ElementAPI &api) {
      const size_t minThreads =
          std::min(size_t(std::thread::hardware_concurrency()),
                   stats.archiveFiles.size());

      if (minThreads < 2) {
        return;
      }

      for (size_t t = 0; t < minThreads; t++) {
        auto progBar = std::make_unique<ProgressBar>("Thread:");
        auto progBarRaw = progBar.get();
        bars.emplace(t, progBarRaw);
        api.Append(std::move(progBar));
      }
    });

    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(stats.totalFiles);
    totalCount = prog;
  }

  UILines(size_t totalInputFiles) {
    totalCount = AppendNewLogLine<ProcessedFiles>();
    auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");
    prog->ItemCount(totalInputFiles);
    totalProgress = prog;
  }

  ~UILines() {
    ModifyElements([&](ElementAPI &api) {
      if (totalCount) {
        // Wait a little bit for internal queues to finish printing
        std::this_thread::sleep_for(std::chrono::milliseconds(100));
        if (totalProgress) {
          auto data = static_cast<ProcessedFiles *>(totalCount);
          data->Finish();
          api.Release(data);
        } else {
          auto data = static_cast<DetailedProgressBar *>(totalCount);
          api.Remove(data);
        }
      }
      api.Clean();
      std::this_thread::sleep_for(std::chrono::milliseconds(100));
    });
  }
};

bool ScanModules(const std::string &appFolder, const std::string &appName) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  bool isOkay = true;

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      APPContext ctx(moduleNameStr.data(), appFolder, appName);
      ctx.FromConfig();
    } catch (const std::runtime_error &e) {
      printerror(e.what());
      isOkay = false;
    }
  }

  return isOkay;
}

void GenerateDocumentation(const std::string &appFolder,
                           const std::string &appName,
                           const std::string &templatePath) {
  DirectoryScanner sc;
  sc.AddFilter(std::string_view(".spk$"));
  sc.Scan(appFolder);
  std::set<std::string> modules;
  pugi::xml_document doc;

  if (!templatePath.empty()) {
    doc = XMLFromFile(templatePath);
  }

  for (auto &m : sc) {
    try {
      AFileInfo modulePath(m);
      auto moduleName = modulePath.GetFilename();
      const size_t firstDotPos = moduleName.find_first_of('.');
      std::string moduleNameStr(moduleName.substr(0, firstDotPos));
      modules.emplace(moduleNameStr);
    } catch (const std::runtime_error &e) {
      printerror(e.what());
    }
  }

  BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");

  const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";

  if (auto child = doc.child("toolset_description"); child) {
    toolsetDescription = child.text().as_string();
  }

  wr.BaseStream() << toolsetDescription << "<h2>Module list</h2>\n<ul>\n";
  std::stringstream str;

  for (auto &m : modules) {
    pugi::xml_node node = doc.child(m.data());
    APPContext ctx(m.data(), appFolder, appName);
    ctx.GetMarkdownDoc(str, node);
    std::string className = ctx.GetClassName(node);
    std::string classNameLink = className;
    std::replace_if(
        classNameLink.begin(), classNameLink.end(),
        [](char c) { return c == ' '; }, '-');

    wr.BaseStream() << "<li><a href=\"#" << classNameLink << "\">" << className
                    << "</a></li>\n";
  }

  wr.BaseStream() << "</ul>\n\n" << str.str() << "\n\n";

  if (auto child = doc.child("toolset_footer"); child) {
    wr.BaseStream() << child.text().as_string();
  }
}

void PackModeBatch(Batch &batch) {
  struct PackData {
    size_t index = 0;
    std::unique_ptr<AppPackContext> archiveContext;
    std::string pbarLabel;
    DetailedProgressBar *progBar = nullptr;
    std::string folderPath;
  };

  auto payload = std::make_shared<PackData>();

  batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,
                                                   size_t numFiles) {
    payload->folderPath = path;
    payload->archiveContext.reset(ctx->NewArchive(path));
    payload->pbarLabel = "Folder id " + std::to_string(payload->index++);
    payload->progBar =
        AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);
    payload->progBar->ItemCount(numFiles);
    uint8 consoleDetail = 1 | uint8(ctx->info->multithreaded) << 1;
    ConsolePrintDetail(consoleDetail);
    printline("Processing: " << path);
  };

  batch.forEachFile = [payload](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {
      int notSlash = !payload->folderPath.ends_with('/');
      payload->archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +
                                                 notSlash),
          iCtx->GetStream());
    } else {
      payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                                        iCtx->GetStream());
    }
    (*payload->progBar)++;
  };

  batch.forEachFolderFinish = [payload] {
    ConsolePrintDetail(1);
    payload->archiveContext->Finish();
    payload->archiveContext.reset();
    RemoveLogLines(payload->progBar);
  };
}

void MergePackModeBatch(Batch &batch, const std::string &folderPath,
                        AppPackContext *archiveContext) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  printline("Processing: " << folderPath);

  batch.forEachFile = [=](AppContextShare *iCtx) {
    if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {
      int notSlash = !folderPath.ends_with('/');
      archiveContext->SendFile(
          iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),
          iCtx->GetStream());
    } else {
      archiveContext->SendFile(iCtx->workingFile.GetFullPath(),
                               iCtx->GetStream());
    }
  };
}

auto ExtractStatBatch(Batch &batch) {
  struct ExtractStatsMaker : ExtractStats {
    std::mutex mtx;
    LoadingBar *scanBar;

    void Push(AppContextShare *ctx, size_t numFiles) {
      std::unique_lock<std::mutex> lg(mtx);
      archiveFiles.emplace(ctx->Hash(), numFiles);
      totalFiles += numFiles;
    }

    ~ExtractStatsMaker() { RemoveLogLines(scanBar); }
  };

  batch.keepFinishLines = false;
  auto sharedData = std::make_shared<ExtractStatsMaker>();
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  sharedData->scanBar =
      AppendNewLogLine<LoadingBar>("Processing extract stats.");

  batch.forEachFile = [payload = sharedData,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    payload->Push(iCtx, ctx->ExtractStat(std::bind(
                            [&](size_t offset, size_t size) {
                              return iCtx->GetBuffer(size, offset);
                            },
                            std::placeholders::_1, std::placeholders::_2)));
  };

  return sharedData;
}

void ProcessBatch(Batch &batch, ExtractStats *stats) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  batch.forEachFile = [payload = std::make_shared<UILines>(*stats),
                       archiveFiles =
                           std::make_shared<decltype(stats->archiveFiles)>(
                               std::move(stats->archiveFiles)),
                       ctx = batch.ctx](AppContextShare *iCtx) {
    auto currentBar = payload->ChooseBar();
    if (currentBar) {
      currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));
    }

    iCtx->forEachFile = [=] {
      if (currentBar) {
        (*currentBar)++;
      }

      if (payload->totalCount) {
        (*payload->totalCount)++;
      }
    };

    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
  };
}

void ProcessBatch(Batch &batch, size_t numFiles) {
  uint8 consoleDetail = 1 | uint8(batch.ctx->info->multithreaded) << 1;
  ConsolePrintDetail(consoleDetail);
  auto payload = std::make_shared<UILines>(numFiles);
  batch.forEachFile = [payload = payload,
                       ctx = batch.ctx](AppContextShare *iCtx) {
    printline("Processing: " << iCtx->FullPath());
    ctx->ProcessFile(iCtx);
    if (payload->totalProgress) {
      (*payload->totalProgress)++;
    }
    if (payload->totalCount) {
      (*payload->totalCount)++;
    }
  };

  auto totalFiles = std::make_shared<size_t>(numFiles);
  batch.updateFileCount = [payload = payload,
                           totalFiles = totalFiles](size_t addedFiles) {
    *totalFiles.get() += addedFiles;
    payload->totalProgress->ItemCount(*totalFiles);
  };
}

int CreateContent(const std::string &moduleName, const std::string &appFolder,
                  const std::string &appName, APPContext &ctx) {
  try {
    ctx = APPContext(moduleName.c_str(), appFolder, appName);
  } catch (const std::exception &e) {
    printerror(e.what());
    return 2;
  }

  ConsolePrintDetail(0);

  printline(ctx.info->header);
  printline(appHeader0 << appName << ' ' << moduleName << appHeader1);

  return 0;
}

int LoadProject(const std::string &path, const std::string &appFolder,
                const std::string &appName) {
  std::ifstream str(path);
  nlohmann::json project(nlohmann::json::parse(str));

  APPContext ctx;
  std::string moduleName = project["module"];

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  ConsolePrintDetail(1);

  std::string outputDir;

  if (!project["output_dir"].is_null()) {
    std::string outputDir = project["output_dir"];
    ctx.ApplySetting("out", outputDir);
  }

  bool noConfig = !project["no_config"].is_null() && project["no_config"];

  if (!noConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  nlohmann::json settings = project["settings"];

  for (auto &s : settings.items()) {
    std::string dumped = s.value().dump();
    if (dumped.front() == '"') {
      dumped.erase(0, 1);
    }
    if (dumped.back() == '"') {
      dumped.erase(dumped.size() - 1);
    }
    ctx.ApplySetting(s.key(), dumped);
  }

  nlohmann::json inputs = project["inputs"];

  InitTempStorage();
  ctx.SetupModule();
  std::unique_ptr<AppPackContext> archiveContext;

  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);
    AFileInfo batchPath(path);
    std::string batchBase(batchPath.GetFolder());

    if (ctx.NewArchive) {
      std::string folder(batchPath.GetFolder());
      std::string archive(batchPath.GetFullPathNoExt());
      archiveContext.reset(batch.ctx->NewArchive(archive));
      MergePackModeBatch(batch, folder, archiveContext.get());
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (std::string input : inputs) {
          batch.AddFile(batchBase + input);
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += inputs.size();
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, inputs.size());
      }
    }

    if (ctx.info->batchControlFilters.size() > 0) {
      std::string pathDir(AFileInfo(path).GetFolder());
      for (nlohmann::json input : inputs) {
        if (input.is_array()) {
          batch.AddBatch(input, pathDir);
        } else {
          printwarning("Expected group, got " << input.type_name()
                                              << " instead. Skipping input.");
        }
      }
    } else {
      for (nlohmann::json input : inputs) {
        if (input.is_string()) {
          batch.AddFile(batchBase + std::string(input));
        } else {
          printwarning("Expected path string, got "
                       << input.type_name() << " instead. Skipping input.");
        }
      }
    }

    batch.FinishBatch();
  }

  if (archiveContext) {
    ConsolePrintDetail(1);
    archiveContext->Finish();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int Main(int argc, TCHAR *argv[]) {
  ConsolePrintDetail(1);
  AFileInfo appLocation(std::to_string(*argv));
  std::string appFolder(appLocation.GetFolder());
  std::string appName(appLocation.GetFilename());
  es::SetDllRunPath(appFolder + "lib");

  if (argc < 2) {
    printwarning(
        "No parameters provided, entering scan mode and generating config.");
    return !ScanModules(appFolder, appName);
  }

  auto moduleName = std::to_string(argv[1]);

  if (moduleName == "--make-doc") {
    std::string templatePath;

    if (argc < 3) {
      printwarning("Expexted template path!");
    } else {
      templatePath = std::to_string(argv[2]);
    }

    GenerateDocumentation(appFolder, appName, templatePath);
    return 0;
  } else if (moduleName.ends_with(".json")) {
    return LoadProject(moduleName, appFolder, appName);
  }

  if (argc < 3) {
    printerror("Insufficient argument count, expected parameters.");
    return 1;
  }

  APPContext ctx;

  if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {
    return ret;
  }

  if (IsHelp(argv[2])) {
    ctx.PrintCLIHelp();
    return 0;
  }

  ConsolePrintDetail(1);
  bool dontLoadConfig = false;
  std::vector<bool> markedFiles(size_t(argc), false);
  size_t totalFiles = 0;

  // Handle cli options and switches
  for (int a = 2; a < argc; a++) {
    auto opt = argv[a];

    if (opt[0] == '-') {
      auto optStr = std::to_string(opt);
      std::string_view optsw(optStr);

      if (optsw != "--out") {
        // We won't use config file, reset all booleans to false,
        // so we can properly use cli switches
        [&] {
          if (dontLoadConfig) {
            return;
          }

          printinfo("CLI option detected, config won't be loaded, all booleans "
                    "set to false!");
          ctx.ResetSwitchSettings();
        }();
        dontLoadConfig = true;
      }
      optsw.remove_prefix(1);

      if (opt[0] == '-') {
        optsw.remove_prefix(1);
      }

      auto valStr = std::to_string(argv[a + 1]);

      if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {
        a++;
      }

    } else {
      markedFiles[a] = true;
      totalFiles++;
    }
  }

  if (!dontLoadConfig) {
    printinfo("Loading config: " << appName << ".config");
    ctx.FromConfig();
  }

  InitTempStorage();
  ctx.SetupModule();
  {
    Batch batch(&ctx, ctx.info->multithreaded * 50);

    if (ctx.NewArchive) {
      PackModeBatch(batch);
    } else {
      if (ctx.ExtractStat) {
        auto stats = ExtractStatBatch(batch);
        for (int a = 2; a < argc; a++) {
          if (!markedFiles.at(a)) {
            continue;
          }
          batch.AddFile(std::to_string(argv[a]));
        }

        batch.FinishBatch();
        batch.Clean();
        stats.get()->totalFiles += totalFiles;
        ProcessBatch(batch, stats.get());
      } else {
        ProcessBatch(batch, totalFiles);
      }
    }

    for (int a = 2; a < argc; a++) {
      if (!markedFiles.at(a)) {
        continue;
      }
      batch.AddFile(std::to_string(argv[a]));
    }

    batch.FinishBatch();
  }

  if (ctx.FinishContext) {
    ctx.FinishContext();
  }

  return 0;
}

int _tmain(int argc, TCHAR *argv[]) {
  es::SetupWinApiConsole();
  InitConsole();
  CleanTempStorages();

  int retVal = Main(argc, argv);

  CleanCurrentTempStorage();

#ifndef NDEBUG
  auto cacheStats = CacheGenerator::GlobalMetrics();
  PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
            " search misses: ", cacheStats.numSearchMisses);
  std::this_thread::sleep_for(std::chrono::milliseconds(100));
#endif
  TerminateConsole();
  return retVal;
}

1	/* Spike is universal dedicated module handler
2	This source contains code for CLI master app
3
4	Copyright 2021-2023 Lukas Cone
5
6	Licensed under the Apache License, Version 2.0 (the "License");
7	you may not use this file except in compliance with the License.
8	You may obtain a copy of the License at
9
10	http://www.apache.org/licenses/LICENSE-2.0
11
12	Unless required by applicable law or agreed to in writing, software
13	distributed under the License is distributed on an "AS IS" BASIS,
14	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15	See the License for the specific language governing permissions and
16	limitations under the License.
17	*/
18
19	#include "nlohmann/json.hpp"
20	#include "project.h"
21	#include "spike/app/batch.hpp"
22	#include "spike/app/console.hpp"
23	#include "spike/app/tmp_storage.hpp"
24	#include "spike/io/binwritter.hpp"
25	#include "spike/io/stat.hpp"
26	#include "spike/master_printer.hpp"
27	#include "spike/type/tchar.hpp"
28	#include "spike/util/pugiex.hpp"
29	#include <thread>
30
31	static const char appHeader0[] =
32	"Simply drag'n'drop files/folders onto application or "
33	"use as ";
34	static const char appHeader1[] =
35	" [options] path1 path2 ...\nTool can detect and scan folders and "
36	"uncompressed zip archives.";
37
38	struct ProcessedFiles : LoadingBar, CounterLine {
39	char buffer[128]{};
40
41	ProcessedFiles() : LoadingBar({buffer, sizeof(buffer)}) {}	×
42	void PrintLine() override {	×
43	snprintf(buffer, sizeof(buffer), "Processed %4" PRIuMAX " files.",	×
44	curitem.load(std::memory_order_relaxed));
45	LoadingBar::PrintLine();	×
46	}
47	};
48
49	struct ExtractStats {
50	std::map<JenHash, size_t> archiveFiles;
51	size_t totalFiles = 0;
52	};
53
54	struct UILines {
55	ProgressBar *totalProgress{nullptr};
56	CounterLine *totalCount{nullptr};
57	std::map<uint32, ProgressBar *> bars;
58	std::mutex barsMutex;
59
60	auto ChooseBar() {	×
61	if (bars.empty()) {	×
62	return (ProgressBar *)(nullptr);
63	}
64
65	auto threadId = std::this_thread::get_id();
66	auto id = reinterpret_cast<const uint32 &>(threadId);	×
67	auto found = bars.find(id);
68
69	if (es::IsEnd(bars, found)) {	×
70	std::lock_guard<std::mutex> lg(barsMutex);	×
71	auto retVal = bars.begin()->second;	×
72	bars.emplace(id, retVal);	×
73	bars.erase(bars.begin());
74
75	return retVal;
76	}
77
78	return found->second;	×
79	};
80
81	UILines(const ExtractStats &stats) {	×
82	ModifyElements([&](ElementAPI &api) {	×
83	const size_t minThreads =
84	std::min(size_t(std::thread::hardware_concurrency()),	×
85	stats.archiveFiles.size());	×
86
87	if (minThreads < 2) {	×
88	return;
89	}
90
91	for (size_t t = 0; t < minThreads; t++) {	×
92	auto progBar = std::make_unique<ProgressBar>("Thread:");	×
93	auto progBarRaw = progBar.get();	×
94	bars.emplace(t, progBarRaw);	×
95	api.Append(std::move(progBar));	×
96	}
97	});
98
99	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
100	prog->ItemCount(stats.totalFiles);	×
101	totalCount = prog;	×
102	}
103
104	UILines(size_t totalInputFiles) {	×
105	totalCount = AppendNewLogLine<ProcessedFiles>();	×
106	auto prog = AppendNewLogLine<DetailedProgressBar>("Total: ");	×
107	prog->ItemCount(totalInputFiles);	×
108	totalProgress = prog;	×
109	}
110
111	~UILines() {	×
112	ModifyElements([&](ElementAPI &api) {	×
113	if (totalCount) {	×
114	// Wait a little bit for internal queues to finish printing
115	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
116	if (totalProgress) {	×
117	auto data = static_cast<ProcessedFiles *>(totalCount);	×
118	data->Finish();
119	api.Release(data);	×
120	} else {
121	auto data = static_cast<DetailedProgressBar *>(totalCount);	×
122	api.Remove(data);	×
123	}
124	}
125	api.Clean();	×
126	std::this_thread::sleep_for(std::chrono::milliseconds(100));	×
127	});	×
128	}
129	};
130
131	bool ScanModules(const std::string &appFolder, const std::string &appName) {	×
132	DirectoryScanner sc;	×
133	sc.AddFilter(std::string_view(".spk$"));	×
134	sc.Scan(appFolder);	×
135	bool isOkay = true;
136
137	for (auto &m : sc) {	×
138	try {
139	AFileInfo modulePath(m);	×
140	auto moduleName = modulePath.GetFilename();	×
141	const size_t firstDotPos = moduleName.find_first_of('.');
142	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
143	APPContext ctx(moduleNameStr.data(), appFolder, appName);	×
144	ctx.FromConfig();	×
145	} catch (const std::runtime_error &e) {	×
146	printerror(e.what());	×
147	isOkay = false;
148	}	×
149	}
150
151	return isOkay;	×
152	}
153
154	void GenerateDocumentation(const std::string &appFolder,	×
155	const std::string &appName,
156	const std::string &templatePath) {
157	DirectoryScanner sc;	×
158	sc.AddFilter(std::string_view(".spk$"));	×
159	sc.Scan(appFolder);	×
160	std::set<std::string> modules;
161	pugi::xml_document doc;	×
162
163	if (!templatePath.empty()) {	×
164	doc = XMLFromFile(templatePath);	×
165	}
166
167	for (auto &m : sc) {	×
168	try {
169	AFileInfo modulePath(m);	×
170	auto moduleName = modulePath.GetFilename();	×
171	const size_t firstDotPos = moduleName.find_first_of('.');
172	std::string moduleNameStr(moduleName.substr(0, firstDotPos));	×
173	modules.emplace(moduleNameStr);
174	} catch (const std::runtime_error &e) {	×
175	printerror(e.what());	×
176	}	×
177	}
178
179	BinWritter_t<BinCoreOpenMode::Text> wr(appFolder + "/README.md");	×
180
181	const char *toolsetDescription = "[[TOOLSET DESCRIPTION]]";
182
183	if (auto child = doc.child("toolset_description"); child) {	×
184	toolsetDescription = child.text().as_string();	×
185	}
186
NEW 187	wr.BaseStream() << toolsetDescription << "<h2>Module list</h2>\n<ul>\n";	×
NEW 188	std::stringstream str;	×
189
190	for (auto &m : modules) {	×
191	pugi::xml_node node = doc.child(m.data());	×
192	APPContext ctx(m.data(), appFolder, appName);	×
NEW 193	ctx.GetMarkdownDoc(str, node);	×
NEW 194	std::string className = ctx.GetClassName(node);	×
195	std::string classNameLink = className;
196	std::replace_if(
197	classNameLink.begin(), classNameLink.end(),
198	[](char c) { return c == ' '; }, '-');
199
200	wr.BaseStream() << "<li><a href=\"#" << classNameLink << "\">" << className
NEW 201	<< "</a></li>\n";	×
202	}
203
NEW 204	wr.BaseStream() << "</ul>\n\n" << str.str() << "\n\n";	×
205
206	if (auto child = doc.child("toolset_footer"); child) {	×
207	wr.BaseStream() << child.text().as_string();	×
208	}
209	}
210
211	void PackModeBatch(Batch &batch) {	×
212	struct PackData {
213	size_t index = 0;
214	std::unique_ptr<AppPackContext> archiveContext;
215	std::string pbarLabel;
216	DetailedProgressBar *progBar = nullptr;
217	std::string folderPath;
218	};
219
220	auto payload = std::make_shared<PackData>();
221
222	batch.forEachFolder = [payload, ctx = batch.ctx](const std::string &path,	×
223	size_t numFiles) {	×
224	payload->folderPath = path;	×
225	payload->archiveContext.reset(ctx->NewArchive(path));
226	payload->pbarLabel = "Folder id " + std::to_string(payload->index++);	×
227	payload->progBar =	×
228	AppendNewLogLine<DetailedProgressBar>(payload->pbarLabel);	×
229	payload->progBar->ItemCount(numFiles);	×
230	uint8 consoleDetail = 1 \| uint8(ctx->info->multithreaded) << 1;	×
231	ConsolePrintDetail(consoleDetail);	×
232	printline("Processing: " << path);	×
233	};
234
235	batch.forEachFile = [payload](AppContextShare *iCtx) {	×
236	if (iCtx->workingFile.GetFullPath().starts_with(payload->folderPath)) {	×
237	int notSlash = !payload->folderPath.ends_with('/');	×
238	payload->archiveContext->SendFile(	×
239	iCtx->workingFile.GetFullPath().substr(payload->folderPath.size() +	×
240	notSlash),	×
241	iCtx->GetStream());	×
242	} else {
243	payload->archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
244	iCtx->GetStream());	×
245	}
246	(*payload->progBar)++;	×
247	};
248
249	batch.forEachFolderFinish = [payload] {	×
250	ConsolePrintDetail(1);	×
251	payload->archiveContext->Finish();	×
252	payload->archiveContext.reset();
253	RemoveLogLines(payload->progBar);	×
254	};
255	}
256
257	void MergePackModeBatch(Batch &batch, const std::string &folderPath,	×
258	AppPackContext *archiveContext) {
259	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
260	ConsolePrintDetail(consoleDetail);	×
261	printline("Processing: " << folderPath);	×
262
263	batch.forEachFile = [=](AppContextShare *iCtx) {	×
264	if (iCtx->workingFile.GetFullPath().starts_with(folderPath)) {	×
265	int notSlash = !folderPath.ends_with('/');	×
266	archiveContext->SendFile(	×
267	iCtx->workingFile.GetFullPath().substr(folderPath.size() + notSlash),	×
268	iCtx->GetStream());	×
269	} else {
270	archiveContext->SendFile(iCtx->workingFile.GetFullPath(),	×
271	iCtx->GetStream());	×
272	}
273	};
274	}
275
276	auto ExtractStatBatch(Batch &batch) {	×
277	struct ExtractStatsMaker : ExtractStats {
278	std::mutex mtx;
279	LoadingBar *scanBar;
280
281	void Push(AppContextShare *ctx, size_t numFiles) {	×
282	std::unique_lock<std::mutex> lg(mtx);	×
283	archiveFiles.emplace(ctx->Hash(), numFiles);	×
284	totalFiles += numFiles;	×
285	}
286
287	~ExtractStatsMaker() { RemoveLogLines(scanBar); }	×
288	};
289
290	batch.keepFinishLines = false;	×
291	auto sharedData = std::make_shared<ExtractStatsMaker>();
292	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
293	ConsolePrintDetail(consoleDetail);	×
294	sharedData->scanBar =	×
295	AppendNewLogLine<LoadingBar>("Processing extract stats.");	×
296
297	batch.forEachFile = [payload = sharedData,	×
298	ctx = batch.ctx](AppContextShare *iCtx) {	×
299	payload->Push(iCtx, ctx->ExtractStat(std::bind(	×
300	[&](size_t offset, size_t size) {
301	return iCtx->GetBuffer(size, offset);	×
302	},
303	std::placeholders::_1, std::placeholders::_2)));
304	};
305
306	return sharedData;	×
307	}
308
309	void ProcessBatch(Batch &batch, ExtractStats *stats) {	×
310	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
311	ConsolePrintDetail(consoleDetail);	×
312	batch.forEachFile = [payload = std::make_shared<UILines>(*stats),	×
313	archiveFiles =
314	std::make_shared<decltype(stats->archiveFiles)>(
315	std::move(stats->archiveFiles)),	×
316	ctx = batch.ctx](AppContextShare *iCtx) {	×
317	auto currentBar = payload->ChooseBar();	×
318	if (currentBar) {	×
319	currentBar->ItemCount(archiveFiles->at(iCtx->Hash()));	×
320	}
321
322	iCtx->forEachFile = [=] {	×
323	if (currentBar) {	×
324	(*currentBar)++;
325	}
326
327	if (payload->totalCount) {	×
328	(*payload->totalCount)++;
329	}
330	};
331
332	printline("Processing: " << iCtx->FullPath());	×
333	ctx->ProcessFile(iCtx);
334	if (payload->totalProgress) {	×
335	(*payload->totalProgress)++;
336	}
337	};
338	}
339
340	void ProcessBatch(Batch &batch, size_t numFiles) {	×
341	uint8 consoleDetail = 1 \| uint8(batch.ctx->info->multithreaded) << 1;	×
342	ConsolePrintDetail(consoleDetail);	×
343	auto payload = std::make_shared<UILines>(numFiles);
344	batch.forEachFile = [payload = payload,	×
345	ctx = batch.ctx](AppContextShare *iCtx) {	×
346	printline("Processing: " << iCtx->FullPath());	×
347	ctx->ProcessFile(iCtx);
348	if (payload->totalProgress) {	×
349	(*payload->totalProgress)++;
350	}
351	if (payload->totalCount) {	×
352	(*payload->totalCount)++;
353	}
354	};
355
356	auto totalFiles = std::make_shared<size_t>(numFiles);	×
357	batch.updateFileCount = [payload = payload,	×
358	totalFiles = totalFiles](size_t addedFiles) {	×
359	*totalFiles.get() += addedFiles;	×
360	payload->totalProgress->ItemCount(*totalFiles);	×
361	};
362	}
363
364	int CreateContent(const std::string &moduleName, const std::string &appFolder,	×
365	const std::string &appName, APPContext &ctx) {
366	try {
367	ctx = APPContext(moduleName.c_str(), appFolder, appName);	×
368	} catch (const std::exception &e) {	×
369	printerror(e.what());	×
370	return 2;
371	}	×
372
373	ConsolePrintDetail(0);	×
374
375	printline(ctx.info->header);	×
376	printline(appHeader0 << appName << ' ' << moduleName << appHeader1);	×
377
378	return 0;	×
379	}
380
381	int LoadProject(const std::string &path, const std::string &appFolder,	×
382	const std::string &appName) {
383	std::ifstream str(path);	×
384	nlohmann::json project(nlohmann::json::parse(str));	×
385
386	APPContext ctx;
387	std::string moduleName = project["module"];	×
388
389	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
390	return ret;
391	}
392
393	ConsolePrintDetail(1);	×
394
395	std::string outputDir;
396
397	if (!project["output_dir"].is_null()) {	×
398	std::string outputDir = project["output_dir"];	×
399	ctx.ApplySetting("out", outputDir);	×
400	}
401
402	bool noConfig = !project["no_config"].is_null() && project["no_config"];	×
403
404	if (!noConfig) {
405	printinfo("Loading config: " << appName << ".config");	×
406	ctx.FromConfig();	×
407	}
408
409	nlohmann::json settings = project["settings"];	×
410
411	for (auto &s : settings.items()) {	×
412	std::string dumped = s.value().dump();	×
413	if (dumped.front() == '"') {	×
414	dumped.erase(0, 1);	×
415	}
416	if (dumped.back() == '"') {	×
417	dumped.erase(dumped.size() - 1);	×
418	}
419	ctx.ApplySetting(s.key(), dumped);	×
420	}
421
422	nlohmann::json inputs = project["inputs"];	×
423
424	InitTempStorage();	×
425	ctx.SetupModule();	×
426	std::unique_ptr<AppPackContext> archiveContext;
427
428	{
429	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
430	AFileInfo batchPath(path);	×
431	std::string batchBase(batchPath.GetFolder());	×
432
433	if (ctx.NewArchive) {	×
434	std::string folder(batchPath.GetFolder());	×
435	std::string archive(batchPath.GetFullPathNoExt());	×
436	archiveContext.reset(batch.ctx->NewArchive(archive));	×
437	MergePackModeBatch(batch, folder, archiveContext.get());	×
438	} else {
439	if (ctx.ExtractStat) {	×
440	auto stats = ExtractStatBatch(batch);	×
441	for (std::string input : inputs) {	×
442	batch.AddFile(batchBase + input);	×
443	}
444
445	batch.FinishBatch();	×
446	batch.Clean();	×
447	stats.get()->totalFiles += inputs.size();	×
448	ProcessBatch(batch, stats.get());	×
449	} else {
450	ProcessBatch(batch, inputs.size());	×
451	}
452	}
453
454	if (ctx.info->batchControlFilters.size() > 0) {	×
455	std::string pathDir(AFileInfo(path).GetFolder());	×
456	for (nlohmann::json input : inputs) {	×
457	if (input.is_array()) {	×
458	batch.AddBatch(input, pathDir);	×
459	} else {
460	printwarning("Expected group, got " << input.type_name()	×
461	<< " instead. Skipping input.");
462	}
463	}
464	} else {
465	for (nlohmann::json input : inputs) {	×
466	if (input.is_string()) {	×
467	batch.AddFile(batchBase + std::string(input));	×
468	} else {
469	printwarning("Expected path string, got "	×
470	<< input.type_name() << " instead. Skipping input.");
471	}
472	}
473	}
474
475	batch.FinishBatch();	×
476	}
477
478	if (archiveContext) {	×
479	ConsolePrintDetail(1);	×
480	archiveContext->Finish();	×
481	}
482
483	if (ctx.FinishContext) {	×
484	ctx.FinishContext();
485	}
486
487	return 0;
488	}
489
490	int Main(int argc, TCHAR *argv[]) {	×
491	ConsolePrintDetail(1);	×
492	AFileInfo appLocation(std::to_string(*argv));	×
493	std::string appFolder(appLocation.GetFolder());	×
494	std::string appName(appLocation.GetFilename());	×
495	es::SetDllRunPath(appFolder + "lib");	×
496
497	if (argc < 2) {	×
498	printwarning(	×
499	"No parameters provided, entering scan mode and generating config.");
500	return !ScanModules(appFolder, appName);	×
501	}
502
503	auto moduleName = std::to_string(argv[1]);	×
504
505	if (moduleName == "--make-doc") {	×
506	std::string templatePath;
507
508	if (argc < 3) {	×
509	printwarning("Expexted template path!");	×
510	} else {
511	templatePath = std::to_string(argv[2]);	×
512	}
513
514	GenerateDocumentation(appFolder, appName, templatePath);	×
515	return 0;
516	} else if (moduleName.ends_with(".json")) {	×
517	return LoadProject(moduleName, appFolder, appName);	×
518	}
519
520	if (argc < 3) {	×
521	printerror("Insufficient argument count, expected parameters.");	×
522	return 1;
523	}
524
525	APPContext ctx;
526
527	if (int ret = CreateContent(moduleName, appFolder, appName, ctx); ret != 0) {	×
528	return ret;
529	}
530
531	if (IsHelp(argv[2])) {	×
532	ctx.PrintCLIHelp();	×
533	return 0;
534	}
535
536	ConsolePrintDetail(1);	×
537	bool dontLoadConfig = false;	×
538	std::vector<bool> markedFiles(size_t(argc), false);	×
539	size_t totalFiles = 0;
540
541	// Handle cli options and switches
542	for (int a = 2; a < argc; a++) {	×
543	auto opt = argv[a];	×
544
545	if (opt[0] == '-') {	×
546	auto optStr = std::to_string(opt);
547	std::string_view optsw(optStr);
548
549	if (optsw != "--out") {	×
550	// We won't use config file, reset all booleans to false,
551	// so we can properly use cli switches
552	[&] {	×
553	if (dontLoadConfig) {	×
554	return;
555	}
556
557	printinfo("CLI option detected, config won't be loaded, all booleans "	×
558	"set to false!");
559	ctx.ResetSwitchSettings();	×
560	}();	×
561	dontLoadConfig = true;	×
562	}
563	optsw.remove_prefix(1);
564
565	if (opt[0] == '-') {	×
566	optsw.remove_prefix(1);
567	}
568
569	auto valStr = std::to_string(argv[a + 1]);	×
570
571	if (auto retVal = ctx.ApplySetting(optsw, valStr); retVal > 0) {	×
572	a++;	×
573	}
574
575	} else {
576	markedFiles[a] = true;
577	totalFiles++;	×
578	}
579	}
580
581	if (!dontLoadConfig) {	×
582	printinfo("Loading config: " << appName << ".config");	×
583	ctx.FromConfig();	×
584	}
585
586	InitTempStorage();	×
587	ctx.SetupModule();	×
588	{
589	Batch batch(&ctx, ctx.info->multithreaded * 50);	×
590
591	if (ctx.NewArchive) {	×
592	PackModeBatch(batch);	×
593	} else {
594	if (ctx.ExtractStat) {	×
595	auto stats = ExtractStatBatch(batch);	×
596	for (int a = 2; a < argc; a++) {	×
597	if (!markedFiles.at(a)) {	×
598	continue;	×
599	}
600	batch.AddFile(std::to_string(argv[a]));	×
601	}
602
603	batch.FinishBatch();	×
604	batch.Clean();	×
605	stats.get()->totalFiles += totalFiles;	×
606	ProcessBatch(batch, stats.get());	×
607	} else {
608	ProcessBatch(batch, totalFiles);	×
609	}
610	}
611
612	for (int a = 2; a < argc; a++) {	×
613	if (!markedFiles.at(a)) {	×
614	continue;	×
615	}
616	batch.AddFile(std::to_string(argv[a]));	×
617	}
618
619	batch.FinishBatch();	×
620	}
621
622	if (ctx.FinishContext) {	×
623	ctx.FinishContext();
624	}
625
626	return 0;
627	}
628
629	int _tmain(int argc, TCHAR *argv[]) {	×
630	es::SetupWinApiConsole();
631	InitConsole();	×
632	CleanTempStorages();	×
633
634	int retVal = Main(argc, argv);	×
635
636	CleanCurrentTempStorage();	×
637
638	#ifndef NDEBUG
639	auto cacheStats = CacheGenerator::GlobalMetrics();
640	PrintInfo("Cache search hits: ", cacheStats.numSearchHits,
641	" search misses: ", cacheStats.numSearchMisses);
642	std::this_thread::sleep_for(std::chrono::milliseconds(100));
643	#endif
644	TerminateConsole();	×
645	return retVal;
646	}

PredatorCZ / PreCore / 514

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous