4485341203

Build Type

push

github

Committed by Hyo-Kyung Lee

Commit Message

ci: update checkout and cache action versions

Run Details

6193 of 7556 relevant lines covered (81.96%)

646738.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.66

/src/config_parser.cc

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Distributed under BSD 3-Clause license.                                   *
 * Copyright by The HDF Group.                                               *
 * Copyright by the Illinois Institute of Technology.                        *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of Hermes. The full Hermes copyright notice, including  *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the top directory. If you do not  *
 * have access to the file, you may request a copy from help@hdfgroup.org.   *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#include <float.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <yaml-cpp/yaml.h>
#include <ostream>
#include <glog/logging.h>
#include "utils.h"
#include "config_parser.h"
#include <iomanip>

// Steps to add a new configuration variable:
// 1. Add an "if" statement similar to ParseConfigYAML function for the variable
// 2. Add an Assert to config_parser_test.cc to test the functionality.
// 3. Set a default value in InitDefaultConfig
// 4. Add the variable with documentation to test/data/hermes.yaml

namespace hermes {

// TODO(llogan): Use to check for invalid params
/*
static const char *kConfigVariableStrings[] = {
  "unknown",
  "num_devices",
  "num_targets",
  "capacities_bytes",
  "capacities_kb",
  "capacities_mb",
  "capacities_gb",
  "block_sizes_bytes",
  "block_sizes_kb",
  "block_sizes_mb",
  "block_sizes_gb",
  "num_slabs",
  "slab_unit_sizes",
  "desired_slab_percentages",
  "bandwidths_mbps",
  "latencies_us",
  "buffer_pool_arena_percentage",
  "metadata_arena_percentage",
  "transient_arena_percentage",
  "mount_points",
  "swap_mount",
  "num_buffer_organizer_retries",
  "max_buckets_per_node",
  "max_vbuckets_per_node",
  "system_view_state_update_interval_ms",
  "rpc_server_host_file",
  "rpc_server_base_name",
  "rpc_server_suffix",
  "buffer_pool_shmem_name",
  "rpc_protocol",
  "rpc_domain",
  "rpc_port",
  "buffer_organizer_port",
  "rpc_host_number_range",
  "rpc_num_threads",
  "default_placement_policy",
  "is_shared_device",
  "buffer_organizer_num_threads",
  "default_rr_split",
  "bo_capacity_thresholds",
};*/

/** print \a expected value and fail when an error occurs */
void PrintExpectedAndFail(const std::string &expected, u32 line_number = 0) {
  std::ostringstream msg;
  msg << "Configuration parser expected '" << expected << "'";
  if (line_number > 0) {
    msg << " on line " << line_number;
  }
  msg << "\n";

  LOG(FATAL) << msg.str();
}

/** log an error message when the number of devices is 0 in \a config */
void RequireNumDevices(Config *config) {
  if (config->num_devices == 0) {
    LOG(FATAL) << "The configuration variable 'num_devices' must be defined "
               << "first" << std::endl;
  }
}

/** log an error message when the number of slabs is 0 in \a config  */
void RequireNumSlabs(Config *config) {
  if (config->num_slabs == 0) {
    LOG(FATAL) << "The configuration variable 'num_slabs' must be defined first"
               << std::endl;
  }
}

/** log an error message when capacities are specified multiple times */
void RequireCapacitiesUnset(bool &already_specified) {
  if (already_specified) {
    LOG(FATAL) << "Capacities are specified multiple times in the configuration"
               << " file. Only use one of 'capacities_bytes', 'capacities_kb',"
               << "'capacities_mb', or 'capacities_gb'\n";
  } else {
    already_specified = true;
  }
}

/** log an error message when block sizes are specified multiple times */
void RequireBlockSizesUnset(bool &already_specified) {
  if (already_specified) {
    LOG(FATAL) << "Block sizes are specified multiple times in the "
               << "configuration file. Only use one of 'block_sizes_bytes',"
               << "'block_sizes_kb', 'block_sizes_mb', or 'block_sizes_gb'\n";
  } else {
    already_specified = true;
  }
}

/** parse capacities from configuration file in YAML */
void ParseCapacities(Config *config, YAML::Node capacities,
                     int unit_conversion, bool &already_specified) {
  int i = 0;
  RequireNumDevices(config);
  RequireCapacitiesUnset(already_specified);
  for (auto val_node : capacities) {
    config->capacities[i++] = val_node.as<size_t>() * unit_conversion;
  }
}

/** parse block sizes from configuration file in YAML */
void ParseBlockSizes(Config *config, YAML::Node block_sizes,
                     int unit_conversion, bool &already_specified) {
  int i = 0;
  RequireNumDevices(config);
  RequireBlockSizesUnset(already_specified);
  for (auto val_node : block_sizes) {
    size_t block_size = val_node.as<size_t>() * unit_conversion;
    if (block_size > INT_MAX) {
      LOG(FATAL) << "Max supported block size is " << INT_MAX << " bytes. "
                 << "Config file requested " << block_size << " bytes\n";
    }
    config->block_sizes[i++] = block_size;
  }
}

/** parse \a var array from configuration file in YAML */
template<typename T>
void ParseArray(YAML::Node list_node, const std::string var,
                T *list, int max_list_len) {
  int i = 0;
  if (max_list_len < (int)list_node.size()) {
    LOG(FATAL) << var << " (array) had "
               << list_node.size() << " arguments "
               << "but up to " << max_list_len << " expected\n";
  }
  for (auto val_node : list_node) {
    list[i++] = val_node.as<T>();
  }
}

/** parse \a list_node vector from configuration file in YAML */
template<typename T>
void ParseVector(YAML::Node list_node, std::vector<T> &list) {
  for (auto val_node : list_node) {
    list.emplace_back(val_node.as<T>());
  }
}

/** parse \a matrix_node matrix using \a col_len column length */
template<typename T>
void ParseMatrix(YAML::Node matrix_node, const std::string var, T *matrix,
                 int max_row_len, int max_col_len, int *col_len) {
  int i = 0;
  if (max_row_len < (int)matrix_node.size()) {
    LOG(FATAL) << var << " (matrix) had "
               << matrix_node.size() << " arguments "
               << "but up to " << max_row_len << " expected\n";
  }
  for (auto row : matrix_node) {
    ParseArray<T>(row, var, &matrix[i*max_col_len], col_len[i]);
    ++i;
  }
}

/** parse \a matrix_node matrix from configuration file in YAML */
template<typename T>
void ParseMatrix(YAML::Node matrix_node, std::string var, T *matrix,
                 int max_row_len, int max_col_len) {
  int i = 0;
  if (max_row_len < (int)matrix_node.size()) {
    LOG(FATAL) << var << " (matrix) had "
               << matrix_node.size() << " arguments "
               << "but up to " << max_row_len << " expected\n";
  }
  for (auto row : matrix_node) {
    ParseArray<T>(row, var, &matrix[i*max_col_len], max_col_len);
    ++i;
  }
}

/** parse range list from configuration file in YAML */
void ParseRangeList(YAML::Node list_node, std::string var,
                    std::vector<std::string> &list) {
  int min, max, width = 0;
  for (auto val_node : list_node) {
    std::string val = val_node.as<std::string>();
    if (val.find('-') == std::string::npos) {
      min = val_node.as<int>();
      max = min;
    } else {
      std::stringstream ss(val);
      std::string word;
      std::vector<std::string> words;
      while (std::getline(ss, word, '-')) {
        words.push_back(word);
      }
      if (words.size() != 2) {
        LOG(FATAL) << var <<
            " has invalid range definition " << val << std::endl;
        return;
      }
      min = std::stoi(words[0]);
      max = std::stoi(words[1]);
      width = words[0].size();
    }

    if (width > 0) {
      for (int i = min; i <= max; ++i) {
        std::stringstream ss;
        ss << std::setw(width) << std::setfill('0') << i;
        list.emplace_back(ss.str());
      }
    } else {
      for (int i = min; i <= max; ++i) {
        list.emplace_back(std::to_string(i));
      }
    }
  }
}

/** parse host names from configuration file in YAML */
void ParseHostNames(YAML::Node yaml_conf, hermes::Config *config) {
  if (yaml_conf["rpc_server_host_file"]) {
    config->rpc_server_host_file =
        yaml_conf["rpc_server_host_file"].as<std::string>();
  }
  if (yaml_conf["rpc_server_base_name"]) {
    config->rpc_server_base_name =
        yaml_conf["rpc_server_base_name"].as<std::string>();
  }
  if (yaml_conf["rpc_server_suffix"]) {
    config->rpc_server_suffix =
        yaml_conf["rpc_server_suffix"].as<std::string>();
  }
  if (yaml_conf["rpc_host_number_range"]) {
    ParseRangeList(yaml_conf["rpc_host_number_range"],
                   "rpc_host_number_range",
                   config->host_numbers);
  }

  if (config->rpc_server_host_file.empty()) {
    config->host_names.clear();
    if (config->host_numbers.size() == 0) {
      config->host_numbers.emplace_back("");
    }
    for (auto &host_number : config->host_numbers) {
      config->host_names.emplace_back(
          config->rpc_server_base_name +
          host_number +
          config->rpc_server_suffix);
    }
  }
}

/** check constraints in \a config configuration */
void CheckConstraints(Config *config) {
  // rpc_domain must be present if rpc_protocol is "verbs"
  if (config->rpc_protocol.find("verbs") != std::string::npos &&
      config->rpc_domain.empty()) {
    PrintExpectedAndFail("a non-empty value for rpc_domain");
  }

  double tolerance = 0.0000001;

  // arena_percentages must add up to 1.0
  double arena_percentage_sum = 0;
  for (int i = 0; i < kArenaType_Count; ++i) {
    arena_percentage_sum += config->arena_percentages[i];
  }
  if (fabs(1.0 - arena_percentage_sum) > tolerance) {
    std::ostringstream msg;
    msg << "the values in arena_percentages to add up to 1.0 but got ";
    msg << arena_percentage_sum << "\n";
    PrintExpectedAndFail(msg.str());
  }

  // Each slab's desired_slab_percentages should add up to 1.0
  for (int device = 0; device < config->num_devices; ++device) {
    double total_slab_percentage = 0;
    for (int slab = 0; slab < config->num_slabs[device]; ++slab) {
      total_slab_percentage += config->desired_slab_percentages[device][slab];
    }
    if (fabs(1.0 - total_slab_percentage) > tolerance) {
      std::ostringstream msg;
      msg << "the values in desired_slab_percentages[";
      msg << device;
      msg << "] to add up to 1.0 but got ";
      msg << total_slab_percentage << "\n";
      PrintExpectedAndFail(msg.str());
    }
  }
}

void ParseConfigYAML(YAML::Node &yaml_conf, Config *config) {
  bool capcities_specified = false, block_sizes_specified = false;
  std::vector<std::string> host_numbers;
  std::vector<std::string> host_basename;
  std::string host_suffix;

  if (yaml_conf["num_devices"]) {
    config->num_devices = yaml_conf["num_devices"].as<int>();
    config->num_targets = config->num_devices;
  }
  if (yaml_conf["num_targets"]) {
    config->num_targets = yaml_conf["num_targets"].as<int>();
  }

  if (yaml_conf["capacities_bytes"]) {
    ParseCapacities(config, yaml_conf["capacities_bytes"], 1,
                    capcities_specified);
  }
  if (yaml_conf["capacities_kb"]) {
    ParseCapacities(config, yaml_conf["capacities_kb"], KILOBYTES(1),
                    capcities_specified);
  }
  if (yaml_conf["capacities_mb"]) {
    ParseCapacities(config, yaml_conf["capacities_mb"], MEGABYTES(1),
                    capcities_specified);
  }
  if (yaml_conf["capacities_gb"]) {
    ParseCapacities(config, yaml_conf["capacities_gb"], GIGABYTES(1),
                    capcities_specified);
  }

  if (yaml_conf["block_sizes_bytes"]) {
    ParseBlockSizes(config, yaml_conf["block_sizes_bytes"],
                    1, block_sizes_specified);
  }
  if (yaml_conf["block_sizes_kb"]) {
    ParseBlockSizes(config, yaml_conf["block_sizes_kb"], KILOBYTES(1),
                    block_sizes_specified);
  }
  if (yaml_conf["block_sizes_mb"]) {
    ParseBlockSizes(config, yaml_conf["block_sizes_mb"], MEGABYTES(1),
                    block_sizes_specified);
  }
  if (yaml_conf["block_sizes_gb"]) {
    ParseBlockSizes(config, yaml_conf["block_sizes_gb"], GIGABYTES(1),
                    block_sizes_specified);
  }

  if (yaml_conf["num_slabs"]) {
    RequireNumDevices(config);
    ParseArray<int>(yaml_conf["num_slabs"], "num_slabs",
                    config->num_slabs,
                    config->num_devices);
  }
  if (yaml_conf["slab_unit_sizes"]) {
    RequireNumDevices(config);
    RequireNumSlabs(config);
    ParseMatrix<int>(yaml_conf["slab_unit_sizes"],
                     "slab_unit_sizes",
                     reinterpret_cast<int*>(config->slab_unit_sizes),
                     kMaxDevices, kMaxBufferPoolSlabs, config->num_slabs);
  }
  if (yaml_conf["desired_slab_percentages"]) {
    RequireNumDevices(config);
    RequireNumSlabs(config);
    ParseMatrix<f32>(yaml_conf["desired_slab_percentages"],
                     "desired_slab_percentages",
                     reinterpret_cast<f32*>(config->desired_slab_percentages),
                     kMaxDevices, kMaxBufferPoolSlabs, config->num_slabs);
  }
  if (yaml_conf["bandwidths_mbps"]) {
    RequireNumDevices(config);
    ParseArray<f32>(yaml_conf["bandwidths_mbps"],
                    "bandwidths_mbps",
                    config->bandwidths, config->num_devices);
  }
  if (yaml_conf["latencies"]) {
    RequireNumDevices(config);
    ParseArray<f32>(yaml_conf["latencies"],
                    "latencies",
                    config->latencies, config->num_devices);
  }
  if (yaml_conf["buffer_pool_arena_percentage"]) {
    config->arena_percentages[hermes::kArenaType_BufferPool] =
        yaml_conf["buffer_pool_arena_percentage"].as<f32>();
  }
  if (yaml_conf["metadata_arena_percentage"]) {
    config->arena_percentages[hermes::kArenaType_MetaData] =
        yaml_conf["metadata_arena_percentage"].as<f32>();
  }
  if (yaml_conf["transient_arena_percentage"]) {
    config->arena_percentages[hermes::kArenaType_Transient] =
        yaml_conf["transient_arena_percentage"].as<f32>();
  }
  if (yaml_conf["mount_points"]) {
    RequireNumDevices(config);
    ParseArray<std::string>(yaml_conf["mount_points"],
                            "mount_points",
                            config->mount_points, config->num_devices);
  }
  if (yaml_conf["swap_mount"]) {
    config->swap_mount = yaml_conf["swap_mount"].as<std::string>();
  }
  if (yaml_conf["num_buffer_organizer_retries"]) {
    config->num_buffer_organizer_retries =
        yaml_conf["num_buffer_organizer_retries"].as<int>();
  }
  if (yaml_conf["max_buckets_per_node"]) {
    config->max_buckets_per_node =
        yaml_conf["max_buckets_per_node"].as<int>();
  }
  if (yaml_conf["max_vbuckets_per_node"]) {
    config->max_vbuckets_per_node =
        yaml_conf["max_vbuckets_per_node"].as<int>();
  }
  if (yaml_conf["system_view_state_update_interval_ms"]) {
    config->system_view_state_update_interval_ms =
        yaml_conf["system_view_state_update_interval_ms"].as<int>();
  }
  if (yaml_conf["buffer_pool_shmem_name"]) {
    std::string name = yaml_conf["buffer_pool_shmem_name"].as<std::string>();
    std::snprintf(config->buffer_pool_shmem_name,
                  kMaxBufferPoolShmemNameLength,
                  "%s", name.c_str());
  }
  if (yaml_conf["rpc_protocol"]) {
    config->rpc_protocol = yaml_conf["rpc_protocol"].as<std::string>();
  }
  if (yaml_conf["rpc_domain"]) {
    config->rpc_domain = yaml_conf["rpc_domain"].as<std::string>();
  }
  if (yaml_conf["rpc_port"]) {
    config->rpc_port = yaml_conf["rpc_port"].as<int>();
  }
  if (yaml_conf["buffer_organizer_port"]) {
    config->buffer_organizer_port =
        yaml_conf["buffer_organizer_port"].as<int>();
  }
  if (yaml_conf["rpc_num_threads"]) {
    config->rpc_num_threads =
        yaml_conf["rpc_num_threads"].as<int>();
  }
  if (yaml_conf["default_placement_policy"]) {
    std::string policy =
        yaml_conf["default_placement_policy"].as<std::string>();

    if (policy == "MinimizeIoTime") {
      config->default_placement_policy =
          api::PlacementPolicy::kMinimizeIoTime;
    } else if (policy == "Random") {
      config->default_placement_policy = api::PlacementPolicy::kRandom;
    } else if (policy == "RoundRobin") {
      config->default_placement_policy = api::PlacementPolicy::kRoundRobin;
    } else {
      LOG(FATAL) << "Unknown default_placement_policy: '" << policy << "'"
                 << std::endl;
    }
  }
  if (yaml_conf["is_shared_device"]) {
    RequireNumDevices(config);
    ParseArray<int>(yaml_conf["is_shared_device"],
                    "is_shared_device",
                    config->is_shared_device, config->num_devices);
  }
  if (yaml_conf["buffer_organizer_num_threads"]) {
    config->bo_num_threads =
        yaml_conf["buffer_organizer_num_threads"].as<int>();
  }
  if (yaml_conf["default_rr_split"]) {
    config->default_rr_split = yaml_conf["default_rr_split"].as<int>();
  }
  if (yaml_conf["bo_num_threads"]) {
    config->bo_num_threads = yaml_conf["bo_num_threads"].as<int>();
  }
  if (yaml_conf["bo_capacity_thresholds"]) {
    RequireNumDevices(config);
    f32 thresholds[kMaxDevices][2] = {0};
    ParseMatrix<f32>(yaml_conf["bo_capacity_thresholds"],
                     "bo_capacity_thresholds",
                     reinterpret_cast<f32*>(thresholds),
                     kMaxDevices, 2);
    for (int i = 0; i < config->num_devices; ++i) {
      config->bo_capacity_thresholds[i].min = thresholds[i][0];
      config->bo_capacity_thresholds[i].max = thresholds[i][1];
    }
  }
  if (yaml_conf["path_exclusions"]) {
    ParseVector<std::string>(
        yaml_conf["path_exclusions"], config->path_exclusions);
  }
  if (yaml_conf["path_inclusions"]) {
    ParseVector<std::string>(
        yaml_conf["path_inclusions"], config->path_inclusions);
  }
  ParseHostNames(yaml_conf, config);
  CheckConstraints(config);
}

/**
   parse YAML configuration file
 */
void ParseConfig(Arena *arena, const char *path, Config *config) {
  ScopedTemporaryMemory scratch(arena);
  InitDefaultConfig(config);
  LOG(INFO) << "ParseConfig-LoadFile" << std::endl;
  YAML::Node yaml_conf = YAML::LoadFile(path);
  LOG(INFO) << "ParseConfig-LoadComplete" << std::endl;
  ParseConfigYAML(yaml_conf, config);
}

/**
   parse configuration string
 */
void ParseConfigString(
    Arena *arena, const std::string &config_string, Config *config) {
  ScopedTemporaryMemory scratch(arena);
  InitDefaultConfig(config);
  YAML::Node yaml_conf = YAML::Load(config_string);
  ParseConfigYAML(yaml_conf, config);
}

void InitConfig(hermes::Config *config, const char *config_file) {
  const size_t kConfigMemorySize = KILOBYTES(16);
  hermes::u8 config_memory[kConfigMemorySize];
  if (config_file) {
    hermes::Arena config_arena = {};
    hermes::InitArena(&config_arena, kConfigMemorySize, config_memory);
    hermes::ParseConfig(&config_arena, config_file, config);
  } else {
    InitDefaultConfig(config);
  }
}

}  // namespace hermes

1	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2	* Distributed under BSD 3-Clause license. *
3	* Copyright by The HDF Group. *
4	* Copyright by the Illinois Institute of Technology. *
5	* All rights reserved. *
6	* *
7	* This file is part of Hermes. The full Hermes copyright notice, including *
8	* terms governing use, modification, and redistribution, is contained in *
9	* the COPYING file, which can be found at the top directory. If you do not *
10	* have access to the file, you may request a copy from help@hdfgroup.org. *
11	* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
12
13	#include <float.h>
14	#include <limits.h>
15	#include <stdio.h>
16	#include <stdlib.h>
17	#include <string.h>
18	#include <yaml-cpp/yaml.h>
19	#include <ostream>
20	#include <glog/logging.h>
21	#include "utils.h"
22	#include "config_parser.h"
23	#include <iomanip>
24
25	// Steps to add a new configuration variable:
26	// 1. Add an "if" statement similar to ParseConfigYAML function for the variable
27	// 2. Add an Assert to config_parser_test.cc to test the functionality.
28	// 3. Set a default value in InitDefaultConfig
29	// 4. Add the variable with documentation to test/data/hermes.yaml
30
31	namespace hermes {
32
33	// TODO(llogan): Use to check for invalid params
34	/*
35	static const char *kConfigVariableStrings[] = {
36	"unknown",
37	"num_devices",
38	"num_targets",
39	"capacities_bytes",
40	"capacities_kb",
41	"capacities_mb",
42	"capacities_gb",
43	"block_sizes_bytes",
44	"block_sizes_kb",
45	"block_sizes_mb",
46	"block_sizes_gb",
47	"num_slabs",
48	"slab_unit_sizes",
49	"desired_slab_percentages",
50	"bandwidths_mbps",
51	"latencies_us",
52	"buffer_pool_arena_percentage",
53	"metadata_arena_percentage",
54	"transient_arena_percentage",
55	"mount_points",
56	"swap_mount",
57	"num_buffer_organizer_retries",
58	"max_buckets_per_node",
59	"max_vbuckets_per_node",
60	"system_view_state_update_interval_ms",
61	"rpc_server_host_file",
62	"rpc_server_base_name",
63	"rpc_server_suffix",
64	"buffer_pool_shmem_name",
65	"rpc_protocol",
66	"rpc_domain",
67	"rpc_port",
68	"buffer_organizer_port",
69	"rpc_host_number_range",
70	"rpc_num_threads",
71	"default_placement_policy",
72	"is_shared_device",
73	"buffer_organizer_num_threads",
74	"default_rr_split",
75	"bo_capacity_thresholds",
76	};*/
77
78	/** print \a expected value and fail when an error occurs */
79	void PrintExpectedAndFail(const std::string &expected, u32 line_number = 0) {	×
80	std::ostringstream msg;	×
81	msg << "Configuration parser expected '" << expected << "'";	×
82	if (line_number > 0) {	×
83	msg << " on line " << line_number;	×
84	}
85	msg << "\n";	×
86
87	LOG(FATAL) << msg.str();	×
88	}
89
90	/** log an error message when the number of devices is 0 in \a config */
91	void RequireNumDevices(Config *config) {	444✔
92	if (config->num_devices == 0) {	444✔
93	LOG(FATAL) << "The configuration variable 'num_devices' must be defined "	×
94	<< "first" << std::endl;	×
95	}
96	}	444✔
97
98	/** log an error message when the number of slabs is 0 in \a config */
99	void RequireNumSlabs(Config *config) {	124✔
100	if (config->num_slabs == 0) {	124✔
101	LOG(FATAL) << "The configuration variable 'num_slabs' must be defined first"	×
102	<< std::endl;	×
103	}
104	}	124✔
105
106	/** log an error message when capacities are specified multiple times */
107	void RequireCapacitiesUnset(bool &already_specified) {	66✔
108	if (already_specified) {	66✔
109	LOG(FATAL) << "Capacities are specified multiple times in the configuration"	×
110	<< " file. Only use one of 'capacities_bytes', 'capacities_kb',"
111	<< "'capacities_mb', or 'capacities_gb'\n";	×
112	} else {
113	already_specified = true;	66✔
114	}
115	}	66✔
116
117	/** log an error message when block sizes are specified multiple times */
118	void RequireBlockSizesUnset(bool &already_specified) {	66✔
119	if (already_specified) {	66✔
120	LOG(FATAL) << "Block sizes are specified multiple times in the "	×
121	<< "configuration file. Only use one of 'block_sizes_bytes',"
122	<< "'block_sizes_kb', 'block_sizes_mb', or 'block_sizes_gb'\n";	×
123	} else {
124	already_specified = true;	66✔
125	}
126	}	66✔
127
128	/** parse capacities from configuration file in YAML */
129	void ParseCapacities(Config *config, YAML::Node capacities,	66✔
130	int unit_conversion, bool &already_specified) {
131	int i = 0;	66✔
132	RequireNumDevices(config);	66✔
133	RequireCapacitiesUnset(already_specified);	66✔
134	for (auto val_node : capacities) {	330✔
135	config->capacities[i++] = val_node.as<size_t>() * unit_conversion;	264✔
136	}
137	}	66✔
138
139	/** parse block sizes from configuration file in YAML */
140	void ParseBlockSizes(Config *config, YAML::Node block_sizes,	66✔
141	int unit_conversion, bool &already_specified) {
142	int i = 0;	66✔
143	RequireNumDevices(config);	66✔
144	RequireBlockSizesUnset(already_specified);	66✔
145	for (auto val_node : block_sizes) {	330✔
146	size_t block_size = val_node.as<size_t>() * unit_conversion;	264✔
147	if (block_size > INT_MAX) {	264✔
148	LOG(FATAL) << "Max supported block size is " << INT_MAX << " bytes. "	×
149	<< "Config file requested " << block_size << " bytes\n";	×
150	}
151	config->block_sizes[i++] = block_size;	264✔
152	}
153	}	66✔
154
155	/** parse \a var array from configuration file in YAML */
156	template<typename T>
157	void ParseArray(YAML::Node list_node, const std::string var,	699✔
158	T *list, int max_list_len) {
159	int i = 0;	699✔
160	if (max_list_len < (int)list_node.size()) {	699✔
161	LOG(FATAL) << var << " (array) had "	×
162	<< list_node.size() << " arguments "	×
163	<< "but up to " << max_list_len << " expected\n";	×
164	}
165	for (auto val_node : list_node) {	6,211✔
166	list[i++] = val_node.as<T>();	2,756✔
167	}
168	}	699✔
169
170	/** parse \a list_node vector from configuration file in YAML */
171	template<typename T>
172	void ParseVector(YAML::Node list_node, std::vector<T> &list) {	14✔
173	for (auto val_node : list_node) {	294✔
174	list.emplace_back(val_node.as<T>());	140✔
175	}
176	}	14✔
177
178	/** parse \a matrix_node matrix using \a col_len column length */
179	template<typename T>
180	void ParseMatrix(YAML::Node matrix_node, const std::string var, T *matrix,	124✔
181	int max_row_len, int max_col_len, int *col_len) {
182	int i = 0;	124✔
183	if (max_row_len < (int)matrix_node.size()) {	124✔
184	LOG(FATAL) << var << " (matrix) had "	×
185	<< matrix_node.size() << " arguments "	×
186	<< "but up to " << max_row_len << " expected\n";	×
187	}
188	for (auto row : matrix_node) {	1,116✔
189	ParseArray<T>(row, var, &matrix[i*max_col_len], col_len[i]);	496✔
190	++i;	496✔
191	}
192	}	124✔
193
194	/** parse \a matrix_node matrix from configuration file in YAML */
195	template<typename T>
196	void ParseMatrix(YAML::Node matrix_node, std::string var, T *matrix,	5✔
197	int max_row_len, int max_col_len) {
198	int i = 0;	5✔
199	if (max_row_len < (int)matrix_node.size()) {	5✔
200	LOG(FATAL) << var << " (matrix) had "	×
201	<< matrix_node.size() << " arguments "	×
202	<< "but up to " << max_row_len << " expected\n";	×
203	}
204	for (auto row : matrix_node) {	45✔
205	ParseArray<T>(row, var, &matrix[i*max_col_len], max_col_len);	20✔
206	++i;	20✔
207	}
208	}	5✔
209
210	/** parse range list from configuration file in YAML */
211	void ParseRangeList(YAML::Node list_node, std::string var,	66✔
212	std::vector<std::string> &list) {
213	int min, max, width = 0;	66✔
214	for (auto val_node : list_node) {	72✔
215	std::string val = val_node.as<std::string>();	6✔
216	if (val.find('-') == std::string::npos) {	6✔
217	min = val_node.as<int>();	3✔
218	max = min;	3✔
219	} else {
220	std::stringstream ss(val);	3✔
221	std::string word;	3✔
222	std::vector<std::string> words;	3✔
223	while (std::getline(ss, word, '-')) {	9✔
224	words.push_back(word);	6✔
225	}
226	if (words.size() != 2) {	3✔
227	LOG(FATAL) << var <<	×
228	" has invalid range definition " << val << std::endl;	×
229	return;
230	}
231	min = std::stoi(words[0]);	3✔
232	max = std::stoi(words[1]);	3✔
233	width = words[0].size();	3✔
234	}
235
236	if (width > 0) {	6✔
237	for (int i = min; i <= max; ++i) {	14✔
238	std::stringstream ss;	20✔
239	ss << std::setw(width) << std::setfill('0') << i;	10✔
240	list.emplace_back(ss.str());	10✔
241	}
242	} else {
243	for (int i = min; i <= max; ++i) {	4✔
244	list.emplace_back(std::to_string(i));	2✔
245	}
246	}
247	}
248	}
249
250	/** parse host names from configuration file in YAML */
251	void ParseHostNames(YAML::Node yaml_conf, hermes::Config *config) {	75✔
252	if (yaml_conf["rpc_server_host_file"]) {	75✔
253	config->rpc_server_host_file =	5✔
254	yaml_conf["rpc_server_host_file"].as<std::string>();	5✔
255	}
256	if (yaml_conf["rpc_server_base_name"]) {	75✔
257	config->rpc_server_base_name =	66✔
258	yaml_conf["rpc_server_base_name"].as<std::string>();	66✔
259	}
260	if (yaml_conf["rpc_server_suffix"]) {	75✔
261	config->rpc_server_suffix =	5✔
262	yaml_conf["rpc_server_suffix"].as<std::string>();	5✔
263	}
264	if (yaml_conf["rpc_host_number_range"]) {	75✔
265	ParseRangeList(yaml_conf["rpc_host_number_range"],	66✔
266	"rpc_host_number_range",
267	config->host_numbers);	66✔
268	}
269
270	if (config->rpc_server_host_file.empty()) {	75✔
271	config->host_names.clear();	71✔
272	if (config->host_numbers.size() == 0) {	71✔
273	config->host_numbers.emplace_back("");	68✔
274	}
275	for (auto &host_number : config->host_numbers) {	151✔
276	config->host_names.emplace_back(	80✔
277	config->rpc_server_base_name +	160✔
278	host_number +	160✔
279	config->rpc_server_suffix);	160✔
280	}
281	}
282	}	75✔
283
284	/** check constraints in \a config configuration */
285	void CheckConstraints(Config *config) {	75✔
286	// rpc_domain must be present if rpc_protocol is "verbs"
287	if (config->rpc_protocol.find("verbs") != std::string::npos &&	75✔
288	config->rpc_domain.empty()) {	×
289	PrintExpectedAndFail("a non-empty value for rpc_domain");	×
290	}
291
292	double tolerance = 0.0000001;	75✔
293
294	// arena_percentages must add up to 1.0
295	double arena_percentage_sum = 0;	75✔
296	for (int i = 0; i < kArenaType_Count; ++i) {	300✔
297	arena_percentage_sum += config->arena_percentages[i];	225✔
298	}
299	if (fabs(1.0 - arena_percentage_sum) > tolerance) {	75✔
300	std::ostringstream msg;	×
301	msg << "the values in arena_percentages to add up to 1.0 but got ";	×
302	msg << arena_percentage_sum << "\n";	×
303	PrintExpectedAndFail(msg.str());	×
304	}
305
306	// Each slab's desired_slab_percentages should add up to 1.0
307	for (int device = 0; device < config->num_devices; ++device) {	375✔
308	double total_slab_percentage = 0;	300✔
309	for (int slab = 0; slab < config->num_slabs[device]; ++slab) {	1,500✔
310	total_slab_percentage += config->desired_slab_percentages[device][slab];	1,200✔
311	}
312	if (fabs(1.0 - total_slab_percentage) > tolerance) {	300✔
313	std::ostringstream msg;	×
314	msg << "the values in desired_slab_percentages[";	×
315	msg << device;	×
316	msg << "] to add up to 1.0 but got ";	×
317	msg << total_slab_percentage << "\n";	×
318	PrintExpectedAndFail(msg.str());	×
319	}
320	}
321	}	75✔
322
323	void ParseConfigYAML(YAML::Node &yaml_conf, Config *config) {	75✔
324	bool capcities_specified = false, block_sizes_specified = false;	75✔
325	std::vector<std::string> host_numbers;	150✔
326	std::vector<std::string> host_basename;	150✔
327	std::string host_suffix;	150✔
328
329	if (yaml_conf["num_devices"]) {	75✔
330	config->num_devices = yaml_conf["num_devices"].as<int>();	70✔
331	config->num_targets = config->num_devices;	70✔
332	}
333	if (yaml_conf["num_targets"]) {	75✔
334	config->num_targets = yaml_conf["num_targets"].as<int>();	62✔
335	}
336
337	if (yaml_conf["capacities_bytes"]) {	75✔
338	ParseCapacities(config, yaml_conf["capacities_bytes"], 1,	1✔
339	capcities_specified);
340	}
341	if (yaml_conf["capacities_kb"]) {	75✔
342	ParseCapacities(config, yaml_conf["capacities_kb"], KILOBYTES(1),	1✔
343	capcities_specified);
344	}
345	if (yaml_conf["capacities_mb"]) {	75✔
346	ParseCapacities(config, yaml_conf["capacities_mb"], MEGABYTES(1),	63✔
347	capcities_specified);
348	}
349	if (yaml_conf["capacities_gb"]) {	75✔
350	ParseCapacities(config, yaml_conf["capacities_gb"], GIGABYTES(1),	1✔
351	capcities_specified);
352	}
353
354	if (yaml_conf["block_sizes_bytes"]) {	75✔
355	ParseBlockSizes(config, yaml_conf["block_sizes_bytes"],	1✔
356	1, block_sizes_specified);
357	}
358	if (yaml_conf["block_sizes_kb"]) {	75✔
359	ParseBlockSizes(config, yaml_conf["block_sizes_kb"], KILOBYTES(1),	63✔
360	block_sizes_specified);
361	}
362	if (yaml_conf["block_sizes_mb"]) {	75✔
363	ParseBlockSizes(config, yaml_conf["block_sizes_mb"], MEGABYTES(1),	1✔
364	block_sizes_specified);
365	}
366	if (yaml_conf["block_sizes_gb"]) {	75✔
367	ParseBlockSizes(config, yaml_conf["block_sizes_gb"], GIGABYTES(1),	1✔
368	block_sizes_specified);
369	}
370
371	if (yaml_conf["num_slabs"]) {	75✔
372	RequireNumDevices(config);	62✔
373	ParseArray<int>(yaml_conf["num_slabs"], "num_slabs",	62✔
374	config->num_slabs,	62✔
375	config->num_devices);
376	}
377	if (yaml_conf["slab_unit_sizes"]) {	75✔
378	RequireNumDevices(config);	62✔
379	RequireNumSlabs(config);	62✔
380	ParseMatrix<int>(yaml_conf["slab_unit_sizes"],	62✔
381	"slab_unit_sizes",
382	reinterpret_cast<int*>(config->slab_unit_sizes),	62✔
383	kMaxDevices, kMaxBufferPoolSlabs, config->num_slabs);	62✔
384	}
385	if (yaml_conf["desired_slab_percentages"]) {	75✔
386	RequireNumDevices(config);	62✔
387	RequireNumSlabs(config);	62✔
388	ParseMatrix<f32>(yaml_conf["desired_slab_percentages"],	62✔
389	"desired_slab_percentages",
390	reinterpret_cast<f32*>(config->desired_slab_percentages),	62✔
391	kMaxDevices, kMaxBufferPoolSlabs, config->num_slabs);	62✔
392	}
393	if (yaml_conf["bandwidths_mbps"]) {	75✔
394	RequireNumDevices(config);	54✔
395	ParseArray<f32>(yaml_conf["bandwidths_mbps"],	54✔
396	"bandwidths_mbps",
397	config->bandwidths, config->num_devices);	54✔
398	}
399	if (yaml_conf["latencies"]) {	75✔
400	RequireNumDevices(config);	×
401	ParseArray<f32>(yaml_conf["latencies"],	×
402	"latencies",
403	config->latencies, config->num_devices);	×
404	}
405	if (yaml_conf["buffer_pool_arena_percentage"]) {	75✔
406	config->arena_percentages[hermes::kArenaType_BufferPool] =	62✔
407	yaml_conf["buffer_pool_arena_percentage"].as<f32>();	62✔
408	}
409	if (yaml_conf["metadata_arena_percentage"]) {	75✔
410	config->arena_percentages[hermes::kArenaType_MetaData] =	62✔
411	yaml_conf["metadata_arena_percentage"].as<f32>();	62✔
412	}
413	if (yaml_conf["transient_arena_percentage"]) {	75✔
414	config->arena_percentages[hermes::kArenaType_Transient] =	62✔
415	yaml_conf["transient_arena_percentage"].as<f32>();	62✔
416	}
417	if (yaml_conf["mount_points"]) {	75✔
418	RequireNumDevices(config);	62✔
419	ParseArray<std::string>(yaml_conf["mount_points"],	62✔
420	"mount_points",
421	config->mount_points, config->num_devices);	62✔
422	}
423	if (yaml_conf["swap_mount"]) {	75✔
424	config->swap_mount = yaml_conf["swap_mount"].as<std::string>();	62✔
425	}
426	if (yaml_conf["num_buffer_organizer_retries"]) {	75✔
427	config->num_buffer_organizer_retries =	62✔
428	yaml_conf["num_buffer_organizer_retries"].as<int>();	62✔
429	}
430	if (yaml_conf["max_buckets_per_node"]) {	75✔
431	config->max_buckets_per_node =	62✔
432	yaml_conf["max_buckets_per_node"].as<int>();	62✔
433	}
434	if (yaml_conf["max_vbuckets_per_node"]) {	75✔
435	config->max_vbuckets_per_node =	62✔
436	yaml_conf["max_vbuckets_per_node"].as<int>();	62✔
437	}
438	if (yaml_conf["system_view_state_update_interval_ms"]) {	75✔
439	config->system_view_state_update_interval_ms =	62✔
440	yaml_conf["system_view_state_update_interval_ms"].as<int>();	62✔
441	}
442	if (yaml_conf["buffer_pool_shmem_name"]) {	75✔
443	std::string name = yaml_conf["buffer_pool_shmem_name"].as<std::string>();	124✔
444	std::snprintf(config->buffer_pool_shmem_name,	62✔
445	kMaxBufferPoolShmemNameLength,
446	"%s", name.c_str());
447	}
448	if (yaml_conf["rpc_protocol"]) {	75✔
449	config->rpc_protocol = yaml_conf["rpc_protocol"].as<std::string>();	62✔
450	}
451	if (yaml_conf["rpc_domain"]) {	75✔
452	config->rpc_domain = yaml_conf["rpc_domain"].as<std::string>();	5✔
453	}
454	if (yaml_conf["rpc_port"]) {	75✔
455	config->rpc_port = yaml_conf["rpc_port"].as<int>();	62✔
456	}
457	if (yaml_conf["buffer_organizer_port"]) {	75✔
458	config->buffer_organizer_port =	62✔
459	yaml_conf["buffer_organizer_port"].as<int>();	62✔
460	}
461	if (yaml_conf["rpc_num_threads"]) {	75✔
462	config->rpc_num_threads =	62✔
463	yaml_conf["rpc_num_threads"].as<int>();	62✔
464	}
465	if (yaml_conf["default_placement_policy"]) {	75✔
466	std::string policy =	5✔
467	yaml_conf["default_placement_policy"].as<std::string>();	15✔
468
469	if (policy == "MinimizeIoTime") {	5✔
470	config->default_placement_policy =	5✔
471	api::PlacementPolicy::kMinimizeIoTime;
472	} else if (policy == "Random") {	×
473	config->default_placement_policy = api::PlacementPolicy::kRandom;	×
474	} else if (policy == "RoundRobin") {	×
475	config->default_placement_policy = api::PlacementPolicy::kRoundRobin;	×
476	} else {
477	LOG(FATAL) << "Unknown default_placement_policy: '" << policy << "'"	×
478	<< std::endl;	×
479	}
480	}
481	if (yaml_conf["is_shared_device"]) {	75✔
482	RequireNumDevices(config);	5✔
483	ParseArray<int>(yaml_conf["is_shared_device"],	5✔
484	"is_shared_device",
485	config->is_shared_device, config->num_devices);	5✔
486	}
487	if (yaml_conf["buffer_organizer_num_threads"]) {	75✔
488	config->bo_num_threads =	62✔
489	yaml_conf["buffer_organizer_num_threads"].as<int>();	62✔
490	}
491	if (yaml_conf["default_rr_split"]) {	75✔
492	config->default_rr_split = yaml_conf["default_rr_split"].as<int>();	5✔
493	}
494	if (yaml_conf["bo_num_threads"]) {	75✔
495	config->bo_num_threads = yaml_conf["bo_num_threads"].as<int>();	×
496	}
497	if (yaml_conf["bo_capacity_thresholds"]) {	75✔
498	RequireNumDevices(config);	5✔
499	f32 thresholds[kMaxDevices][2] = {0};	5✔
500	ParseMatrix<f32>(yaml_conf["bo_capacity_thresholds"],	5✔
501	"bo_capacity_thresholds",
502	reinterpret_cast<f32*>(thresholds),
503	kMaxDevices, 2);
504	for (int i = 0; i < config->num_devices; ++i) {	25✔
505	config->bo_capacity_thresholds[i].min = thresholds[i][0];	20✔
506	config->bo_capacity_thresholds[i].max = thresholds[i][1];	20✔
507	}
508	}
509	if (yaml_conf["path_exclusions"]) {	75✔
510	ParseVector<std::string>(	9✔
511	yaml_conf["path_exclusions"], config->path_exclusions);	9✔
512	}
513	if (yaml_conf["path_inclusions"]) {	75✔
514	ParseVector<std::string>(	5✔
515	yaml_conf["path_inclusions"], config->path_inclusions);	5✔
516	}
517	ParseHostNames(yaml_conf, config);	75✔
518	CheckConstraints(config);	75✔
519	}	75✔
520
521	/**
522	parse YAML configuration file
523	*/
524	void ParseConfig(Arena arena, const char path, Config *config) {	62✔
525	ScopedTemporaryMemory scratch(arena);	124✔
526	InitDefaultConfig(config);	62✔
527	LOG(INFO) << "ParseConfig-LoadFile" << std::endl;	62✔
528	YAML::Node yaml_conf = YAML::LoadFile(path);	248✔
529	LOG(INFO) << "ParseConfig-LoadComplete" << std::endl;	62✔
530	ParseConfigYAML(yaml_conf, config);	62✔
531	}	62✔
532
533	/**
534	parse configuration string
535	*/
536	void ParseConfigString(	13✔
537	Arena arena, const std::string &config_string, Config config) {
538	ScopedTemporaryMemory scratch(arena);	26✔
539	InitDefaultConfig(config);	13✔
540	YAML::Node yaml_conf = YAML::Load(config_string);	26✔
541	ParseConfigYAML(yaml_conf, config);	13✔
542	}	13✔
543
544	void InitConfig(hermes::Config config, const char config_file) {	71✔
545	const size_t kConfigMemorySize = KILOBYTES(16);	71✔
546	hermes::u8 config_memory[kConfigMemorySize];	71✔
547	if (config_file) {	71✔
548	hermes::Arena config_arena = {};	61✔
549	hermes::InitArena(&config_arena, kConfigMemorySize, config_memory);	61✔
550	hermes::ParseConfig(&config_arena, config_file, config);	61✔
551	} else {
552	InitDefaultConfig(config);	10✔
553	}
554	}	71✔
555
556	} // namespace hermes

HDFGroup / hermes / 4485341203

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous