4485341203

Build Type

push

github

Committed by Hyo-Kyung Lee

Commit Message

ci: update checkout and cache action versions

Run Details

6193 of 7556 relevant lines covered (81.96%)

646738.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.63

/src/utils.cc

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Distributed under BSD 3-Clause license.                                   *
 * Copyright by The HDF Group.                                               *
 * Copyright by the Illinois Institute of Technology.                        *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of Hermes. The full Hermes copyright notice, including  *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the top directory. If you do not  *
 * have access to the file, you may request a copy from help@hdfgroup.org.   *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#include "utils.h"

#include <iostream>
#include <vector>
#include <random>
#include <utility>

namespace hermes {

size_t RoundUpToMultiple(size_t val, size_t multiple) {
  if (multiple == 0) {
    return val;
  }

  size_t result = val;
  size_t remainder = val % multiple;

  if (remainder != 0) {
    result += multiple - remainder;
  }

  return result;
}

size_t RoundDownToMultiple(size_t val, size_t multiple) {
  if (multiple == 0) {
    return val;
  }

  size_t result = val;
  size_t remainder = val % multiple;
  result -= remainder;

  return result;
}

void InitDefaultConfig(Config *config) {
  config->num_devices = 4;
  config->num_targets = 4;
  assert(config->num_devices < kMaxDevices);

  for (int dev = 0; dev < config->num_devices; ++dev) {
    config->capacities[dev] = MEGABYTES(50);
    config->block_sizes[dev] = KILOBYTES(4);
    config->num_slabs[dev] = 4;

    config->slab_unit_sizes[dev][0] = 1;
    config->slab_unit_sizes[dev][1] = 4;
    config->slab_unit_sizes[dev][2] = 16;
    config->slab_unit_sizes[dev][3] = 32;

    config->desired_slab_percentages[dev][0] = 0.25f;
    config->desired_slab_percentages[dev][1] = 0.25f;
    config->desired_slab_percentages[dev][2] = 0.25f;
    config->desired_slab_percentages[dev][3] = 0.25f;
  }

  config->bandwidths[0] = 6000.0f;
  config->bandwidths[1] = 300.f;
  config->bandwidths[2] = 150.0f;
  config->bandwidths[3] = 70.0f;

  config->latencies[0] = 15.0f;
  config->latencies[1] = 250000;
  config->latencies[2] = 500000.0f;
  config->latencies[3] = 1000000.0f;

  // TODO(chogan): Express these in bytes instead of percentages to avoid
  // rounding errors?
  config->arena_percentages[kArenaType_BufferPool] = 0.85f;
  config->arena_percentages[kArenaType_MetaData] = 0.04f;
  config->arena_percentages[kArenaType_Transient] = 0.11f;

  config->mount_points[0] = "";
  config->mount_points[1] = "./";
  config->mount_points[2] = "./";
  config->mount_points[3] = "./";
  config->swap_mount = "./";

  config->num_buffer_organizer_retries = 3;

  config->rpc_server_host_file = "";
  config->rpc_server_base_name = "localhost";
  config->rpc_server_suffix = "";
  config->host_numbers = std::vector<std::string>();
  config->host_names.emplace_back("localhost");
  config->rpc_protocol = "ofi+sockets";
  config->rpc_domain = "";
  config->rpc_port = 8080;
  config->buffer_organizer_port = 8081;
  config->rpc_num_threads = 1;

  config->max_buckets_per_node = 16;
  config->max_vbuckets_per_node = 8;
  config->system_view_state_update_interval_ms = 100;

  const std::string buffer_pool_shmem_name = "/hermes_buffer_pool_";
  std::snprintf(config->buffer_pool_shmem_name,
                kMaxBufferPoolShmemNameLength,
                "%s", buffer_pool_shmem_name.c_str());

  config->default_placement_policy = api::PlacementPolicy::kMinimizeIoTime;

  config->is_shared_device[0] = 0;
  config->is_shared_device[1] = 0;
  config->is_shared_device[2] = 0;
  config->is_shared_device[3] = 0;

  config->bo_num_threads = 4;
  config->default_rr_split = false;

  for (int i = 0; i < config->num_devices; ++i) {
    config->bo_capacity_thresholds[i].min = 0.0f;
    config->bo_capacity_thresholds[i].max = 1.0f;
  }
}
/**
   print an error message for \a func function that failed
 */   
void FailedLibraryCall(std::string func) {
  int saved_errno = errno;
  LOG(FATAL) << func << " failed with error: "  << strerror(saved_errno)
             << std::endl;
}

namespace testing {

TargetViewState InitDeviceState(u64 total_target, bool homo_dist) {
  TargetViewState result = {};
  result.num_devices = total_target;
  std::vector<double> tgt_fraction;
  std::vector<double> tgt_homo_dist {0.25, 0.25, 0.25, 0.25};
  std::vector<double> tgt_heto_dist {0.1, 0.2, 0.3, 0.4};
  /** Use Megabytes */
  std::vector<i64> device_size {128, 1024, 4096, 16384};
  /** Use Megabytes/Sec */
  std::vector<double> device_bandwidth {8192, 3072, 550, 120};

  if (homo_dist)
    tgt_fraction = tgt_homo_dist;
  else
    tgt_fraction = tgt_heto_dist;

  std::vector<u64> tgt_num_per_type;
  u64 used_tgt {0};
  for (size_t i {0}; i < tgt_fraction.size()-1; ++i) {
    u64 used_tgt_tmp {static_cast<u64>(tgt_fraction[i] * total_target)};
    tgt_num_per_type.push_back(used_tgt_tmp);
    used_tgt += used_tgt_tmp;
  }
  tgt_num_per_type.push_back(total_target - used_tgt);

  using hermes::TargetID;
  std::vector<TargetID> targets = GetDefaultTargets(total_target);

  for (size_t i {0}; i < tgt_num_per_type.size(); ++i) {
    for (size_t j {0}; j < tgt_num_per_type[i]; ++j) {
      result.bandwidth.push_back(device_bandwidth[i]);

      result.bytes_available.push_back(MEGABYTES(device_size[i]));
      result.bytes_capacity.push_back(MEGABYTES(device_size[i]));
    }
  }

  return result;
}

u64 UpdateDeviceState(PlacementSchema &schema,
                      TargetViewState &node_state) {
  u64 result {0};
  for (auto [size, target] : schema) {
    result += size;
    node_state.bytes_available[target.bits.device_id] -= size;
  }
  return result;
}

void PrintNodeState(TargetViewState &node_state) {
  for (int i {0}; i < node_state.num_devices; ++i) {
    std::cout << "  capacity of device[" << i << "]: "
              << node_state.bytes_available[i] / MEGABYTES(1)
              << " MB\n" << std::flush;
    std::cout << "  available ratio of device["<< i << "]: "
              << static_cast<double>(node_state.bytes_available[i])/
                 node_state.bytes_capacity[i]
              << "\n\n" << std::flush;
  }
}

std::vector<TargetID> GetDefaultTargets(size_t n) {
  std::vector<TargetID> result(n);
  for (size_t i = 0; i < n; ++i) {
    TargetID id = {};
    id.bits.node_id = 1;
    id.bits.device_id = (DeviceID)i;
    id.bits.index = i;
    result[i] = id;
  }

  return result;
}

std::pair<size_t, size_t> GetBlobBound(BlobSizeRange blob_size_range) {
  if (blob_size_range == BlobSizeRange::kSmall)
    return {0, KILOBYTES(64)};
  else if (blob_size_range == BlobSizeRange::kMedium)
    return {KILOBYTES(64), MEGABYTES(1)};
  else if (blob_size_range == BlobSizeRange::kLarge)
    return {MEGABYTES(1), MEGABYTES(4)};
  else if (blob_size_range == BlobSizeRange::kXLarge)
    return {MEGABYTES(4), MEGABYTES(64)};
  else if (blob_size_range == BlobSizeRange::kHuge)
    return {GIGABYTES(1), GIGABYTES(1)};

  std::cout << "No specified blob range is found.\n"
            << "Use small blob range (0, 64KB].";
  return {0, KILOBYTES(64)};
}

std::vector<size_t> GenFixedTotalBlobSize(size_t total_size,
                                          BlobSizeRange range) {
  std::vector<size_t> result;
  size_t used_size {};
  size_t size {};
  std::random_device dev;
  std::mt19937 rng(dev());

  std::pair bound = GetBlobBound(range);
  size_t lo_bound  = bound.first;
  size_t hi_bound = bound.second;

  while (used_size < total_size) {
    std::vector<hermes::api::Blob> input_blobs;
    if (total_size - used_size > hi_bound) {
      std::uniform_int_distribution<std::mt19937::result_type>
        distribution(lo_bound, hi_bound);
      size = distribution(rng);
      used_size += size;
    } else {
      size = total_size - used_size;
      used_size = total_size;
    }
    result.push_back(size);
  }
  return result;
}

}  // namespace testing
}  // namespace hermes

1	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2	* Distributed under BSD 3-Clause license. *
3	* Copyright by The HDF Group. *
4	* Copyright by the Illinois Institute of Technology. *
5	* All rights reserved. *
6	* *
7	* This file is part of Hermes. The full Hermes copyright notice, including *
8	* terms governing use, modification, and redistribution, is contained in *
9	* the COPYING file, which can be found at the top directory. If you do not *
10	* have access to the file, you may request a copy from help@hdfgroup.org. *
11	* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
12
13	#include "utils.h"
14
15	#include <iostream>
16	#include <vector>
17	#include <random>
18	#include <utility>
19
20	namespace hermes {
21
22	size_t RoundUpToMultiple(size_t val, size_t multiple) {	42✔
23	if (multiple == 0) {	42✔
24	return val;	×
25	}
26
27	size_t result = val;	42✔
28	size_t remainder = val % multiple;	42✔
29
30	if (remainder != 0) {	42✔
31	result += multiple - remainder;	42✔
32	}
33
34	return result;	42✔
35	}
36
37	size_t RoundDownToMultiple(size_t val, size_t multiple) {	59✔
38	if (multiple == 0) {	59✔
39	return val;	×
40	}
41
42	size_t result = val;	59✔
43	size_t remainder = val % multiple;	59✔
44	result -= remainder;	59✔
45
46	return result;	59✔
47	}
48
49	void InitDefaultConfig(Config *config) {	98✔
50	config->num_devices = 4;	98✔
51	config->num_targets = 4;	98✔
52	assert(config->num_devices < kMaxDevices);	98✔
53
54	for (int dev = 0; dev < config->num_devices; ++dev) {	490✔
55	config->capacities[dev] = MEGABYTES(50);	392✔
56	config->block_sizes[dev] = KILOBYTES(4);	392✔
57	config->num_slabs[dev] = 4;	392✔
58
59	config->slab_unit_sizes[dev][0] = 1;	392✔
60	config->slab_unit_sizes[dev][1] = 4;	392✔
61	config->slab_unit_sizes[dev][2] = 16;	392✔
62	config->slab_unit_sizes[dev][3] = 32;	392✔
63
64	config->desired_slab_percentages[dev][0] = 0.25f;	392✔
65	config->desired_slab_percentages[dev][1] = 0.25f;	392✔
66	config->desired_slab_percentages[dev][2] = 0.25f;	392✔
67	config->desired_slab_percentages[dev][3] = 0.25f;	392✔
68	}
69
70	config->bandwidths[0] = 6000.0f;	98✔
71	config->bandwidths[1] = 300.f;	98✔
72	config->bandwidths[2] = 150.0f;	98✔
73	config->bandwidths[3] = 70.0f;	98✔
74
75	config->latencies[0] = 15.0f;	98✔
76	config->latencies[1] = 250000;	98✔
77	config->latencies[2] = 500000.0f;	98✔
78	config->latencies[3] = 1000000.0f;	98✔
79
80	// TODO(chogan): Express these in bytes instead of percentages to avoid
81	// rounding errors?
82	config->arena_percentages[kArenaType_BufferPool] = 0.85f;	98✔
83	config->arena_percentages[kArenaType_MetaData] = 0.04f;	98✔
84	config->arena_percentages[kArenaType_Transient] = 0.11f;	98✔
85
86	config->mount_points[0] = "";	98✔
87	config->mount_points[1] = "./";	98✔
88	config->mount_points[2] = "./";	98✔
89	config->mount_points[3] = "./";	98✔
90	config->swap_mount = "./";	98✔
91
92	config->num_buffer_organizer_retries = 3;	98✔
93
94	config->rpc_server_host_file = "";	98✔
95	config->rpc_server_base_name = "localhost";	98✔
96	config->rpc_server_suffix = "";	98✔
97	config->host_numbers = std::vector<std::string>();	98✔
98	config->host_names.emplace_back("localhost");	98✔
99	config->rpc_protocol = "ofi+sockets";	98✔
100	config->rpc_domain = "";	98✔
101	config->rpc_port = 8080;	98✔
102	config->buffer_organizer_port = 8081;	98✔
103	config->rpc_num_threads = 1;	98✔
104
105	config->max_buckets_per_node = 16;	98✔
106	config->max_vbuckets_per_node = 8;	98✔
107	config->system_view_state_update_interval_ms = 100;	98✔
108
109	const std::string buffer_pool_shmem_name = "/hermes_buffer_pool_";	294✔
110	std::snprintf(config->buffer_pool_shmem_name,	98✔
111	kMaxBufferPoolShmemNameLength,
112	"%s", buffer_pool_shmem_name.c_str());
113
114	config->default_placement_policy = api::PlacementPolicy::kMinimizeIoTime;	98✔
115
116	config->is_shared_device[0] = 0;	98✔
117	config->is_shared_device[1] = 0;	98✔
118	config->is_shared_device[2] = 0;	98✔
119	config->is_shared_device[3] = 0;	98✔
120
121	config->bo_num_threads = 4;	98✔
122	config->default_rr_split = false;	98✔
123
124	for (int i = 0; i < config->num_devices; ++i) {	490✔
125	config->bo_capacity_thresholds[i].min = 0.0f;	392✔
126	config->bo_capacity_thresholds[i].max = 1.0f;	392✔
127	}
128	}	98✔
129	/**
130	print an error message for \a func function that failed
131	*/
132	void FailedLibraryCall(std::string func) {	×
133	int saved_errno = errno;	×
134	LOG(FATAL) << func << " failed with error: " << strerror(saved_errno)	×
135	<< std::endl;	×
136	}
137
138	namespace testing {
139
140	TargetViewState InitDeviceState(u64 total_target, bool homo_dist) {	6✔
141	TargetViewState result = {};	6✔
142	result.num_devices = total_target;	6✔
143	std::vector<double> tgt_fraction;	12✔
144	std::vector<double> tgt_homo_dist {0.25, 0.25, 0.25, 0.25};	18✔
145	std::vector<double> tgt_heto_dist {0.1, 0.2, 0.3, 0.4};	18✔
146	/** Use Megabytes */
147	std::vector<i64> device_size {128, 1024, 4096, 16384};	18✔
148	/** Use Megabytes/Sec */
149	std::vector<double> device_bandwidth {8192, 3072, 550, 120};	18✔
150
151	if (homo_dist)	6✔
152	tgt_fraction = tgt_homo_dist;	6✔
153	else
154	tgt_fraction = tgt_heto_dist;	×
155
156	std::vector<u64> tgt_num_per_type;	12✔
157	u64 used_tgt {0};	6✔
158	for (size_t i {0}; i < tgt_fraction.size()-1; ++i) {	24✔
159	u64 used_tgt_tmp {static_cast<u64>(tgt_fraction[i] * total_target)};	18✔
160	tgt_num_per_type.push_back(used_tgt_tmp);	18✔
161	used_tgt += used_tgt_tmp;	18✔
162	}
163	tgt_num_per_type.push_back(total_target - used_tgt);	6✔
164
165	using hermes::TargetID;
166	std::vector<TargetID> targets = GetDefaultTargets(total_target);	12✔
167
168	for (size_t i {0}; i < tgt_num_per_type.size(); ++i) {	30✔
169	for (size_t j {0}; j < tgt_num_per_type[i]; ++j) {	48✔
170	result.bandwidth.push_back(device_bandwidth[i]);	24✔
171
172	result.bytes_available.push_back(MEGABYTES(device_size[i]));	24✔
173	result.bytes_capacity.push_back(MEGABYTES(device_size[i]));	24✔
174	}
175	}
176
177	return result;	12✔
178	}
179
180	u64 UpdateDeviceState(PlacementSchema &schema,	14✔
181	TargetViewState &node_state) {
182	u64 result {0};	14✔
183	for (auto [size, target] : schema) {	41✔
184	result += size;	27✔
185	node_state.bytes_available[target.bits.device_id] -= size;	27✔
186	}
187	return result;	14✔
188	}
189
190	void PrintNodeState(TargetViewState &node_state) {	20✔
191	for (int i {0}; i < node_state.num_devices; ++i) {	100✔
192	std::cout << " capacity of device[" << i << "]: "	80✔
193	<< node_state.bytes_available[i] / MEGABYTES(1)	80✔
194	<< " MB\n" << std::flush;	80✔
195	std::cout << " available ratio of device["<< i << "]: "	80✔
196	<< static_cast<double>(node_state.bytes_available[i])/	80✔
197	node_state.bytes_capacity[i]	80✔
198	<< "\n\n" << std::flush;	80✔
199	}
200	}	20✔
201
202	std::vector<TargetID> GetDefaultTargets(size_t n) {	20✔
203	std::vector<TargetID> result(n);	40✔
204	for (size_t i = 0; i < n; ++i) {	100✔
205	TargetID id = {};	80✔
206	id.bits.node_id = 1;	80✔
207	id.bits.device_id = (DeviceID)i;	80✔
208	id.bits.index = i;	80✔
209	result[i] = id;	80✔
210	}
211
212	return result;	20✔
213	}
214
215	std::pair<size_t, size_t> GetBlobBound(BlobSizeRange blob_size_range) {	×
216	if (blob_size_range == BlobSizeRange::kSmall)	×
217	return {0, KILOBYTES(64)};	×
218	else if (blob_size_range == BlobSizeRange::kMedium)	×
219	return {KILOBYTES(64), MEGABYTES(1)};	×
220	else if (blob_size_range == BlobSizeRange::kLarge)	×
221	return {MEGABYTES(1), MEGABYTES(4)};	×
222	else if (blob_size_range == BlobSizeRange::kXLarge)	×
223	return {MEGABYTES(4), MEGABYTES(64)};	×
224	else if (blob_size_range == BlobSizeRange::kHuge)	×
225	return {GIGABYTES(1), GIGABYTES(1)};	×
226
227	std::cout << "No specified blob range is found.\n"
228	<< "Use small blob range (0, 64KB].";	×
229	return {0, KILOBYTES(64)};	×
230	}
231
232	std::vector<size_t> GenFixedTotalBlobSize(size_t total_size,	×
233	BlobSizeRange range) {
234	std::vector<size_t> result;	×
235	size_t used_size {};	×
236	size_t size {};	×
237	std::random_device dev;	×
238	std::mt19937 rng(dev());	×
239
240	std::pair bound = GetBlobBound(range);	×
241	size_t lo_bound = bound.first;	×
242	size_t hi_bound = bound.second;	×
243
244	while (used_size < total_size) {	×
245	std::vector<hermes::api::Blob> input_blobs;	×
246	if (total_size - used_size > hi_bound) {	×
247	std::uniform_int_distribution<std::mt19937::result_type>
248	distribution(lo_bound, hi_bound);	×
249	size = distribution(rng);	×
250	used_size += size;	×
251	} else {
252	size = total_size - used_size;	×
253	used_size = total_size;	×
254	}
255	result.push_back(size);	×
256	}
257	return result;	×
258	}
259
260	} // namespace testing
261	} // namespace hermes

HDFGroup / hermes / 4485341203

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous