4837525566

Build Type

github

Committed by GitHub

Commit Message

Merge f018521fc into 87672e106

Pull Request Pull Request #515: v1.0

Run Details

5502 of 5502 new or added lines in 117 files covered. (100.0%)

4997 of 7300 relevant lines covered (68.45%)

6194762.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.39

/src/api/bucket.cc

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Distributed under BSD 3-Clause license.                                   *
 * Copyright by The HDF Group.                                               *
 * Copyright by the Illinois Institute of Technology.                        *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of Hermes. The full Hermes copyright notice, including  *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the top directory. If you do not  *
 * have access to the file, you may request a copy from help@hdfgroup.org.   *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#include "bucket.h"
#include "data_placement_engine_factory.h"

namespace hermes::api {

using hermes::adapter::AdapterMode;

/**====================================
 * Bucket Operations
 * ===================================*/

/**
 * Either initialize or fetch the bucket.
 * */
Bucket::Bucket(const std::string &bkt_name,
               Context &ctx,
               size_t backend_size)
: mdm_(&HERMES->mdm_), bpm_(&HERMES->bpm_), name_(bkt_name) {
  std::vector<TraitId> traits;
  auto ret = mdm_->GlobalGetOrCreateTag(bkt_name, true, traits, backend_size);
  id_ = ret.first;
  did_create_ = ret.second;
}

/**
 * Either initialize or fetch the bucket.
 * */
Bucket::Bucket(TagId tag_id)
    : mdm_(&HERMES->mdm_), bpm_(&HERMES->bpm_) {
  name_ = mdm_->GlobalGetTagName(tag_id);
  id_ = tag_id;
  did_create_ = false;
}

/**
 * Attach a trait to the bucket
 * */
void Bucket::AttachTrait(TraitId trait_id) {
  HERMES->AttachTrait(id_, trait_id);
}

/**
 * Get the current size of the bucket
 * */
size_t Bucket::GetSize() {
  return mdm_->GlobalGetBucketSize(id_);
}

/**
   * Update the size of the bucket
   * Needed for the adapters for now.
   * */
void Bucket::SetSize(size_t new_size) {
  mdm_->GlobalSetBucketSize(id_, new_size);
}

/**
 * Lock a bucket
 * */
void Bucket::LockBucket(MdLockType type) {
  mdm_->GlobalLockBucket(id_, type);
}

/**
 * Unlock a bucket
 * */
void Bucket::UnlockBucket(MdLockType type) {
  mdm_->GlobalUnlockBucket(id_, type);
}

/**
 * Rename this bucket
 * */
void Bucket::Rename(const std::string &new_bkt_name) {
  mdm_->GlobalRenameTag(id_, new_bkt_name);
}

/**
   * Clears the buckets contents, but doesn't destroy its metadata
   * */
void Bucket::Clear() {
  mdm_->GlobalClearBucket(id_);
}

/**
 * Destroys this bucket along with all its contents.
 * */
void Bucket::Destroy() {
  mdm_->GlobalDestroyTag(id_);
}

/**====================================
 * Blob Operations
 * ===================================*/

/**
 * Get the id of a blob from the blob name
 * */
Status Bucket::GetBlobId(const std::string &blob_name,
                         BlobId &blob_id) {
  blob_id = mdm_->GlobalGetBlobId(GetId(), blob_name);
  return Status();
}

/**
 * Get the name of a blob from the blob id
 *
 * @param blob_id the blob_id
 * @return The Status of the operation
 * */
std::string Bucket::GetBlobName(const BlobId &blob_id) {
  return mdm_->GlobalGetBlobName(blob_id);
}

/**
 * Get the score of a blob from the blob id
 *
 * @param blob_id the blob_id
 * @return The Status of the operation
 * */
float Bucket::GetBlobScore(const BlobId &blob_id) {
  return mdm_->GlobalGetBlobScore(blob_id);
}


/**
 * Lock the bucket
 * */
bool Bucket::LockBlob(BlobId blob_id, MdLockType lock_type) {
  return mdm_->GlobalLockBlob(blob_id, lock_type);
}

/**
 * Unlock the bucket
 * */
bool Bucket::UnlockBlob(BlobId blob_id, MdLockType lock_type) {
  return mdm_->GlobalUnlockBlob(blob_id, lock_type);
}

/**
 * Put \a blob_id Blob into the bucket
 * */
Status Bucket::TryCreateBlob(const std::string &blob_name,
                             BlobId &blob_id,
                             Context &ctx) {
  std::pair<BlobId, bool> ret = mdm_->GlobalTryCreateBlob(id_, blob_name);
  blob_id = ret.first;
  if (ret.second) {
    mdm_->GlobalTagAddBlob(id_,
                           blob_id);
  }
  return Status();
}

/**
 * Label \a blob_id blob with \a tag_name TAG
 * */
Status Bucket::TagBlob(BlobId &blob_id,
                       TagId &tag_id) {
  mdm_->GlobalTagAddBlob(tag_id, blob_id);
  return mdm_->GlobalTagBlob(blob_id, tag_id);
}

/**
 * Put \a blob_id Blob into the bucket
 * */
Status Bucket::Put(std::string blob_name,
                   const Blob &blob,
                   BlobId &blob_id,
                   Context &ctx) {
  // Calculate placement
  auto dpe = DPEFactory::Get(ctx.policy);
  std::vector<size_t> blob_sizes(1, blob.size());
  std::vector<PlacementSchema> schemas;
  dpe->CalculatePlacement(blob_sizes, schemas, ctx);

  // Allocate buffers for the blob & enqueue placement
  for (auto &schema : schemas) {
    auto buffers = bpm_->GlobalAllocateAndSetBuffers(schema, blob);
    auto put_ret = mdm_->GlobalPutBlobMetadata(id_, blob_name,
                                               blob.size(), buffers,
                                               ctx.blob_score_);
    blob_id = std::get<0>(put_ret);
    bool did_create = std::get<1>(put_ret);
    if (did_create) {
      mdm_->GlobalTagAddBlob(id_, blob_id);
    }
  }

  // Update the local MDM I/O log
  mdm_->AddIoStat(id_, blob_id, blob.size(), IoType::kWrite);

  return Status();
}

/**
 * Get \a blob_id Blob from the bucket
 * */
Status Bucket::Get(BlobId blob_id, Blob &blob, Context &ctx) {
  std::vector<BufferInfo> buffers = mdm_->GlobalGetBlobBuffers(blob_id);
  blob = HERMES->borg_.GlobalReadBlobFromBuffers(buffers);
  // Update the local MDM I/O log
  mdm_->AddIoStat(id_, blob_id, blob.size(), IoType::kRead);
  return Status();
}

/**
 * Determine if the bucket contains \a blob_id BLOB
 * */
bool Bucket::ContainsBlob(const std::string &blob_name,
                          BlobId &blob_id) {
  GetBlobId(blob_name, blob_id);
  return !blob_id.IsNull();
}

/**
 * Determine if the bucket contains \a blob_id BLOB
 * */
bool Bucket::ContainsBlob(BlobId blob_id) {
  return mdm_->GlobalBlobHasTag(blob_id, id_);
}

/**
 * Rename \a blob_id blob to \a new_blob_name new name
 * */
void Bucket::RenameBlob(BlobId blob_id,
                        std::string new_blob_name,
                        Context &ctx) {
  mdm_->GlobalRenameBlob(id_, blob_id, new_blob_name);
}

/**
 * Delete \a blob_id blob
 * */
void Bucket::DestroyBlob(BlobId blob_id,
                         Context &ctx) {
  mdm_->GlobalTagRemoveBlob(id_, blob_id);
  mdm_->GlobalDestroyBlob(id_, blob_id);
}

/**
 * Get the set of blob IDs owned by the bucket
 * */
std::vector<BlobId> Bucket::GetContainedBlobIds() {
  return mdm_->GlobalGroupByTag(id_);
}

}  // namespace hermes::api

1	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2	* Distributed under BSD 3-Clause license. *
3	* Copyright by The HDF Group. *
4	* Copyright by the Illinois Institute of Technology. *
5	* All rights reserved. *
6	* *
7	* This file is part of Hermes. The full Hermes copyright notice, including *
8	* terms governing use, modification, and redistribution, is contained in *
9	* the COPYING file, which can be found at the top directory. If you do not *
10	* have access to the file, you may request a copy from help@hdfgroup.org. *
11	* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
12
13	#include "bucket.h"
14	#include "data_placement_engine_factory.h"
15
16	namespace hermes::api {
17
18	using hermes::adapter::AdapterMode;
19
20	/**====================================
21	* Bucket Operations
22	* ===================================*/
23
24	/**
25	* Either initialize or fetch the bucket.
26	* */
27	Bucket::Bucket(const std::string &bkt_name,	2,400✔
28	Context &ctx,
29	size_t backend_size)	2,400✔
30	: mdm_(&HERMES->mdm_), bpm_(&HERMES->bpm_), name_(bkt_name) {	2,400✔
31	std::vector<TraitId> traits;	2,400✔
32	auto ret = mdm_->GlobalGetOrCreateTag(bkt_name, true, traits, backend_size);	2,400✔
33	id_ = ret.first;	2,400✔
34	did_create_ = ret.second;	2,400✔
35	}	2,400✔
36
37	/**
38	* Either initialize or fetch the bucket.
39	* */
40	Bucket::Bucket(TagId tag_id)	3,109✔
41	: mdm_(&HERMES->mdm_), bpm_(&HERMES->bpm_) {	3,109✔
42	name_ = mdm_->GlobalGetTagName(tag_id);	3,109✔
43	id_ = tag_id;	3,109✔
44	did_create_ = false;	3,109✔
45	}	3,109✔
46
47	/**
48	* Attach a trait to the bucket
49	* */
50	void Bucket::AttachTrait(TraitId trait_id) {	1,313✔
51	HERMES->AttachTrait(id_, trait_id);	1,313✔
52	}	1,313✔
53
54	/**
55	* Get the current size of the bucket
56	* */
57	size_t Bucket::GetSize() {	73,134✔
58	return mdm_->GlobalGetBucketSize(id_);	73,134✔
59	}
60
61	/**
62	* Update the size of the bucket
63	* Needed for the adapters for now.
64	* */
65	void Bucket::SetSize(size_t new_size) {	5,075✔
66	mdm_->GlobalSetBucketSize(id_, new_size);	5,075✔
67	}	5,075✔
68
69	/**
70	* Lock a bucket
71	* */
72	void Bucket::LockBucket(MdLockType type) {	17,735✔
73	mdm_->GlobalLockBucket(id_, type);	17,735✔
74	}	17,735✔
75
76	/**
77	* Unlock a bucket
78	* */
79	void Bucket::UnlockBucket(MdLockType type) {	17,735✔
80	mdm_->GlobalUnlockBucket(id_, type);	17,735✔
81	}	17,735✔
82
83	/**
84	* Rename this bucket
85	* */
86	void Bucket::Rename(const std::string &new_bkt_name) {	1✔
87	mdm_->GlobalRenameTag(id_, new_bkt_name);	1✔
88	}	1✔
89
90	/**
91	* Clears the buckets contents, but doesn't destroy its metadata
92	* */
93	void Bucket::Clear() {	187✔
94	mdm_->GlobalClearBucket(id_);	187✔
95	}	187✔
96
97	/**
98	* Destroys this bucket along with all its contents.
99	* */
100	void Bucket::Destroy() {	221✔
101	mdm_->GlobalDestroyTag(id_);	221✔
102	}	221✔
103
104	/**====================================
105	* Blob Operations
106	* ===================================*/
107
108	/**
109	* Get the id of a blob from the blob name
110	* */
111	Status Bucket::GetBlobId(const std::string &blob_name,	47,450✔
112	BlobId &blob_id) {
113	blob_id = mdm_->GlobalGetBlobId(GetId(), blob_name);	47,450✔
114	return Status();	47,450✔
115	}
116
117	/**
118	* Get the name of a blob from the blob id
119	*
120	* @param blob_id the blob_id
121	* @return The Status of the operation
122	* */
123	std::string Bucket::GetBlobName(const BlobId &blob_id) {	×
124	return mdm_->GlobalGetBlobName(blob_id);	×
125	}
126
127	/**
128	* Get the score of a blob from the blob id
129	*
130	* @param blob_id the blob_id
131	* @return The Status of the operation
132	* */
133	float Bucket::GetBlobScore(const BlobId &blob_id) {	×
134	return mdm_->GlobalGetBlobScore(blob_id);	×
135	}
136
137
138	/**
139	* Lock the bucket
140	* */
141	bool Bucket::LockBlob(BlobId blob_id, MdLockType lock_type) {	45,143✔
142	return mdm_->GlobalLockBlob(blob_id, lock_type);	45,143✔
143	}
144
145	/**
146	* Unlock the bucket
147	* */
148	bool Bucket::UnlockBlob(BlobId blob_id, MdLockType lock_type) {	45,143✔
149	return mdm_->GlobalUnlockBlob(blob_id, lock_type);	45,143✔
150	}
151
152	/**
153	* Put \a blob_id Blob into the bucket
154	* */
155	Status Bucket::TryCreateBlob(const std::string &blob_name,	45,143✔
156	BlobId &blob_id,
157	Context &ctx) {
158	std::pair<BlobId, bool> ret = mdm_->GlobalTryCreateBlob(id_, blob_name);	45,143✔
159	blob_id = ret.first;	45,143✔
160	if (ret.second) {	45,143✔
161	mdm_->GlobalTagAddBlob(id_,	2,527✔
162	blob_id);
163	}
164	return Status();	45,143✔
165	}
166
167	/**
168	* Label \a blob_id blob with \a tag_name TAG
169	* */
170	Status Bucket::TagBlob(BlobId &blob_id,	100✔
171	TagId &tag_id) {
172	mdm_->GlobalTagAddBlob(tag_id, blob_id);	100✔
173	return mdm_->GlobalTagBlob(blob_id, tag_id);	100✔
174	}
175
176	/**
177	* Put \a blob_id Blob into the bucket
178	* */
179	Status Bucket::Put(std::string blob_name,	27,953✔
180	const Blob &blob,
181	BlobId &blob_id,
182	Context &ctx) {
183	// Calculate placement
184	auto dpe = DPEFactory::Get(ctx.policy);	27,953✔
185	std::vector<size_t> blob_sizes(1, blob.size());	27,953✔
186	std::vector<PlacementSchema> schemas;	55,906✔
187	dpe->CalculatePlacement(blob_sizes, schemas, ctx);	27,953✔
188
189	// Allocate buffers for the blob & enqueue placement
190	for (auto &schema : schemas) {	55,906✔
191	auto buffers = bpm_->GlobalAllocateAndSetBuffers(schema, blob);	55,906✔
192	auto put_ret = mdm_->GlobalPutBlobMetadata(id_, blob_name,	27,953✔
193	blob.size(), buffers,	×
194	ctx.blob_score_);	27,953✔
195	blob_id = std::get<0>(put_ret);	27,953✔
196	bool did_create = std::get<1>(put_ret);	27,953✔
197	if (did_create) {	27,953✔
198	mdm_->GlobalTagAddBlob(id_, blob_id);	2,423✔
199	}
200	}
201
202	// Update the local MDM I/O log
203	mdm_->AddIoStat(id_, blob_id, blob.size(), IoType::kWrite);	27,953✔
204
205	return Status();	55,906✔
206	}
207
208	/**
209	* Get \a blob_id Blob from the bucket
210	* */
211	Status Bucket::Get(BlobId blob_id, Blob &blob, Context &ctx) {	51,580✔
212	std::vector<BufferInfo> buffers = mdm_->GlobalGetBlobBuffers(blob_id);	51,580✔
213	blob = HERMES->borg_.GlobalReadBlobFromBuffers(buffers);	103,160✔
214	// Update the local MDM I/O log
215	mdm_->AddIoStat(id_, blob_id, blob.size(), IoType::kRead);	51,580✔
216	return Status();	100,485✔
217	}
218
219	/**
220	* Determine if the bucket contains \a blob_id BLOB
221	* */
222	bool Bucket::ContainsBlob(const std::string &blob_name,	45,143✔
223	BlobId &blob_id) {
224	GetBlobId(blob_name, blob_id);	45,143✔
225	return !blob_id.IsNull();	45,143✔
226	}
227
228	/**
229	* Determine if the bucket contains \a blob_id BLOB
230	* */
231	bool Bucket::ContainsBlob(BlobId blob_id) {	×
232	return mdm_->GlobalBlobHasTag(blob_id, id_);	×
233	}
234
235	/**
236	* Rename \a blob_id blob to \a new_blob_name new name
237	* */
238	void Bucket::RenameBlob(BlobId blob_id,	1✔
239	std::string new_blob_name,
240	Context &ctx) {
241	mdm_->GlobalRenameBlob(id_, blob_id, new_blob_name);	1✔
242	}	1✔
243
244	/**
245	* Delete \a blob_id blob
246	* */
247	void Bucket::DestroyBlob(BlobId blob_id,	1✔
248	Context &ctx) {
249	mdm_->GlobalTagRemoveBlob(id_, blob_id);	1✔
250	mdm_->GlobalDestroyBlob(id_, blob_id);	1✔
251	}	1✔
252
253	/**
254	* Get the set of blob IDs owned by the bucket
255	* */
256	std::vector<BlobId> Bucket::GetContainedBlobIds() {	1✔
257	return mdm_->GlobalGroupByTag(id_);	1✔
258	}
259
260	} // namespace hermes::api

HDFGroup / hermes / 4837525566

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous