6546858227

Committed 17 Oct 2023 12:02PM UTC coverage: 91.71% (+0.002%) from 91.708%

Build # 6546858227

Build Type

push

github

Committed by

randombit

Commit Message

Merge GH #3760 Move constant time memory comparisons to ct_utils.h

Run Details

80095 of 87335 relevant lines covered (91.71%)

8508512.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.46

/src/cli/zfec.cpp

/*
* (C) 2021 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include "cli.h"

#if defined(BOTAN_HAS_ZFEC) && defined(BOTAN_HAS_SHA2_64)
   #include <botan/hash.h>
   #include <botan/mem_ops.h>
   #include <botan/zfec.h>
   #include <botan/internal/loadstor.h>
   #include <fstream>
   #include <sstream>
#endif

namespace Botan_CLI {

#if defined(BOTAN_HAS_ZFEC) && defined(BOTAN_HAS_SHA2_64)

static const uint32_t FEC_MAGIC = 0xFECC0DEC;
const char* const FEC_SHARE_HASH = "SHA-512-256";

class FEC_Share final {
   public:
      FEC_Share() : m_share(0), m_k(0), m_n(0), m_padding(0), m_bits() {}

      FEC_Share(size_t share, size_t k, size_t n, size_t padding, const uint8_t bits[], size_t len) :
            m_share(share), m_k(k), m_n(n), m_padding(padding), m_bits(bits, bits + len) {}

      size_t share_id() const { return m_share; }

      size_t k() const { return m_k; }

      size_t n() const { return m_n; }

      size_t padding() const { return m_padding; }

      size_t share_size() const { return m_bits.size(); }

      const uint8_t* share_data() const { return m_bits.data(); }

      static FEC_Share deserialize(const uint8_t bits[], size_t len, Botan::HashFunction& hash) {
         const size_t hash_len = hash.output_length();

         if(len < FEC_SHARE_HEADER_LEN + hash_len) {
            throw CLI_Error("FEC share is too short to be valid");
         }

         if(Botan::load_be<uint32_t>(bits, 0) != FEC_MAGIC) {
            throw CLI_Error("FEC share does not have expected magic bytes");
         }

         // verify that reserved bytes are zero
         for(size_t i = 8; i != 12; ++i) {
            if(bits[i] != 0) {
               throw CLI_Error("FEC share has reserved header bytes set");
            }
         }

         size_t share_id = bits[4];
         size_t k = bits[5];
         size_t n = bits[6];
         size_t padding = bits[7];

         if(share_id >= n || k >= n || padding >= k) {
            throw CLI_Error("FEC share has invalid k/n/padding fields");
         }

         hash.update(bits, len - hash_len);
         auto share_hash = hash.final();

         const bool digest_ok = Botan::constant_time_compare(share_hash.data(), &bits[len - hash_len], hash_len);

         if(!digest_ok) {
            throw CLI_Error("FEC share has invalid hash");
         }

         return FEC_Share(
            share_id, k, n, padding, bits + FEC_SHARE_HEADER_LEN, len - (FEC_SHARE_HEADER_LEN + hash_len));
      }

      void serialize_to(Botan::HashFunction& hash, std::ostream& out) const {
         uint8_t header[FEC_SHARE_HEADER_LEN] = {0};

         Botan::store_be(FEC_MAGIC, header);
         header[4] = static_cast<uint8_t>(m_share);
         header[5] = static_cast<uint8_t>(m_k);
         header[6] = static_cast<uint8_t>(m_n);
         header[7] = static_cast<uint8_t>(m_padding);
         // bytes 8..12 left as zero/reserved

         out.write(reinterpret_cast<const char*>(header), sizeof(header));
         out.write(reinterpret_cast<const char*>(m_bits.data()), m_bits.size());

         hash.update(header, sizeof(header));
         hash.update(m_bits);
         auto digest = hash.final();

         out.write(reinterpret_cast<const char*>(digest.data()), digest.size());
      }

   private:
      static const size_t FEC_SHARE_HEADER_LEN = 12;

      size_t m_share;
      size_t m_k;
      size_t m_n;
      size_t m_padding;
      std::vector<uint8_t> m_bits;
};

class FEC_Encode final : public Command {
   public:
      FEC_Encode() : Command("fec_encode --suffix=fec --prefix= --output-dir= k n input") {}

      std::string group() const override { return "fec"; }

      std::string description() const override { return "Forward error encode a file"; }

      void go() override {
         const size_t k = get_arg_sz("k");
         const size_t n = get_arg_sz("n");

         const std::string suffix = get_arg("suffix");
         const std::string prefix = get_arg("prefix");
         const std::string input = get_arg("input");
         const std::string output_dir = get_arg("output-dir");

         Botan::ZFEC fec(k, n);  // checks k/n for validity

         auto hash = Botan::HashFunction::create_or_throw(FEC_SHARE_HASH);

         auto input_data = slurp_file(get_arg("input"));

         // append a hash of the input
         hash->update(input_data);
         const auto hash_of_input = hash->final();
         input_data.insert(input_data.end(), hash_of_input.begin(), hash_of_input.end());

         // add padding 0x00 bytes as needed to round up to k multiple
         size_t padding = 0;
         while(input_data.size() % k != 0) {
            padding += 1;
            input_data.push_back(0x00);
         }

         auto encoder_fn = [&](size_t share, const uint8_t bits[], size_t len) {
            std::ostringstream output_fsname;

            if(!output_dir.empty()) {
               output_fsname << output_dir << "/";
            }

            if(!prefix.empty()) {
               output_fsname << prefix;
            } else {
               output_fsname << input;
            }

            output_fsname << "." << (share + 1) << "_" << n;

            if(!suffix.empty()) {
               output_fsname << "." << suffix;
            }

            std::ofstream output(output_fsname.str(), std::ios::binary);

            FEC_Share fec_share(share, k, n, padding, bits, len);
            fec_share.serialize_to(*hash, output);
         };

         fec.encode(input_data.data(), input_data.size(), encoder_fn);
      }
};

BOTAN_REGISTER_COMMAND("fec_encode", FEC_Encode);

class FEC_Decode final : public Command {
   public:
      FEC_Decode() : Command("fec_decode *shares") {}

      std::string group() const override { return "fec"; }

      std::string description() const override { return "Recover data from FEC shares"; }

      void go() override {
         auto hash = Botan::HashFunction::create_or_throw(FEC_SHARE_HASH);
         const size_t hash_len = hash->output_length();

         std::vector<FEC_Share> shares;

         for(const auto& share_fsname : get_arg_list("shares")) {
            const auto share_bits = slurp_file(share_fsname);

            try {
               auto share = FEC_Share::deserialize(share_bits.data(), share_bits.size(), *hash);
               shares.push_back(share);
            } catch(std::exception& e) {
               error_output() << "Ignoring invalid share '" << share_fsname << "': " << e.what() << "\n";
            }
         }

         if(shares.empty()) {
            error_output() << "Must provide a list of at least k shares\n";
            this->set_return_code(1);
            return;
         }

         size_t k = 0;
         size_t n = 0;
         size_t padding = 0;
         size_t share_size = 0;

         for(const auto& share : shares) {
            if(k == 0 && n == 0 && padding == 0) {
               k = share.k();
               n = share.n();
               padding = share.padding();
               share_size = share.share_size();
            } else {
               if(share.k() != k || share.n() != n || share.padding() != padding || share.share_size() != share_size) {
                  error_output() << "Shares have mismatched k/n/padding/size values\n";
                  this->set_return_code(2);
                  return;
               }
            }
         }

         if(shares.size() < k) {
            error_output() << "At least " << k << " shares are required for recovery\n";
            this->set_return_code(2);
            return;
         }

         Botan::ZFEC fec(k, n);

         std::vector<uint8_t> decoded(share_size * k);

         auto decoder_fn = [&](size_t share, const uint8_t bits[], size_t len) {
            std::memcpy(&decoded[share * share_size], bits, len);
         };

         std::map<size_t, const uint8_t*> share_ptrs;

         for(auto& share : shares) {
            share_ptrs[share.share_id()] = share.share_data();
         }

         fec.decode_shares(share_ptrs, share_size, decoder_fn);

         auto decoded_digest = hash->process(decoded.data(), decoded.size() - (hash_len + padding));

         if(!Botan::constant_time_compare(
               decoded_digest.data(), &decoded[decoded.size() - (hash_len + padding)], hash_len)) {
            throw CLI_Error("Recovered data failed digest check");
         }

         for(size_t i = 0; i != padding; ++i) {
            if(decoded[decoded.size() - padding + i] != 0) {
               throw CLI_Error("Recovered data had non-zero padding bytes");
            }
         }

         output().write(reinterpret_cast<const char*>(decoded.data()), decoded.size() - (hash_len + padding));
      }
};

BOTAN_REGISTER_COMMAND("fec_decode", FEC_Decode);

class FEC_Info final : public Command {
   public:
      FEC_Info() : Command("fec_info share") {}

      std::string group() const override { return "fec"; }

      std::string description() const override { return "Display information about a FEC share"; }

      void go() override {
         auto hash = Botan::HashFunction::create_or_throw(FEC_SHARE_HASH);

         const std::string share_fsname = get_arg("share");
         const auto share_bits = slurp_file(share_fsname);

         try {
            auto share = FEC_Share::deserialize(share_bits.data(), share_bits.size(), *hash);
            output() << "FEC share " << share.share_id() + 1 << "/" << share.n() << " with " << share.k()
                     << " needed for recovery\n";
         } catch(std::exception& e) {
            error_output() << "Invalid share '" << share_fsname << "': " << e.what() << "\n";
         }
      }
};

BOTAN_REGISTER_COMMAND("fec_info", FEC_Info);

#endif

}  // namespace Botan_CLI

1	/*
2	* (C) 2021 Jack Lloyd
3	*
4	* Botan is released under the Simplified BSD License (see license.txt)
5	*/
6
7	#include "cli.h"
8
9	#if defined(BOTAN_HAS_ZFEC) && defined(BOTAN_HAS_SHA2_64)
10	#include <botan/hash.h>
11	#include <botan/mem_ops.h>
12	#include <botan/zfec.h>
13	#include <botan/internal/loadstor.h>
14	#include <fstream>
15	#include <sstream>
16	#endif
17
18	namespace Botan_CLI {
19
20	#if defined(BOTAN_HAS_ZFEC) && defined(BOTAN_HAS_SHA2_64)
21
22	static const uint32_t FEC_MAGIC = 0xFECC0DEC;
23	const char* const FEC_SHARE_HASH = "SHA-512-256";
24
25	class FEC_Share final {	34✔
26	public:
27	FEC_Share() : m_share(0), m_k(0), m_n(0), m_padding(0), m_bits() {}
28
29	FEC_Share(size_t share, size_t k, size_t n, size_t padding, const uint8_t bits[], size_t len) :	15✔
30	m_share(share), m_k(k), m_n(n), m_padding(padding), m_bits(bits, bits + len) {}	30✔
31
32	size_t share_id() const { return m_share; }	3✔
33
34	size_t k() const { return m_k; }	10✔
35
36	size_t n() const { return m_n; }	10✔
37
38	size_t padding() const { return m_padding; }	5✔
39
40	size_t share_size() const { return m_bits.size(); }	3✔
41
42	const uint8_t* share_data() const { return m_bits.data(); }	3✔
43
44	static FEC_Share deserialize(const uint8_t bits[], size_t len, Botan::HashFunction& hash) {	10✔
45	const size_t hash_len = hash.output_length();	10✔
46
47	if(len < FEC_SHARE_HEADER_LEN + hash_len) {	10✔
48	throw CLI_Error("FEC share is too short to be valid");	×
49	}
50
51	if(Botan::load_be<uint32_t>(bits, 0) != FEC_MAGIC) {	10✔
52	throw CLI_Error("FEC share does not have expected magic bytes");	×
53	}
54
55	// verify that reserved bytes are zero
56	for(size_t i = 8; i != 12; ++i) {	50✔
57	if(bits[i] != 0) {	40✔
58	throw CLI_Error("FEC share has reserved header bytes set");	×
59	}
60	}
61
62	size_t share_id = bits[4];	10✔
63	size_t k = bits[5];	10✔
64	size_t n = bits[6];	10✔
65	size_t padding = bits[7];	10✔
66
67	if(share_id >= n \|\| k >= n \|\| padding >= k) {	10✔
68	throw CLI_Error("FEC share has invalid k/n/padding fields");	×
69	}
70
71	hash.update(bits, len - hash_len);	10✔
72	auto share_hash = hash.final();	10✔
73
74	const bool digest_ok = Botan::constant_time_compare(share_hash.data(), &bits[len - hash_len], hash_len);	10✔
75
76	if(!digest_ok) {	10✔
77	throw CLI_Error("FEC share has invalid hash");	×
78	}
79
80	return FEC_Share(	10✔
81	share_id, k, n, padding, bits + FEC_SHARE_HEADER_LEN, len - (FEC_SHARE_HEADER_LEN + hash_len));	10✔
82	}	10✔
83
84	void serialize_to(Botan::HashFunction& hash, std::ostream& out) const {	5✔
85	uint8_t header[FEC_SHARE_HEADER_LEN] = {0};	5✔
86
87	Botan::store_be(FEC_MAGIC, header);	5✔
88	header[4] = static_cast<uint8_t>(m_share);	5✔
89	header[5] = static_cast<uint8_t>(m_k);	5✔
90	header[6] = static_cast<uint8_t>(m_n);	5✔
91	header[7] = static_cast<uint8_t>(m_padding);	5✔
92	// bytes 8..12 left as zero/reserved
93
94	out.write(reinterpret_cast<const char*>(header), sizeof(header));	5✔
95	out.write(reinterpret_cast<const char*>(m_bits.data()), m_bits.size());	5✔
96
97	hash.update(header, sizeof(header));	5✔
98	hash.update(m_bits);	5✔
99	auto digest = hash.final();	5✔
100
101	out.write(reinterpret_cast<const char*>(digest.data()), digest.size());	5✔
102	}	5✔
103
104	private:
105	static const size_t FEC_SHARE_HEADER_LEN = 12;
106
107	size_t m_share;
108	size_t m_k;
109	size_t m_n;
110	size_t m_padding;
111	std::vector<uint8_t> m_bits;
112	};
113
114	class FEC_Encode final : public Command {
115	public:
116	FEC_Encode() : Command("fec_encode --suffix=fec --prefix= --output-dir= k n input") {}	4✔
117
118	std::string group() const override { return "fec"; }	1✔
119
120	std::string description() const override { return "Forward error encode a file"; }	1✔
121
122	void go() override {	1✔
123	const size_t k = get_arg_sz("k");	1✔
124	const size_t n = get_arg_sz("n");	1✔
125
126	const std::string suffix = get_arg("suffix");	1✔
127	const std::string prefix = get_arg("prefix");	1✔
128	const std::string input = get_arg("input");	1✔
129	const std::string output_dir = get_arg("output-dir");	1✔
130
131	Botan::ZFEC fec(k, n); // checks k/n for validity	1✔
132
133	auto hash = Botan::HashFunction::create_or_throw(FEC_SHARE_HASH);	1✔
134
135	auto input_data = slurp_file(get_arg("input"));	2✔
136
137	// append a hash of the input
138	hash->update(input_data);	1✔
139	const auto hash_of_input = hash->final();	1✔
140	input_data.insert(input_data.end(), hash_of_input.begin(), hash_of_input.end());	1✔
141
142	// add padding 0x00 bytes as needed to round up to k multiple
143	size_t padding = 0;	1✔
144	while(input_data.size() % k != 0) {	2✔
145	padding += 1;	1✔
146	input_data.push_back(0x00);	1✔
147	}
148
149	auto encoder_fn = [&](size_t share, const uint8_t bits[], size_t len) {	6✔
150	std::ostringstream output_fsname;	5✔
151
152	if(!output_dir.empty()) {	5✔
153	output_fsname << output_dir << "/";	5✔
154	}
155
156	if(!prefix.empty()) {	5✔
157	output_fsname << prefix;	5✔
158	} else {
159	output_fsname << input;	×
160	}
161
162	output_fsname << "." << (share + 1) << "_" << n;	5✔
163
164	if(!suffix.empty()) {	5✔
165	output_fsname << "." << suffix;	5✔
166	}
167
168	std::ofstream output(output_fsname.str(), std::ios::binary);	5✔
169
170	FEC_Share fec_share(share, k, n, padding, bits, len);	5✔
171	fec_share.serialize_to(*hash, output);	5✔
172	};	5✔
173
174	fec.encode(input_data.data(), input_data.size(), encoder_fn);	2✔
175	}	6✔
176	};
177
178	BOTAN_REGISTER_COMMAND("fec_encode", FEC_Encode);	2✔
179
180	class FEC_Decode final : public Command {
181	public:
182	FEC_Decode() : Command("fec_decode *shares") {}	6✔
183
184	std::string group() const override { return "fec"; }	1✔
185
186	std::string description() const override { return "Recover data from FEC shares"; }	1✔
187
188	void go() override {	2✔
189	auto hash = Botan::HashFunction::create_or_throw(FEC_SHARE_HASH);	2✔
190	const size_t hash_len = hash->output_length();	2✔
191
192	std::vector<FEC_Share> shares;	2✔
193
194	for(const auto& share_fsname : get_arg_list("shares")) {	7✔
195	const auto share_bits = slurp_file(share_fsname);	5✔
196
197	try {	5✔
198	auto share = FEC_Share::deserialize(share_bits.data(), share_bits.size(), *hash);	5✔
199	shares.push_back(share);	5✔
200	} catch(std::exception& e) {	5✔
201	error_output() << "Ignoring invalid share '" << share_fsname << "': " << e.what() << "\n";	×
202	}	×
203	}	7✔
204
205	if(shares.empty()) {	2✔
206	error_output() << "Must provide a list of at least k shares\n";	×
207	this->set_return_code(1);	×
208	return;	×
209	}
210
211	size_t k = 0;	2✔
212	size_t n = 0;	2✔
213	size_t padding = 0;	2✔
214	size_t share_size = 0;	2✔
215
216	for(const auto& share : shares) {	7✔
217	if(k == 0 && n == 0 && padding == 0) {	5✔
218	k = share.k();	2✔
219	n = share.n();	2✔
220	padding = share.padding();	2✔
221	share_size = share.share_size();	2✔
222	} else {
223	if(share.k() != k \|\| share.n() != n \|\| share.padding() != padding \|\| share.share_size() != share_size) {	3✔
224	error_output() << "Shares have mismatched k/n/padding/size values\n";	×
225	this->set_return_code(2);	×
226	return;	×
227	}
228	}
229	}
230
231	if(shares.size() < k) {	2✔
232	error_output() << "At least " << k << " shares are required for recovery\n";	1✔
233	this->set_return_code(2);	1✔
234	return;	1✔
235	}
236
237	Botan::ZFEC fec(k, n);	1✔
238
239	std::vector<uint8_t> decoded(share_size * k);	1✔
240
241	auto decoder_fn = [&](size_t share, const uint8_t bits[], size_t len) {	4✔
242	std::memcpy(&decoded[share * share_size], bits, len);	3✔
243	};	1✔
244
245	std::map<size_t, const uint8_t*> share_ptrs;	1✔
246
247	for(auto& share : shares) {	4✔
248	share_ptrs[share.share_id()] = share.share_data();	3✔
249	}
250
251	fec.decode_shares(share_ptrs, share_size, decoder_fn);	1✔
252
253	auto decoded_digest = hash->process(decoded.data(), decoded.size() - (hash_len + padding));	1✔
254
255	if(!Botan::constant_time_compare(	1✔
256	decoded_digest.data(), &decoded[decoded.size() - (hash_len + padding)], hash_len)) {	1✔
257	throw CLI_Error("Recovered data failed digest check");	×
258	}
259
260	for(size_t i = 0; i != padding; ++i) {	2✔
261	if(decoded[decoded.size() - padding + i] != 0) {	1✔
262	throw CLI_Error("Recovered data had non-zero padding bytes");	×
263	}
264	}
265
266	output().write(reinterpret_cast<const char*>(decoded.data()), decoded.size() - (hash_len + padding));	1✔
267	}	6✔
268	};
269
270	BOTAN_REGISTER_COMMAND("fec_decode", FEC_Decode);	3✔
271
272	class FEC_Info final : public Command {
273	public:
274	FEC_Info() : Command("fec_info share") {}	12✔
275
276	std::string group() const override { return "fec"; }	1✔
277
278	std::string description() const override { return "Display information about a FEC share"; }	1✔
279
280	void go() override {	5✔
281	auto hash = Botan::HashFunction::create_or_throw(FEC_SHARE_HASH);	5✔
282
283	const std::string share_fsname = get_arg("share");	5✔
284	const auto share_bits = slurp_file(share_fsname);	5✔
285
286	try {	5✔
287	auto share = FEC_Share::deserialize(share_bits.data(), share_bits.size(), *hash);	5✔
288	output() << "FEC share " << share.share_id() + 1 << "/" << share.n() << " with " << share.k()	5✔
289	<< " needed for recovery\n";	5✔
290	} catch(std::exception& e) {	5✔
291	error_output() << "Invalid share '" << share_fsname << "': " << e.what() << "\n";	×
292	}	×
293	}	15✔
294	};
295
296	BOTAN_REGISTER_COMMAND("fec_info", FEC_Info);	6✔
297
298	#endif
299
300	} // namespace Botan_CLI

randombit / botan / 6546858227

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous