30732274987

Committed 01 Aug 2026 02:42PM UTC coverage: 89.532% (+0.01%) from 89.522%

Build # 30732274987

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #5781 from randombit/jack/srp6-opt

Optimize SRP6's server step2

Coverage Stats

116608 of 130242 relevant lines covered (89.53%)

10499235.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.94

/src/lib/utils/ghash/ghash.cpp

/*
* GCM GHASH
* (C) 2013,2015,2017 Jack Lloyd
* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
* (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/ghash.h>

#include <botan/exceptn.h>
#include <botan/internal/ct_utils.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_CPUID)
   #include <botan/internal/cpuid.h>
#endif

namespace Botan {

std::string GHASH::provider() const {
#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(auto feat = CPUID::check(CPUID::Feature::AVX512_CLMUL)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(auto feat = CPUID::check(CPUID::Feature::HW_CLMUL)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
   if(auto feat = CPUID::check(CPUID::Feature::SIMD_4X32)) {
      return *feat;
   }
#endif

   return "base";
}

void GHASH::ghash_multiply(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input, size_t blocks) {
   BOTAN_ASSERT_NOMSG(input.size() % GCM_BS == 0);

#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {
      BOTAN_ASSERT_NOMSG(!m_H_pow.empty());
      return ghash_multiply_avx512_clmul(x.data(), m_H_pow.data(), input.data(), blocks);
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(CPUID::has(CPUID::Feature::HW_CLMUL)) {
      BOTAN_ASSERT_NOMSG(!m_H_pow.empty());
      return ghash_multiply_cpu(x.data(), m_H_pow, input.data(), blocks);
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
   if(CPUID::has(CPUID::Feature::SIMD_2X64)) {
      return ghash_multiply_vperm(x.data(), m_HM.data(), input.data(), blocks);
   }
#endif

   ghash_multiply_base(x, m_HM, input, blocks);
}

void GHASH::ghash_multiply_base(std::span<uint8_t, GCM_BS> x,
                                const secure_vector<uint64_t>& HM,
                                std::span<const uint8_t> input,
                                size_t blocks) {
   auto scope = CT::scoped_poison(x);

   auto X = load_be<std::array<uint64_t, 2>>(x);

   BufferSlicer in(input);
   for(size_t b = 0; b != blocks; ++b) {
      const auto I = load_be<std::array<uint64_t, 2>>(in.take<GCM_BS>());
      X[0] ^= I[0];
      X[1] ^= I[1];

      std::array<uint64_t, 2> Z{};

      for(size_t i = 0; i != 64; ++i) {
         const auto X0MASK = CT::Mask<uint64_t>::expand_top_bit(X[0]);
         const auto X1MASK = CT::Mask<uint64_t>::expand_top_bit(X[1]);

         X[0] <<= 1;
         X[1] <<= 1;

         Z[0] = X0MASK.select(Z[0] ^ HM[4 * i], Z[0]);
         Z[1] = X0MASK.select(Z[1] ^ HM[4 * i + 1], Z[1]);

         Z[0] = X1MASK.select(Z[0] ^ HM[4 * i + 2], Z[0]);
         Z[1] = X1MASK.select(Z[1] ^ HM[4 * i + 3], Z[1]);
      }

      X[0] = Z[0];
      X[1] = Z[1];
   }

   store_be(x, X);
}

bool GHASH::has_keying_material() const {
   return !m_HM.empty() || !m_H_pow.empty();
}

void GHASH::key_schedule(std::span<const uint8_t> key) {
   m_H_ad = {0};
   m_ad_len = 0;
   m_text_len = 0;

   BOTAN_ASSERT_NOMSG(key.size() == GCM_BS);

#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {
      zap(m_HM);
      if(m_H_pow.size() != 32) {
         m_H_pow.resize(32);
      }
      ghash_precompute_avx512_clmul(key.data(), m_H_pow.data());
      // m_HM left empty
      return;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(CPUID::has(CPUID::Feature::HW_CLMUL)) {
      zap(m_HM);
      ghash_precompute_cpu(key.data(), m_H_pow);
      // m_HM left empty
      return;
   }
#endif

   ghash_precompute_base(key.first<GCM_BS>(), m_HM);
}

void GHASH::ghash_precompute_base(std::span<const uint8_t, GCM_BS> key, secure_vector<uint64_t>& HM) {
   auto H = load_be<std::array<uint64_t, 2>>(key);

   const uint64_t R = 0xE100000000000000;

   if(HM.size() != 256) {
      HM.resize(256);
   }

   // precompute the multiples of H
   for(size_t i = 0; i != 2; ++i) {
      for(size_t j = 0; j != 64; ++j) {
         /*
         we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
         to make indexing nicer in the multiplication code
         */
         HM[4 * j + 2 * i] = H[0];
         HM[4 * j + 2 * i + 1] = H[1];

         // GCM's bit ops are reversed so we carry out of the bottom
         const uint64_t carry = CT::Mask<uint64_t>::expand(H[1] & 1).if_set_return(R);
         H[1] = (H[1] >> 1) | (H[0] << 63);
         H[0] = (H[0] >> 1) ^ carry;
      }
   }
}

void GHASH::start(std::span<const uint8_t> nonce) {
   BOTAN_ARG_CHECK(nonce.size() == 16, "GHASH requires a 128-bit nonce");
   auto& n = m_nonce.emplace();
   copy_mem(n, nonce);
   copy_mem(m_ghash, m_H_ad);
   m_buffer.clear();
   m_text_len = 0;
}

void GHASH::set_associated_data(std::span<const uint8_t> input) {
   BOTAN_STATE_CHECK(!m_nonce);

   assert_key_material_set();
   m_H_ad = {0};
   ghash_update(m_H_ad, input);
   ghash_zeropad(m_H_ad);
   m_ad_len = input.size();
}

void GHASH::reset_associated_data() {
   // This should only be called in GMAC context
   BOTAN_STATE_CHECK(m_text_len == 0);
   assert_key_material_set();
   m_H_ad = {0};
   m_ad_len = 0;
}

void GHASH::update_associated_data(std::span<const uint8_t> ad) {
   assert_key_material_set();
   ghash_update(m_ghash, ad);
   m_ad_len += ad.size();
}

void GHASH::update(std::span<const uint8_t> input) {
   assert_key_material_set();
   BOTAN_STATE_CHECK(m_nonce);
   ghash_update(m_ghash, input);
   m_text_len += input.size();

   // NIST SP 800-38D limits plaintext/ciphertext to 2^39 - 256 bits
   constexpr uint64_t GHASH_MAX_BYTES = (((static_cast<uint64_t>(1) << 39)) - 256) / 8;
   if(m_text_len > GHASH_MAX_BYTES) {
      throw Invalid_State("GCM message length limit exceeded");
   }
}

void GHASH::final(std::span<uint8_t> mac) {
   BOTAN_ARG_CHECK(!mac.empty() && mac.size() <= GCM_BS, "GHASH output length");
   BOTAN_STATE_CHECK(m_nonce);
   assert_key_material_set();

   ghash_zeropad(m_ghash);
   ghash_final_block(m_ghash, m_ad_len, m_text_len);

   xor_buf(mac, std::span{m_ghash}.first(mac.size()), std::span{*m_nonce}.first(mac.size()));

   secure_scrub_memory(m_ghash);
   m_text_len = 0;
   m_nonce.reset();
}

void GHASH::nonce_hash(std::span<uint8_t, GCM_BS> y0, std::span<const uint8_t> nonce) {
   assert_key_material_set();
   BOTAN_STATE_CHECK(!m_nonce);

   ghash_update(y0, nonce);
   ghash_zeropad(y0);
   ghash_final_block(y0, 0, nonce.size());
}

void GHASH::clear() {
   zap(m_HM);
   zap(m_H_pow);
   m_H_ad = {0};
   m_ad_len = 0;
   this->reset_state();
}

void GHASH::reset_state() {
   secure_scrub_memory(m_ghash);
   if(m_nonce) {
      secure_scrub_memory(m_nonce.value());
      m_nonce.reset();
   }
   m_buffer.clear();
   m_text_len = 0;
}

void GHASH::ghash_update(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input) {
   BufferSlicer in(input);
   while(!in.empty()) {
      if(const auto one_block = m_buffer.handle_unaligned_data(in)) {
         ghash_multiply(x, one_block.value(), 1);
      }

      if(m_buffer.in_alignment()) {
         const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);
         if(full_blocks > 0) {
            ghash_multiply(x, aligned_data, full_blocks);
         }
      }
   }
   BOTAN_ASSERT_NOMSG(in.empty());
}

void GHASH::ghash_zeropad(std::span<uint8_t, GCM_BS> x) {
   if(!m_buffer.in_alignment()) {
      m_buffer.fill_up_with_zeros();
      ghash_multiply(x, m_buffer.consume(), 1);
   }
}

void GHASH::ghash_final_block(std::span<uint8_t, GCM_BS> x, uint64_t ad_len, uint64_t text_len) {
   BOTAN_STATE_CHECK(m_buffer.in_alignment());
   const auto final_block = store_be(8 * ad_len, 8 * text_len);
   ghash_multiply(x, final_block, 1);
}

}  // namespace Botan

1	/*
2	* GCM GHASH
3	* (C) 2013,2015,2017 Jack Lloyd
4	* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
5	* (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
6	*
7	* Botan is released under the Simplified BSD License (see license.txt)
8	*/
9
10	#include <botan/internal/ghash.h>
11
12	#include <botan/exceptn.h>
13	#include <botan/internal/ct_utils.h>
14	#include <botan/internal/loadstor.h>
15
16	#if defined(BOTAN_HAS_CPUID)
17	#include <botan/internal/cpuid.h>
18	#endif
19
20	namespace Botan {
21
22	std::string GHASH::provider() const {	1,013✔
23	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
24	if(auto feat = CPUID::check(CPUID::Feature::AVX512_CLMUL)) {	1,013✔
25	return *feat;	×
26	}	×
27	#endif
28
29	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
30	if(auto feat = CPUID::check(CPUID::Feature::HW_CLMUL)) {	1,013✔
31	return *feat;	1,014✔
32	}	507✔
33	#endif
34
35	#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
36	if(auto feat = CPUID::check(CPUID::Feature::SIMD_4X32)) {	506✔
37	return *feat;	506✔
38	}	253✔
39	#endif
40
41	return "base";	253✔
42	}
43
44	void GHASH::ghash_multiply(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input, size_t blocks) {	69,618✔
45	BOTAN_ASSERT_NOMSG(input.size() % GCM_BS == 0);	69,618✔
46
47	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
48	if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {	69,618✔
49	BOTAN_ASSERT_NOMSG(!m_H_pow.empty());	×
50	return ghash_multiply_avx512_clmul(x.data(), m_H_pow.data(), input.data(), blocks);	×
51	}
52	#endif
53
54	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
55	if(CPUID::has(CPUID::Feature::HW_CLMUL)) {	69,618✔
56	BOTAN_ASSERT_NOMSG(!m_H_pow.empty());	49,491✔
57	return ghash_multiply_cpu(x.data(), m_H_pow, input.data(), blocks);	49,491✔
58	}
59	#endif
60
61	#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
62	if(CPUID::has(CPUID::Feature::SIMD_2X64)) {	20,127✔
63	return ghash_multiply_vperm(x.data(), m_HM.data(), input.data(), blocks);	10,064✔
64	}
65	#endif
66
67	ghash_multiply_base(x, m_HM, input, blocks);	10,063✔
68	}
69
70	void GHASH::ghash_multiply_base(std::span<uint8_t, GCM_BS> x,	150,066✔
71	const secure_vector<uint64_t>& HM,
72	std::span<const uint8_t> input,
73	size_t blocks) {
74	auto scope = CT::scoped_poison(x);	150,066✔
75
76	auto X = load_be<std::array<uint64_t, 2>>(x);	150,066✔
77
78	BufferSlicer in(input);	150,066✔
79	for(size_t b = 0; b != blocks; ++b) {	333,654✔
80	const auto I = load_be<std::array<uint64_t, 2>>(in.take<GCM_BS>());	183,588✔
81	X[0] ^= I[0];	183,588✔
82	X[1] ^= I[1];	183,588✔
83
84	std::array<uint64_t, 2> Z{};	183,588✔
85
86	for(size_t i = 0; i != 64; ++i) {	11,933,220✔
87	const auto X0MASK = CT::Mask<uint64_t>::expand_top_bit(X[0]);	11,749,632✔
88	const auto X1MASK = CT::Mask<uint64_t>::expand_top_bit(X[1]);	11,749,632✔
89
90	X[0] <<= 1;	11,749,632✔
91	X[1] <<= 1;	11,749,632✔
92
93	Z[0] = X0MASK.select(Z[0] ^ HM[4 * i], Z[0]);	11,749,632✔
94	Z[1] = X0MASK.select(Z[1] ^ HM[4 * i + 1], Z[1]);	11,749,632✔
95
96	Z[0] = X1MASK.select(Z[0] ^ HM[4 * i + 2], Z[0]);	11,749,632✔
97	Z[1] = X1MASK.select(Z[1] ^ HM[4 * i + 3], Z[1]);	11,749,632✔
98	}
99
100	X[0] = Z[0];	183,588✔
101	X[1] = Z[1];	183,588✔
102	}
103
104	store_be(x, X);	150,066✔
105	}	150,066✔
106
107	bool GHASH::has_keying_material() const {	151,446✔
108	return !m_HM.empty() \|\| !m_H_pow.empty();	151,446✔
109	}
110
111	void GHASH::key_schedule(std::span<const uint8_t> key) {	10,711✔
112	m_H_ad = {0};	10,711✔
113	m_ad_len = 0;	10,711✔
114	m_text_len = 0;	10,711✔
115
116	BOTAN_ASSERT_NOMSG(key.size() == GCM_BS);	10,711✔
117
118	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
119	if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {	10,711✔
120	zap(m_HM);	×
121	if(m_H_pow.size() != 32) {	×
122	m_H_pow.resize(32);	×
123	}
124	ghash_precompute_avx512_clmul(key.data(), m_H_pow.data());	×
125	// m_HM left empty
126	return;	×
127	}
128	#endif
129
130	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
131	if(CPUID::has(CPUID::Feature::HW_CLMUL)) {	10,711✔
132	zap(m_HM);	7,427✔
133	ghash_precompute_cpu(key.data(), m_H_pow);	7,427✔
134	// m_HM left empty
135	return;	7,427✔
136	}
137	#endif
138
139	ghash_precompute_base(key.first<GCM_BS>(), m_HM);	3,284✔
140	}
141
142	void GHASH::ghash_precompute_base(std::span<const uint8_t, GCM_BS> key, secure_vector<uint64_t>& HM) {	10,470✔
143	auto H = load_be<std::array<uint64_t, 2>>(key);	10,470✔
144
145	const uint64_t R = 0xE100000000000000;	10,470✔
146
147	if(HM.size() != 256) {	10,470✔
148	HM.resize(256);	10,360✔
149	}
150
151	// precompute the multiples of H
152	for(size_t i = 0; i != 2; ++i) {	31,410✔
153	for(size_t j = 0; j != 64; ++j) {	1,361,100✔
154	/*
155	we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
156	to make indexing nicer in the multiplication code
157	*/
158	HM[4 * j + 2 * i] = H[0];	1,340,160✔
159	HM[4 * j + 2 * i + 1] = H[1];	1,340,160✔
160
161	// GCM's bit ops are reversed so we carry out of the bottom
162	const uint64_t carry = CT::Mask<uint64_t>::expand(H[1] & 1).if_set_return(R);	1,340,160✔
163	H[1] = (H[1] >> 1) \| (H[0] << 63);	1,340,160✔
164	H[0] = (H[0] >> 1) ^ carry;	1,340,160✔
165	}
166	}
167	}	10,470✔
168
169	void GHASH::start(std::span<const uint8_t> nonce) {	20,591✔
170	BOTAN_ARG_CHECK(nonce.size() == 16, "GHASH requires a 128-bit nonce");	20,591✔
171	auto& n = m_nonce.emplace();	20,591✔
172	copy_mem(n, nonce);	20,591✔
173	copy_mem(m_ghash, m_H_ad);	20,591✔
174	m_buffer.clear();	20,591✔
175	m_text_len = 0;	20,591✔
176	}	20,591✔
177
178	void GHASH::set_associated_data(std::span<const uint8_t> input) {	12,497✔
179	BOTAN_STATE_CHECK(!m_nonce);	12,497✔
180
181	assert_key_material_set();	12,057✔
182	m_H_ad = {0};	11,177✔
183	ghash_update(m_H_ad, input);	11,177✔
184	ghash_zeropad(m_H_ad);	11,177✔
185	m_ad_len = input.size();	11,177✔
186	}	11,177✔
187
188	void GHASH::reset_associated_data() {	13,666✔
189	// This should only be called in GMAC context
190	BOTAN_STATE_CHECK(m_text_len == 0);	13,666✔
191	assert_key_material_set();	13,666✔
192	m_H_ad = {0};	13,666✔
193	m_ad_len = 0;	13,666✔
194	}	13,666✔
195
196	void GHASH::update_associated_data(std::span<const uint8_t> ad) {	10,200✔
197	assert_key_material_set();	10,200✔
198	ghash_update(m_ghash, ad);	10,200✔
199	m_ad_len += ad.size();	10,200✔
200	}	10,200✔
201
202	void GHASH::update(std::span<const uint8_t> input) {	97,164✔
203	assert_key_material_set();	97,164✔
204	BOTAN_STATE_CHECK(m_nonce);	97,164✔
205	ghash_update(m_ghash, input);	97,164✔
206	m_text_len += input.size();	97,164✔
207
208	// NIST SP 800-38D limits plaintext/ciphertext to 2^39 - 256 bits
209	constexpr uint64_t GHASH_MAX_BYTES = (((static_cast<uint64_t>(1) << 39)) - 256) / 8;	97,164✔
210	if(m_text_len > GHASH_MAX_BYTES) {	97,164✔
211	throw Invalid_State("GCM message length limit exceeded");	×
212	}
213	}	97,164✔
214
215	void GHASH::final(std::span<uint8_t> mac) {	16,319✔
216	BOTAN_ARG_CHECK(!mac.empty() && mac.size() <= GCM_BS, "GHASH output length");	16,319✔
217	BOTAN_STATE_CHECK(m_nonce);	16,319✔
218	assert_key_material_set();	16,319✔
219
220	ghash_zeropad(m_ghash);	16,319✔
221	ghash_final_block(m_ghash, m_ad_len, m_text_len);	16,319✔
222
223	xor_buf(mac, std::span{m_ghash}.first(mac.size()), std::span{*m_nonce}.first(mac.size()));	16,319✔
224
225	secure_scrub_memory(m_ghash);	16,319✔
226	m_text_len = 0;	16,319✔
227	m_nonce.reset();	16,319✔
228	}	16,319✔
229
230	void GHASH::nonce_hash(std::span<uint8_t, GCM_BS> y0, std::span<const uint8_t> nonce) {	2,040✔
231	assert_key_material_set();	2,040✔
232	BOTAN_STATE_CHECK(!m_nonce);	2,040✔
233
234	ghash_update(y0, nonce);	2,040✔
235	ghash_zeropad(y0);	2,040✔
236	ghash_final_block(y0, 0, nonce.size());	2,040✔
237	}	2,040✔
238
239	void GHASH::clear() {	7,275✔
240	zap(m_HM);	7,275✔
241	zap(m_H_pow);	7,275✔
242	m_H_ad = {0};	7,275✔
243	m_ad_len = 0;	7,275✔
244	this->reset_state();	7,275✔
245	}	7,275✔
246
247	void GHASH::reset_state() {	24,946✔
248	secure_scrub_memory(m_ghash);	24,946✔
249	if(m_nonce) {	24,946✔
250	secure_scrub_memory(m_nonce.value());	3,700✔
251	m_nonce.reset();	3,700✔
252	}
253	m_buffer.clear();	24,946✔
254	m_text_len = 0;	24,946✔
255	}	24,946✔
256
257	void GHASH::ghash_update(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input) {	120,581✔
258	BufferSlicer in(input);	120,581✔
259	while(!in.empty()) {	376,479✔
260	if(const auto one_block = m_buffer.handle_unaligned_data(in)) {	135,317✔
261	ghash_multiply(x, one_block.value(), 1);	6,408✔
262	}
263
264	if(m_buffer.in_alignment()) {	135,317✔
265	const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);	26,725✔
266	if(full_blocks > 0) {	26,725✔
267	ghash_multiply(x, aligned_data, full_blocks);	21,373✔
268	}
269	}
270	}
271	BOTAN_ASSERT_NOMSG(in.empty());	120,581✔
272	}	120,581✔
273
274	void GHASH::ghash_zeropad(std::span<uint8_t, GCM_BS> x) {	29,536✔
275	if(!m_buffer.in_alignment()) {	29,536✔
276	m_buffer.fill_up_with_zeros();	23,478✔
277	ghash_multiply(x, m_buffer.consume(), 1);	23,478✔
278	}
279	}	29,536✔
280
281	void GHASH::ghash_final_block(std::span<uint8_t, GCM_BS> x, uint64_t ad_len, uint64_t text_len) {	18,359✔
282	BOTAN_STATE_CHECK(m_buffer.in_alignment());	18,359✔
283	const auto final_block = store_be(8 * ad_len, 8 * text_len);	18,359✔
284	ghash_multiply(x, final_block, 1);	18,359✔
285	}	18,359✔
286
287	} // namespace Botan

randombit / botan / 30732274987

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous