26995937053

Committed 04 Jun 2026 09:38PM UTC coverage: 89.394% (-2.3%) from 91.672%

Build # 26995937053

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #5642 from randombit/jack/prefetch-in-ks

Improve prefetching for table based implementations

Coverage Stats

110588 of 123708 relevant lines covered (89.39%)

11056434.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.38

/src/lib/utils/ghash/ghash.cpp

/*
* GCM GHASH
* (C) 2013,2015,2017 Jack Lloyd
* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
* (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/ghash.h>

#include <botan/exceptn.h>
#include <botan/internal/ct_utils.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_CPUID)
   #include <botan/internal/cpuid.h>
#endif

namespace Botan {

std::string GHASH::provider() const {
#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(auto feat = CPUID::check(CPUID::Feature::AVX512_CLMUL)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(auto feat = CPUID::check(CPUID::Feature::HW_CLMUL)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
   if(auto feat = CPUID::check(CPUID::Feature::SIMD_4X32)) {
      return *feat;
   }
#endif

   return "base";
}

void GHASH::ghash_multiply(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input, size_t blocks) {
   BOTAN_ASSERT_NOMSG(input.size() % GCM_BS == 0);

#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {
      BOTAN_ASSERT_NOMSG(!m_H_pow.empty());
      return ghash_multiply_avx512_clmul(x.data(), m_H_pow.data(), input.data(), blocks);
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(CPUID::has(CPUID::Feature::HW_CLMUL)) {
      BOTAN_ASSERT_NOMSG(!m_H_pow.empty());
      return ghash_multiply_cpu(x.data(), m_H_pow, input.data(), blocks);
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
   if(CPUID::has(CPUID::Feature::SIMD_2X64)) {
      return ghash_multiply_vperm(x.data(), m_HM.data(), input.data(), blocks);
   }
#endif

   auto scope = CT::scoped_poison(x);

   auto X = load_be<std::array<uint64_t, 2>>(x);

   BufferSlicer in(input);
   for(size_t b = 0; b != blocks; ++b) {
      const auto I = load_be<std::array<uint64_t, 2>>(in.take<GCM_BS>());
      X[0] ^= I[0];
      X[1] ^= I[1];

      std::array<uint64_t, 2> Z{};

      for(size_t i = 0; i != 64; ++i) {
         const auto X0MASK = CT::Mask<uint64_t>::expand_top_bit(X[0]);
         const auto X1MASK = CT::Mask<uint64_t>::expand_top_bit(X[1]);

         X[0] <<= 1;
         X[1] <<= 1;

         Z[0] = X0MASK.select(Z[0] ^ m_HM[4 * i], Z[0]);
         Z[1] = X0MASK.select(Z[1] ^ m_HM[4 * i + 1], Z[1]);

         Z[0] = X1MASK.select(Z[0] ^ m_HM[4 * i + 2], Z[0]);
         Z[1] = X1MASK.select(Z[1] ^ m_HM[4 * i + 3], Z[1]);
      }

      X[0] = Z[0];
      X[1] = Z[1];
   }

   store_be(x, X);
}

bool GHASH::has_keying_material() const {
   return !m_HM.empty() || !m_H_pow.empty();
}

void GHASH::key_schedule(std::span<const uint8_t> key) {
   m_H_ad = {0};
   m_ad_len = 0;
   m_text_len = 0;

   BOTAN_ASSERT_NOMSG(key.size() == GCM_BS);
   auto H = load_be<std::array<uint64_t, 2>>(key.first<GCM_BS>());

#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {
      zap(m_HM);
      if(m_H_pow.size() != 32) {
         m_H_pow.resize(32);
      }
      ghash_precompute_avx512_clmul(key.data(), m_H_pow.data());
      // m_HM left empty
      return;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(CPUID::has(CPUID::Feature::HW_CLMUL)) {
      zap(m_HM);
      ghash_precompute_cpu(key.data(), m_H_pow);
      // m_HM left empty
      return;
   }
#endif

   const uint64_t R = 0xE100000000000000;

   if(m_HM.size() != 256) {
      m_HM.resize(256);
   }

   // precompute the multiples of H
   for(size_t i = 0; i != 2; ++i) {
      for(size_t j = 0; j != 64; ++j) {
         /*
         we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
         to make indexing nicer in the multiplication code
         */
         m_HM[4 * j + 2 * i] = H[0];
         m_HM[4 * j + 2 * i + 1] = H[1];

         // GCM's bit ops are reversed so we carry out of the bottom
         const uint64_t carry = CT::Mask<uint64_t>::expand(H[1] & 1).if_set_return(R);
         H[1] = (H[1] >> 1) | (H[0] << 63);
         H[0] = (H[0] >> 1) ^ carry;
      }
   }
}

void GHASH::start(std::span<const uint8_t> nonce) {
   BOTAN_ARG_CHECK(nonce.size() == 16, "GHASH requires a 128-bit nonce");
   auto& n = m_nonce.emplace();
   copy_mem(n, nonce);
   copy_mem(m_ghash, m_H_ad);
   m_buffer.clear();
   m_text_len = 0;
}

void GHASH::set_associated_data(std::span<const uint8_t> input) {
   BOTAN_STATE_CHECK(!m_nonce);

   assert_key_material_set();
   m_H_ad = {0};
   ghash_update(m_H_ad, input);
   ghash_zeropad(m_H_ad);
   m_ad_len = input.size();
}

void GHASH::reset_associated_data() {
   // This should only be called in GMAC context
   BOTAN_STATE_CHECK(m_text_len == 0);
   assert_key_material_set();
   m_H_ad = {0};
   m_ad_len = 0;
}

void GHASH::update_associated_data(std::span<const uint8_t> ad) {
   assert_key_material_set();
   ghash_update(m_ghash, ad);
   m_ad_len += ad.size();
}

void GHASH::update(std::span<const uint8_t> input) {
   assert_key_material_set();
   BOTAN_STATE_CHECK(m_nonce);
   ghash_update(m_ghash, input);
   m_text_len += input.size();

   // NIST SP 800-38D limits plaintext/ciphertext to 2^39 - 256 bits
   constexpr uint64_t GHASH_MAX_BYTES = (((static_cast<uint64_t>(1) << 39)) - 256) / 8;
   if(m_text_len > GHASH_MAX_BYTES) {
      throw Invalid_State("GCM message length limit exceeded");
   }
}

void GHASH::final(std::span<uint8_t> mac) {
   BOTAN_ARG_CHECK(!mac.empty() && mac.size() <= GCM_BS, "GHASH output length");
   BOTAN_STATE_CHECK(m_nonce);
   assert_key_material_set();

   ghash_zeropad(m_ghash);
   ghash_final_block(m_ghash, m_ad_len, m_text_len);

   xor_buf(mac, std::span{m_ghash}.first(mac.size()), std::span{*m_nonce}.first(mac.size()));

   secure_scrub_memory(m_ghash);
   m_text_len = 0;
   m_nonce.reset();
}

void GHASH::nonce_hash(std::span<uint8_t, GCM_BS> y0, std::span<const uint8_t> nonce) {
   assert_key_material_set();
   BOTAN_STATE_CHECK(!m_nonce);

   ghash_update(y0, nonce);
   ghash_zeropad(y0);
   ghash_final_block(y0, 0, nonce.size());
}

void GHASH::clear() {
   zap(m_HM);
   zap(m_H_pow);
   m_H_ad = {0};
   m_ad_len = 0;
   this->reset_state();
}

void GHASH::reset_state() {
   secure_scrub_memory(m_ghash);
   if(m_nonce) {
      secure_scrub_memory(m_nonce.value());
      m_nonce.reset();
   }
   m_buffer.clear();
   m_text_len = 0;
}

void GHASH::ghash_update(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input) {
   BufferSlicer in(input);
   while(!in.empty()) {
      if(const auto one_block = m_buffer.handle_unaligned_data(in)) {
         ghash_multiply(x, one_block.value(), 1);
      }

      if(m_buffer.in_alignment()) {
         const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);
         if(full_blocks > 0) {
            ghash_multiply(x, aligned_data, full_blocks);
         }
      }
   }
   BOTAN_ASSERT_NOMSG(in.empty());
}

void GHASH::ghash_zeropad(std::span<uint8_t, GCM_BS> x) {
   if(!m_buffer.in_alignment()) {
      m_buffer.fill_up_with_zeros();
      ghash_multiply(x, m_buffer.consume(), 1);
   }
}

void GHASH::ghash_final_block(std::span<uint8_t, GCM_BS> x, uint64_t ad_len, uint64_t text_len) {
   BOTAN_STATE_CHECK(m_buffer.in_alignment());
   const auto final_block = store_be(8 * ad_len, 8 * text_len);
   ghash_multiply(x, final_block, 1);
}

}  // namespace Botan

1	/*
2	* GCM GHASH
3	* (C) 2013,2015,2017 Jack Lloyd
4	* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
5	* (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
6	*
7	* Botan is released under the Simplified BSD License (see license.txt)
8	*/
9
10	#include <botan/internal/ghash.h>
11
12	#include <botan/exceptn.h>
13	#include <botan/internal/ct_utils.h>
14	#include <botan/internal/loadstor.h>
15
16	#if defined(BOTAN_HAS_CPUID)
17	#include <botan/internal/cpuid.h>
18	#endif
19
20	namespace Botan {
21
22	std::string GHASH::provider() const {	1,013✔
23	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
24	if(auto feat = CPUID::check(CPUID::Feature::AVX512_CLMUL)) {	1,013✔
25	return *feat;	×
26	}	×
27	#endif
28
29	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
30	if(auto feat = CPUID::check(CPUID::Feature::HW_CLMUL)) {	1,013✔
31	return *feat;	1,014✔
32	}	507✔
33	#endif
34
35	#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
36	if(auto feat = CPUID::check(CPUID::Feature::SIMD_4X32)) {	506✔
37	return *feat;	506✔
38	}	253✔
39	#endif
40
41	return "base";	253✔
42	}
43
44	void GHASH::ghash_multiply(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input, size_t blocks) {	61,571✔
45	BOTAN_ASSERT_NOMSG(input.size() % GCM_BS == 0);	61,571✔
46
47	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
48	if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {	61,571✔
49	BOTAN_ASSERT_NOMSG(!m_H_pow.empty());	×
50	return ghash_multiply_avx512_clmul(x.data(), m_H_pow.data(), input.data(), blocks);	52,604✔
51	}
52	#endif
53
54	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
55	if(CPUID::has(CPUID::Feature::HW_CLMUL)) {	61,571✔
56	BOTAN_ASSERT_NOMSG(!m_H_pow.empty());	43,638✔
57	return ghash_multiply_cpu(x.data(), m_H_pow, input.data(), blocks);	43,638✔
58	}
59	#endif
60
61	#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
62	if(CPUID::has(CPUID::Feature::SIMD_2X64)) {	17,933✔
63	return ghash_multiply_vperm(x.data(), m_HM.data(), input.data(), blocks);	8,966✔
64	}
65	#endif
66
67	auto scope = CT::scoped_poison(x);	8,967✔
68
69	auto X = load_be<std::array<uint64_t, 2>>(x);	8,967✔
70
71	BufferSlicer in(input);	8,967✔
72	for(size_t b = 0; b != blocks; ++b) {	44,601✔
73	const auto I = load_be<std::array<uint64_t, 2>>(in.take<GCM_BS>());	35,634✔
74	X[0] ^= I[0];	35,634✔
75	X[1] ^= I[1];	35,634✔
76
77	std::array<uint64_t, 2> Z{};	35,634✔
78
79	for(size_t i = 0; i != 64; ++i) {	2,316,210✔
80	const auto X0MASK = CT::Mask<uint64_t>::expand_top_bit(X[0]);	2,280,576✔
81	const auto X1MASK = CT::Mask<uint64_t>::expand_top_bit(X[1]);	2,280,576✔
82
83	X[0] <<= 1;	2,280,576✔
84	X[1] <<= 1;	2,280,576✔
85
86	Z[0] = X0MASK.select(Z[0] ^ m_HM[4 * i], Z[0]);	2,280,576✔
87	Z[1] = X0MASK.select(Z[1] ^ m_HM[4 * i + 1], Z[1]);	2,280,576✔
88
89	Z[0] = X1MASK.select(Z[0] ^ m_HM[4 * i + 2], Z[0]);	2,280,576✔
90	Z[1] = X1MASK.select(Z[1] ^ m_HM[4 * i + 3], Z[1]);	2,280,576✔
91	}
92
93	X[0] = Z[0];	35,634✔
94	X[1] = Z[1];	35,634✔
95	}
96
97	store_be(x, X);	8,967✔
98	}	8,967✔
99
100	bool GHASH::has_keying_material() const {	137,135✔
101	return !m_HM.empty() \|\| !m_H_pow.empty();	137,135✔
102	}
103
104	void GHASH::key_schedule(std::span<const uint8_t> key) {	8,469✔
105	m_H_ad = {0};	8,469✔
106	m_ad_len = 0;	8,469✔
107	m_text_len = 0;	8,469✔
108
109	BOTAN_ASSERT_NOMSG(key.size() == GCM_BS);	8,469✔
110	auto H = load_be<std::array<uint64_t, 2>>(key.first<GCM_BS>());	8,469✔
111
112	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
113	if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {	8,469✔
114	zap(m_HM);	×
115	if(m_H_pow.size() != 32) {	×
116	m_H_pow.resize(32);	×
117	}
118	ghash_precompute_avx512_clmul(key.data(), m_H_pow.data());	×
119	// m_HM left empty
120	return;	×
121	}
122	#endif
123
124	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
125	if(CPUID::has(CPUID::Feature::HW_CLMUL)) {	8,469✔
126	zap(m_HM);	6,043✔
127	ghash_precompute_cpu(key.data(), m_H_pow);	6,043✔
128	// m_HM left empty
129	return;	6,043✔
130	}
131	#endif
132
133	const uint64_t R = 0xE100000000000000;	2,426✔
134
135	if(m_HM.size() != 256) {	2,426✔
136	m_HM.resize(256);	2,316✔
137	}
138
139	// precompute the multiples of H
140	for(size_t i = 0; i != 2; ++i) {	7,278✔
141	for(size_t j = 0; j != 64; ++j) {	315,380✔
142	/*
143	we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
144	to make indexing nicer in the multiplication code
145	*/
146	m_HM[4 * j + 2 * i] = H[0];	310,528✔
147	m_HM[4 * j + 2 * i + 1] = H[1];	310,528✔
148
149	// GCM's bit ops are reversed so we carry out of the bottom
150	const uint64_t carry = CT::Mask<uint64_t>::expand(H[1] & 1).if_set_return(R);	310,528✔
151	H[1] = (H[1] >> 1) \| (H[0] << 63);	310,528✔
152	H[0] = (H[0] >> 1) ^ carry;	310,528✔
153	}
154	}
155	}
156
157	void GHASH::start(std::span<const uint8_t> nonce) {	17,341✔
158	BOTAN_ARG_CHECK(nonce.size() == 16, "GHASH requires a 128-bit nonce");	17,341✔
159	auto& n = m_nonce.emplace();	17,341✔
160	copy_mem(n, nonce);	17,341✔
161	copy_mem(m_ghash, m_H_ad);	17,341✔
162	m_buffer.clear();	17,341✔
163	m_text_len = 0;	17,341✔
164	}	17,341✔
165
166	void GHASH::set_associated_data(std::span<const uint8_t> input) {	11,535✔
167	BOTAN_STATE_CHECK(!m_nonce);	11,535✔
168
169	assert_key_material_set();	11,095✔
170	m_H_ad = {0};	10,215✔
171	ghash_update(m_H_ad, input);	10,215✔
172	ghash_zeropad(m_H_ad);	10,215✔
173	m_ad_len = input.size();	10,215✔
174	}	10,215✔
175
176	void GHASH::reset_associated_data() {	4,545✔
177	// This should only be called in GMAC context
178	BOTAN_STATE_CHECK(m_text_len == 0);	4,545✔
179	assert_key_material_set();	4,545✔
180	m_H_ad = {0};	4,545✔
181	m_ad_len = 0;	4,545✔
182	}	4,545✔
183
184	void GHASH::update_associated_data(std::span<const uint8_t> ad) {	9,056✔
185	assert_key_material_set();	9,056✔
186	ghash_update(m_ghash, ad);	7,912✔
187	m_ad_len += ad.size();	7,912✔
188	}	7,912✔
189
190	void GHASH::update(std::span<const uint8_t> input) {	96,202✔
191	assert_key_material_set();	96,202✔
192	BOTAN_STATE_CHECK(m_nonce);	96,202✔
193	ghash_update(m_ghash, input);	96,202✔
194	m_text_len += input.size();	96,202✔
195
196	// NIST SP 800-38D limits plaintext/ciphertext to 2^39 - 256 bits
197	constexpr uint64_t GHASH_MAX_BYTES = (((static_cast<uint64_t>(1) << 39)) - 256) / 8;	96,202✔
198	if(m_text_len > GHASH_MAX_BYTES) {	96,202✔
199	throw Invalid_State("GCM message length limit exceeded");	×
200	}
201	}	96,202✔
202
203	void GHASH::final(std::span<uint8_t> mac) {	14,213✔
204	BOTAN_ARG_CHECK(!mac.empty() && mac.size() <= GCM_BS, "GHASH output length");	14,213✔
205	BOTAN_STATE_CHECK(m_nonce);	14,213✔
206	assert_key_material_set();	14,213✔
207
208	ghash_zeropad(m_ghash);	14,213✔
209	ghash_final_block(m_ghash, m_ad_len, m_text_len);	14,213✔
210
211	xor_buf(mac, std::span{m_ghash}.first(mac.size()), std::span{*m_nonce}.first(mac.size()));	14,213✔
212
213	secure_scrub_memory(m_ghash);	14,213✔
214	m_text_len = 0;	14,213✔
215	m_nonce.reset();	14,213✔
216	}	14,213✔
217
218	void GHASH::nonce_hash(std::span<uint8_t, GCM_BS> y0, std::span<const uint8_t> nonce) {	2,024✔
219	assert_key_material_set();	2,024✔
220	BOTAN_STATE_CHECK(!m_nonce);	2,024✔
221
222	ghash_update(y0, nonce);	2,024✔
223	ghash_zeropad(y0);	2,024✔
224	ghash_final_block(y0, 0, nonce.size());	2,024✔
225	}	2,024✔
226
227	void GHASH::clear() {	5,559✔
228	zap(m_HM);	5,559✔
229	zap(m_H_pow);	5,559✔
230	m_H_ad = {0};	5,559✔
231	m_ad_len = 0;	5,559✔
232	this->reset_state();	5,559✔
233	}	5,559✔
234
235	void GHASH::reset_state() {	14,727✔
236	secure_scrub_memory(m_ghash);	14,727✔
237	if(m_nonce) {	14,727✔
238	secure_scrub_memory(m_nonce.value());	2,556✔
239	m_nonce.reset();	2,556✔
240	}
241	m_buffer.clear();	14,727✔
242	m_text_len = 0;	14,727✔
243	}	14,727✔
244
245	void GHASH::ghash_update(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input) {	116,353✔
246	BufferSlicer in(input);	116,353✔
247	while(!in.empty()) {	361,030✔
248	if(const auto one_block = m_buffer.handle_unaligned_data(in)) {	128,324✔
249	ghash_multiply(x, one_block.value(), 1);	6,408✔
250	}
251
252	if(m_buffer.in_alignment()) {	128,324✔
253	const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);	23,680✔
254	if(full_blocks > 0) {	23,680✔
255	ghash_multiply(x, aligned_data, full_blocks);	18,328✔
256	}
257	}
258	}
259	BOTAN_ASSERT_NOMSG(in.empty());	116,353✔
260	}	116,353✔
261
262	void GHASH::ghash_zeropad(std::span<uint8_t, GCM_BS> x) {	26,452✔
263	if(!m_buffer.in_alignment()) {	26,452✔
264	m_buffer.fill_up_with_zeros();	20,598✔
265	ghash_multiply(x, m_buffer.consume(), 1);	20,598✔
266	}
267	}	26,452✔
268
269	void GHASH::ghash_final_block(std::span<uint8_t, GCM_BS> x, uint64_t ad_len, uint64_t text_len) {	16,237✔
270	BOTAN_STATE_CHECK(m_buffer.in_alignment());	16,237✔
271	const auto final_block = store_be(8 * ad_len, 8 * text_len);	16,237✔
272	ghash_multiply(x, final_block, 1);	16,237✔
273	}	16,237✔
274
275	} // namespace Botan

randombit / botan / 26995937053

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous