23486908132

Committed 24 Mar 2026 10:53AM UTC coverage: 89.449% (-2.4%) from 91.889%

Build # 23486908132

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #5479 from randombit/jack/aria-hwaes

Add ARIA implementation using hardware AES instructions

Coverage Stats

105058 of 117450 relevant lines covered (89.45%)

11951797.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.81

/src/lib/utils/ghash/ghash.cpp

/*
* GCM GHASH
* (C) 2013,2015,2017 Jack Lloyd
* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
* (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/ghash.h>

#include <botan/internal/ct_utils.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_CPUID)
   #include <botan/internal/cpuid.h>
#endif

namespace Botan {

std::string GHASH::provider() const {
#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(auto feat = CPUID::check(CPUID::Feature::AVX512_CLMUL)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(auto feat = CPUID::check(CPUID::Feature::HW_CLMUL)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
   if(auto feat = CPUID::check(CPUID::Feature::SIMD_4X32)) {
      return *feat;
   }
#endif

   return "base";
}

void GHASH::ghash_multiply(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input, size_t blocks) {
   BOTAN_ASSERT_NOMSG(input.size() % GCM_BS == 0);

#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {
      BOTAN_ASSERT_NOMSG(!m_H_pow.empty());
      return ghash_multiply_avx512_clmul(x.data(), m_H_pow.data(), input.data(), blocks);
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(CPUID::has(CPUID::Feature::HW_CLMUL)) {
      BOTAN_ASSERT_NOMSG(!m_H_pow.empty());
      return ghash_multiply_cpu(x.data(), m_H_pow, input.data(), blocks);
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
   if(CPUID::has(CPUID::Feature::SIMD_2X64)) {
      return ghash_multiply_vperm(x.data(), m_HM.data(), input.data(), blocks);
   }
#endif

   auto scope = CT::scoped_poison(x);

   auto X = load_be<std::array<uint64_t, 2>>(x);

   BufferSlicer in(input);
   for(size_t b = 0; b != blocks; ++b) {
      const auto I = load_be<std::array<uint64_t, 2>>(in.take<GCM_BS>());
      X[0] ^= I[0];
      X[1] ^= I[1];

      std::array<uint64_t, 2> Z{};

      for(size_t i = 0; i != 64; ++i) {
         const auto X0MASK = CT::Mask<uint64_t>::expand_top_bit(X[0]);
         const auto X1MASK = CT::Mask<uint64_t>::expand_top_bit(X[1]);

         X[0] <<= 1;
         X[1] <<= 1;

         Z[0] = X0MASK.select(Z[0] ^ m_HM[4 * i], Z[0]);
         Z[1] = X0MASK.select(Z[1] ^ m_HM[4 * i + 1], Z[1]);

         Z[0] = X1MASK.select(Z[0] ^ m_HM[4 * i + 2], Z[0]);
         Z[1] = X1MASK.select(Z[1] ^ m_HM[4 * i + 3], Z[1]);
      }

      X[0] = Z[0];
      X[1] = Z[1];
   }

   store_be(x, X);
}

bool GHASH::has_keying_material() const {
   return !m_HM.empty() || !m_H_pow.empty();
}

void GHASH::key_schedule(std::span<const uint8_t> key) {
   m_H_ad = {0};
   m_ad_len = 0;
   m_text_len = 0;

   BOTAN_ASSERT_NOMSG(key.size() == GCM_BS);
   auto H = load_be<std::array<uint64_t, 2>>(key.first<GCM_BS>());

#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
   if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {
      zap(m_HM);
      if(m_H_pow.size() != 32) {
         m_H_pow.resize(32);
      }
      ghash_precompute_avx512_clmul(key.data(), m_H_pow.data());
      // m_HM left empty
      return;
   }
#endif

#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
   if(CPUID::has(CPUID::Feature::HW_CLMUL)) {
      zap(m_HM);
      ghash_precompute_cpu(key.data(), m_H_pow);
      // m_HM left empty
      return;
   }
#endif

   const uint64_t R = 0xE100000000000000;

   if(m_HM.size() != 256) {
      m_HM.resize(256);
   }

   // precompute the multiples of H
   for(size_t i = 0; i != 2; ++i) {
      for(size_t j = 0; j != 64; ++j) {
         /*
         we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
         to make indexing nicer in the multiplication code
         */
         m_HM[4 * j + 2 * i] = H[0];
         m_HM[4 * j + 2 * i + 1] = H[1];

         // GCM's bit ops are reversed so we carry out of the bottom
         const uint64_t carry = CT::Mask<uint64_t>::expand(H[1] & 1).if_set_return(R);
         H[1] = (H[1] >> 1) | (H[0] << 63);
         H[0] = (H[0] >> 1) ^ carry;
      }
   }
}

void GHASH::start(std::span<const uint8_t> nonce) {
   BOTAN_ARG_CHECK(nonce.size() == 16, "GHASH requires a 128-bit nonce");
   auto& n = m_nonce.emplace();
   copy_mem(n, nonce);
   copy_mem(m_ghash, m_H_ad);
}

void GHASH::set_associated_data(std::span<const uint8_t> input) {
   BOTAN_STATE_CHECK(!m_nonce);

   assert_key_material_set();
   m_H_ad = {0};
   ghash_update(m_H_ad, input);
   ghash_zeropad(m_H_ad);
   m_ad_len = input.size();
}

void GHASH::reset_associated_data() {
   // This should only be called in GMAC context
   BOTAN_STATE_CHECK(m_text_len == 0);
   assert_key_material_set();
   m_H_ad = {0};
   m_ad_len = 0;
}

void GHASH::update_associated_data(std::span<const uint8_t> ad) {
   assert_key_material_set();
   ghash_update(m_ghash, ad);
   m_ad_len += ad.size();
}

void GHASH::update(std::span<const uint8_t> input) {
   assert_key_material_set();
   BOTAN_STATE_CHECK(m_nonce);
   ghash_update(m_ghash, input);
   m_text_len += input.size();
}

void GHASH::final(std::span<uint8_t> mac) {
   BOTAN_ARG_CHECK(!mac.empty() && mac.size() <= GCM_BS, "GHASH output length");
   BOTAN_STATE_CHECK(m_nonce);
   assert_key_material_set();

   ghash_zeropad(m_ghash);
   ghash_final_block(m_ghash, m_ad_len, m_text_len);

   xor_buf(mac, std::span{m_ghash}.first(mac.size()), std::span{*m_nonce}.first(mac.size()));

   secure_scrub_memory(m_ghash);
   m_text_len = 0;
   m_nonce.reset();
}

void GHASH::nonce_hash(std::span<uint8_t, GCM_BS> y0, std::span<const uint8_t> nonce) {
   assert_key_material_set();
   BOTAN_STATE_CHECK(!m_nonce);

   ghash_update(y0, nonce);
   ghash_zeropad(y0);
   ghash_final_block(y0, 0, nonce.size());
}

void GHASH::clear() {
   zap(m_HM);
   zap(m_H_pow);
   this->reset_state();
}

void GHASH::reset_state() {
   m_H_ad = {0};
   secure_scrub_memory(m_ghash);
   if(m_nonce) {
      secure_scrub_memory(m_nonce.value());
      m_nonce.reset();
   }
   m_buffer.clear();
   m_text_len = m_ad_len = 0;
}

void GHASH::ghash_update(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input) {
   BufferSlicer in(input);
   while(!in.empty()) {
      if(const auto one_block = m_buffer.handle_unaligned_data(in)) {
         ghash_multiply(x, one_block.value(), 1);
      }

      if(m_buffer.in_alignment()) {
         const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);
         if(full_blocks > 0) {
            ghash_multiply(x, aligned_data, full_blocks);
         }
      }
   }
   BOTAN_ASSERT_NOMSG(in.empty());
}

void GHASH::ghash_zeropad(std::span<uint8_t, GCM_BS> x) {
   if(!m_buffer.in_alignment()) {
      m_buffer.fill_up_with_zeros();
      ghash_multiply(x, m_buffer.consume(), 1);
   }
}

void GHASH::ghash_final_block(std::span<uint8_t, GCM_BS> x, uint64_t ad_len, uint64_t text_len) {
   BOTAN_STATE_CHECK(m_buffer.in_alignment());
   const auto final_block = store_be(8 * ad_len, 8 * text_len);
   ghash_multiply(x, final_block, 1);
}

}  // namespace Botan

1	/*
2	* GCM GHASH
3	* (C) 2013,2015,2017 Jack Lloyd
4	* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
5	* (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
6	*
7	* Botan is released under the Simplified BSD License (see license.txt)
8	*/
9
10	#include <botan/internal/ghash.h>
11
12	#include <botan/internal/ct_utils.h>
13	#include <botan/internal/loadstor.h>
14
15	#if defined(BOTAN_HAS_CPUID)
16	#include <botan/internal/cpuid.h>
17	#endif
18
19	namespace Botan {
20
21	std::string GHASH::provider() const {	1,013✔
22	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
23	if(auto feat = CPUID::check(CPUID::Feature::AVX512_CLMUL)) {	1,013✔
24	return *feat;	×
25	}	×
26	#endif
27
28	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
29	if(auto feat = CPUID::check(CPUID::Feature::HW_CLMUL)) {	1,013✔
30	return *feat;	1,014✔
31	}	507✔
32	#endif
33
34	#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
35	if(auto feat = CPUID::check(CPUID::Feature::SIMD_4X32)) {	506✔
36	return *feat;	506✔
37	}	253✔
38	#endif
39
40	return "base";	253✔
41	}
42
43	void GHASH::ghash_multiply(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input, size_t blocks) {	54,403✔
44	BOTAN_ASSERT_NOMSG(input.size() % GCM_BS == 0);	54,403✔
45
46	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
47	if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {	54,403✔
48	BOTAN_ASSERT_NOMSG(!m_H_pow.empty());	×
49	return ghash_multiply_avx512_clmul(x.data(), m_H_pow.data(), input.data(), blocks);	46,697✔
50	}
51	#endif
52
53	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
54	if(CPUID::has(CPUID::Feature::HW_CLMUL)) {	54,403✔
55	BOTAN_ASSERT_NOMSG(!m_H_pow.empty());	38,988✔
56	return ghash_multiply_cpu(x.data(), m_H_pow, input.data(), blocks);	38,988✔
57	}
58	#endif
59
60	#if defined(BOTAN_HAS_GHASH_CLMUL_VPERM)
61	if(CPUID::has(CPUID::Feature::SIMD_2X64)) {	15,415✔
62	return ghash_multiply_vperm(x.data(), m_HM.data(), input.data(), blocks);	7,709✔
63	}
64	#endif
65
66	auto scope = CT::scoped_poison(x);	7,706✔
67
68	auto X = load_be<std::array<uint64_t, 2>>(x);	7,706✔
69
70	BufferSlicer in(input);	7,706✔
71	for(size_t b = 0; b != blocks; ++b) {	39,084✔
72	const auto I = load_be<std::array<uint64_t, 2>>(in.take<GCM_BS>());	31,378✔
73	X[0] ^= I[0];	31,378✔
74	X[1] ^= I[1];	31,378✔
75
76	std::array<uint64_t, 2> Z{};	31,378✔
77
78	for(size_t i = 0; i != 64; ++i) {	2,039,570✔
79	const auto X0MASK = CT::Mask<uint64_t>::expand_top_bit(X[0]);	2,008,192✔
80	const auto X1MASK = CT::Mask<uint64_t>::expand_top_bit(X[1]);	2,008,192✔
81
82	X[0] <<= 1;	2,008,192✔
83	X[1] <<= 1;	2,008,192✔
84
85	Z[0] = X0MASK.select(Z[0] ^ m_HM[4 * i], Z[0]);	2,008,192✔
86	Z[1] = X0MASK.select(Z[1] ^ m_HM[4 * i + 1], Z[1]);	2,008,192✔
87
88	Z[0] = X1MASK.select(Z[0] ^ m_HM[4 * i + 2], Z[0]);	2,008,192✔
89	Z[1] = X1MASK.select(Z[1] ^ m_HM[4 * i + 3], Z[1]);	2,008,192✔
90	}
91
92	X[0] = Z[0];	31,378✔
93	X[1] = Z[1];	31,378✔
94	}
95
96	store_be(x, X);	7,706✔
97	}	7,706✔
98
99	bool GHASH::has_keying_material() const {	133,403✔
100	return !m_HM.empty() \|\| !m_H_pow.empty();	133,403✔
101	}
102
103	void GHASH::key_schedule(std::span<const uint8_t> key) {	8,044✔
104	m_H_ad = {0};	8,044✔
105	m_ad_len = 0;	8,044✔
106	m_text_len = 0;	8,044✔
107
108	BOTAN_ASSERT_NOMSG(key.size() == GCM_BS);	8,044✔
109	auto H = load_be<std::array<uint64_t, 2>>(key.first<GCM_BS>());	8,044✔
110
111	#if defined(BOTAN_HAS_GHASH_AVX512_CLMUL)
112	if(CPUID::has(CPUID::Feature::AVX512_CLMUL)) {	8,044✔
113	zap(m_HM);	×
114	if(m_H_pow.size() != 32) {	×
115	m_H_pow.resize(32);	×
116	}
117	ghash_precompute_avx512_clmul(key.data(), m_H_pow.data());	×
118	// m_HM left empty
119	return;	×
120	}
121	#endif
122
123	#if defined(BOTAN_HAS_GHASH_CLMUL_CPU)
124	if(CPUID::has(CPUID::Feature::HW_CLMUL)) {	8,044✔
125	zap(m_HM);	5,838✔
126	ghash_precompute_cpu(key.data(), m_H_pow);	5,838✔
127	// m_HM left empty
128	return;	5,838✔
129	}
130	#endif
131
132	const uint64_t R = 0xE100000000000000;	2,206✔
133
134	if(m_HM.size() != 256) {	2,206✔
135	m_HM.resize(256);	2,206✔
136	}
137
138	// precompute the multiples of H
139	for(size_t i = 0; i != 2; ++i) {	6,618✔
140	for(size_t j = 0; j != 64; ++j) {	286,780✔
141	/*
142	we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
143	to make indexing nicer in the multiplication code
144	*/
145	m_HM[4 * j + 2 * i] = H[0];	282,368✔
146	m_HM[4 * j + 2 * i + 1] = H[1];	282,368✔
147
148	// GCM's bit ops are reversed so we carry out of the bottom
149	const uint64_t carry = CT::Mask<uint64_t>::expand(H[1] & 1).if_set_return(R);	282,368✔
150	H[1] = (H[1] >> 1) \| (H[0] << 63);	282,368✔
151	H[0] = (H[0] >> 1) ^ carry;	282,368✔
152	}
153	}
154	}
155
156	void GHASH::start(std::span<const uint8_t> nonce) {	15,003✔
157	BOTAN_ARG_CHECK(nonce.size() == 16, "GHASH requires a 128-bit nonce");	15,003✔
158	auto& n = m_nonce.emplace();	15,003✔
159	copy_mem(n, nonce);	15,003✔
160	copy_mem(m_ghash, m_H_ad);	15,003✔
161	}	15,003✔
162
163	void GHASH::set_associated_data(std::span<const uint8_t> input) {	9,961✔
164	BOTAN_STATE_CHECK(!m_nonce);	9,961✔
165
166	assert_key_material_set();	9,521✔
167	m_H_ad = {0};	8,641✔
168	ghash_update(m_H_ad, input);	8,641✔
169	ghash_zeropad(m_H_ad);	8,641✔
170	m_ad_len = input.size();	8,641✔
171	}	8,641✔
172
173	void GHASH::reset_associated_data() {	4,545✔
174	// This should only be called in GMAC context
175	BOTAN_STATE_CHECK(m_text_len == 0);	4,545✔
176	assert_key_material_set();	4,545✔
177	m_H_ad = {0};	4,545✔
178	m_ad_len = 0;	4,545✔
179	}	4,545✔
180
181	void GHASH::update_associated_data(std::span<const uint8_t> ad) {	9,056✔
182	assert_key_material_set();	9,056✔
183	ghash_update(m_ghash, ad);	7,912✔
184	m_ad_len += ad.size();	7,912✔
185	}	7,912✔
186
187	void GHASH::update(std::span<const uint8_t> input) {	96,050✔
188	assert_key_material_set();	96,050✔
189	BOTAN_STATE_CHECK(m_nonce);	95,622✔
190	ghash_update(m_ghash, input);	94,962✔
191	m_text_len += input.size();	94,962✔
192	}	94,962✔
193
194	void GHASH::final(std::span<uint8_t> mac) {	13,427✔
195	BOTAN_ARG_CHECK(!mac.empty() && mac.size() <= GCM_BS, "GHASH output length");	13,427✔
196	BOTAN_STATE_CHECK(m_nonce);	13,427✔
197	assert_key_material_set();	12,975✔
198
199	ghash_zeropad(m_ghash);	12,975✔
200	ghash_final_block(m_ghash, m_ad_len, m_text_len);	12,975✔
201
202	xor_buf(mac, std::span{m_ghash}.first(mac.size()), std::span{*m_nonce}.first(mac.size()));	12,975✔
203
204	secure_scrub_memory(m_ghash);	12,975✔
205	m_text_len = 0;	12,975✔
206	m_nonce.reset();	12,975✔
207	}	12,975✔
208
209	void GHASH::nonce_hash(std::span<uint8_t, GCM_BS> y0, std::span<const uint8_t> nonce) {	1,256✔
210	assert_key_material_set();	1,256✔
211	BOTAN_STATE_CHECK(!m_nonce);	1,256✔
212
213	ghash_update(y0, nonce);	1,256✔
214	ghash_zeropad(y0);	1,256✔
215	ghash_final_block(y0, 0, nonce.size());	1,256✔
216	}	1,256✔
217
218	void GHASH::clear() {	5,559✔
219	zap(m_HM);	5,559✔
220	zap(m_H_pow);	5,559✔
221	this->reset_state();	5,559✔
222	}	5,559✔
223
224	void GHASH::reset_state() {	8,387✔
225	m_H_ad = {0};	8,387✔
226	secure_scrub_memory(m_ghash);	8,387✔
227	if(m_nonce) {	8,387✔
228	secure_scrub_memory(m_nonce.value());	1,456✔
229	m_nonce.reset();	1,456✔
230	}
231	m_buffer.clear();	8,387✔
232	m_text_len = m_ad_len = 0;	8,387✔
233	}	8,387✔
234
235	void GHASH::ghash_update(std::span<uint8_t, GCM_BS> x, std::span<const uint8_t> input) {	112,771✔
236	BufferSlicer in(input);	112,771✔
237	while(!in.empty()) {	348,704✔
238	if(const auto one_block = m_buffer.handle_unaligned_data(in)) {	123,162✔
239	ghash_multiply(x, one_block.value(), 1);	6,408✔
240	}
241
242	if(m_buffer.in_alignment()) {	123,162✔
243	const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);	21,299✔
244	if(full_blocks > 0) {	21,299✔
245	ghash_multiply(x, aligned_data, full_blocks);	15,947✔
246	}
247	}
248	}
249	BOTAN_ASSERT_NOMSG(in.empty());	112,771✔
250	}	112,771✔
251
252	void GHASH::ghash_zeropad(std::span<uint8_t, GCM_BS> x) {	22,872✔
253	if(!m_buffer.in_alignment()) {	22,872✔
254	m_buffer.fill_up_with_zeros();	17,817✔
255	ghash_multiply(x, m_buffer.consume(), 1);	17,817✔
256	}
257	}	22,872✔
258
259	void GHASH::ghash_final_block(std::span<uint8_t, GCM_BS> x, uint64_t ad_len, uint64_t text_len) {	14,231✔
260	BOTAN_STATE_CHECK(m_buffer.in_alignment());	14,231✔
261	const auto final_block = store_be(8 * ad_len, 8 * text_len);	14,231✔
262	ghash_multiply(x, final_block, 1);	14,231✔
263	}	14,231✔
264
265	} // namespace Botan

randombit / botan / 23486908132

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous