23389682302

Committed 21 Mar 2026 05:47PM UTC coverage: 89.396% (-0.005%) from 89.401%

Build # 23389682302

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #5474 from randombit/jack/ctr-avx2

Add AVX2 codepath for CTR counter handling and XOR

Run Details

104458 of 116848 relevant lines covered (89.4%)

11641508.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.7

/src/lib/stream/ctr/ctr.cpp

/*
* Counter mode
* (C) 1999-2011,2014 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/ctr.h>

#include <botan/exceptn.h>
#include <botan/internal/bit_ops.h>
#include <botan/internal/fmt.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_CTR_BE_AVX2)
   #include <botan/internal/cpuid.h>
#endif

namespace Botan {

CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher) :
      m_cipher(std::move(cipher)),
      m_block_size(m_cipher->block_size()),
      m_ctr_size(m_block_size),
      m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),
      m_counter(m_cipher->parallel_bytes()),
      m_pad(m_counter.size()),
      m_pad_pos(0) {}

CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher, size_t ctr_size) :
      m_cipher(std::move(cipher)),
      m_block_size(m_cipher->block_size()),
      m_ctr_size(ctr_size),
      m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),
      m_counter(m_cipher->parallel_bytes()),
      m_pad(m_counter.size()),
      m_pad_pos(0) {
   BOTAN_ARG_CHECK(m_ctr_size >= 4 && m_ctr_size <= m_block_size, "Invalid CTR-BE counter size");
}

void CTR_BE::clear() {
   m_cipher->clear();
   zeroise(m_pad);
   zeroise(m_counter);
   zap(m_iv);
   m_pad_pos = 0;
}

size_t CTR_BE::default_iv_length() const {
   return m_block_size;
}

bool CTR_BE::valid_iv_length(size_t iv_len) const {
   return (iv_len <= m_block_size);
}

size_t CTR_BE::buffer_size() const {
   return m_pad.size();
}

Key_Length_Specification CTR_BE::key_spec() const {
   return m_cipher->key_spec();
}

std::unique_ptr<StreamCipher> CTR_BE::new_object() const {
   return std::make_unique<CTR_BE>(m_cipher->new_object(), m_ctr_size);
}

bool CTR_BE::has_keying_material() const {
   return m_cipher->has_keying_material();
}

void CTR_BE::key_schedule(std::span<const uint8_t> key) {
   m_cipher->set_key(key);

   // Set a default all-zeros IV
   set_iv(nullptr, 0);
}

std::string CTR_BE::name() const {
   if(m_ctr_size == m_block_size) {
      return fmt("CTR-BE({})", m_cipher->name());
   } else {
      return fmt("CTR-BE({},{})", m_cipher->name(), m_ctr_size);
   }
}

void CTR_BE::cipher_bytes(const uint8_t in[], uint8_t out[], size_t length) {
   assert_key_material_set();

   const uint8_t* pad_bits = m_pad.data();
   const size_t pad_size = m_pad.size();

   /* Consume any already computed keystream in m_pad */

   if(m_pad_pos > 0) {
      const size_t avail = pad_size - m_pad_pos;
      const size_t take = std::min(length, avail);
      xor_buf(out, in, pad_bits + m_pad_pos, take);
      length -= take;
      in += take;
      out += take;
      m_pad_pos += take;

      if(take == avail) {
         add_counter(m_ctr_blocks);
         m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
         m_pad_pos = 0;
      }
   }

   /* Bulk processing */

#if defined(BOTAN_HAS_CTR_BE_AVX2)
   if(length >= pad_size && m_block_size == 16 && m_ctr_size == 4 && pad_size % 64 == 0 &&
      CPUID::has(CPUID::Feature::AVX2)) {
      const size_t consumed = ctr_proc_bs16_ctr4_avx2(in, out, length);
      in += consumed;
      out += consumed;
      length -= consumed;
   }
#endif

   while(length >= pad_size) {
      xor_buf(out, in, pad_bits, pad_size);
      length -= pad_size;
      in += pad_size;
      out += pad_size;

      add_counter(m_ctr_blocks);
      m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
   }

   /* Now if length > 0 then we have some remaining text, and m_pad is full - consume as required */
   if(length > 0) {
      xor_buf(out, in, pad_bits, length);
      m_pad_pos = length;
   }
}

void CTR_BE::generate_keystream(uint8_t out[], size_t length) {
   assert_key_material_set();

   const size_t avail = m_pad.size() - m_pad_pos;
   const size_t take = std::min(length, avail);
   copy_mem(out, &m_pad[m_pad_pos], take);
   length -= take;
   out += take;
   m_pad_pos += take;

   while(length >= m_pad.size()) {
      add_counter(m_ctr_blocks);
      m_cipher->encrypt_n(m_counter.data(), out, m_ctr_blocks);

      length -= m_pad.size();
      out += m_pad.size();
   }

   if(m_pad_pos == m_pad.size()) {
      add_counter(m_ctr_blocks);
      m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
      m_pad_pos = 0;
   }

   copy_mem(out, m_pad.data(), length);
   m_pad_pos += length;
   BOTAN_ASSERT_NOMSG(m_pad_pos < m_pad.size());
}

void CTR_BE::set_iv_bytes(const uint8_t iv[], size_t iv_len) {
   if(!valid_iv_length(iv_len)) {
      throw Invalid_IV_Length(name(), iv_len);
   }

   m_iv.resize(m_block_size);
   zeroise(m_iv);
   copy_mem(m_iv.data(), iv, iv_len);

   seek(0);
}

void CTR_BE::add_counter(const uint64_t counter) {
   const size_t ctr_size = m_ctr_size;
   const size_t ctr_blocks = m_ctr_blocks;
   const size_t BS = m_block_size;

   if(ctr_size == 4) {
      const size_t off = (BS - 4);
      const uint32_t low32 = static_cast<uint32_t>(counter + load_be<uint32_t>(&m_counter[off], 0));

      for(size_t i = 0; i != ctr_blocks; ++i) {
         store_be(uint32_t(low32 + i), &m_counter[i * BS + off]);
      }
   } else if(ctr_size == 8) {
      const size_t off = (BS - 8);
      const uint64_t low64 = counter + load_be<uint64_t>(&m_counter[off], 0);

      for(size_t i = 0; i != ctr_blocks; ++i) {
         store_be(uint64_t(low64 + i), &m_counter[i * BS + off]);
      }
   } else if(ctr_size == 16) {
      const size_t off = (BS - 16);
      uint64_t b0 = load_be<uint64_t>(&m_counter[off], 0);
      uint64_t b1 = load_be<uint64_t>(&m_counter[off], 1);
      b1 += counter;
      b0 += (b1 < counter) ? 1 : 0;  // carry

      for(size_t i = 0; i != ctr_blocks; ++i) {
         store_be(b0, &m_counter[i * BS + off]);
         store_be(b1, &m_counter[i * BS + off + 8]);
         b1 += 1;
         if(b1 == 0) {
            b0 += 1;  // carry
         }
      }
   } else {
      for(size_t i = 0; i != ctr_blocks; ++i) {
         uint64_t local_counter = counter;
         uint16_t carry = static_cast<uint8_t>(local_counter);
         for(size_t j = 0; (carry > 0 || local_counter > 0) && j != ctr_size; ++j) {
            const size_t off = i * BS + (BS - 1 - j);
            const uint16_t cnt = static_cast<uint16_t>(m_counter[off]) + carry;
            m_counter[off] = static_cast<uint8_t>(cnt);
            local_counter = (local_counter >> 8);
            carry = (cnt >> 8) + static_cast<uint8_t>(local_counter);
         }
      }
   }
}

void CTR_BE::seek(uint64_t offset) {
   assert_key_material_set();

   const uint64_t base_counter = m_ctr_blocks * (offset / m_counter.size());

   zeroise(m_counter);
   BOTAN_ASSERT_NOMSG(m_counter.size() >= m_iv.size());
   copy_mem(m_counter.data(), m_iv.data(), m_iv.size());

   const size_t BS = m_block_size;

   // Set m_counter blocks to IV, IV + 1, ... IV + n

   if(m_ctr_size == 4 && BS >= 8) {
      const uint32_t low32 = load_be<uint32_t>(&m_counter[BS - 4], 0);

      if(m_ctr_blocks >= 4 && is_power_of_2(m_ctr_blocks)) {
         size_t written = 1;
         while(written < m_ctr_blocks) {
            copy_mem(&m_counter[written * BS], &m_counter[0], BS * written);  // NOLINT(*container-data-pointer)
            written *= 2;
         }
      } else {
         for(size_t i = 1; i != m_ctr_blocks; ++i) {
            copy_mem(&m_counter[i * BS], &m_counter[0], BS - 4);  // NOLINT(*container-data-pointer)
         }
      }

      for(size_t i = 1; i != m_ctr_blocks; ++i) {
         const uint32_t c = static_cast<uint32_t>(low32 + i);
         store_be(c, &m_counter[(BS - 4) + i * BS]);
      }
   } else {
      // do everything sequentially:
      for(size_t i = 1; i != m_ctr_blocks; ++i) {
         copy_mem(&m_counter[i * BS], &m_counter[(i - 1) * BS], BS);

         for(size_t j = 0; j != m_ctr_size; ++j) {
            uint8_t& c = m_counter[i * BS + (BS - 1 - j)];
            c += 1;
            if(c > 0) {
               break;
            }
         }
      }
   }

   if(base_counter > 0) {
      add_counter(base_counter);
   }

   m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
   m_pad_pos = offset % m_counter.size();
}
}  // namespace Botan

1	/*
2	* Counter mode
3	* (C) 1999-2011,2014 Jack Lloyd
4	*
5	* Botan is released under the Simplified BSD License (see license.txt)
6	*/
7
8	#include <botan/internal/ctr.h>
9
10	#include <botan/exceptn.h>
11	#include <botan/internal/bit_ops.h>
12	#include <botan/internal/fmt.h>
13	#include <botan/internal/loadstor.h>
14
15	#if defined(BOTAN_HAS_CTR_BE_AVX2)
16	#include <botan/internal/cpuid.h>
17	#endif
18
19	namespace Botan {
20
21	CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher) :	1,023✔
22	m_cipher(std::move(cipher)),	1,023✔
23	m_block_size(m_cipher->block_size()),	1,023✔
24	m_ctr_size(m_block_size),	1,023✔
25	m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),	1,023✔
26	m_counter(m_cipher->parallel_bytes()),	2,046✔
27	m_pad(m_counter.size()),	1,023✔
28	m_pad_pos(0) {}	2,046✔
29
30	CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher, size_t ctr_size) :	100,401✔
31	m_cipher(std::move(cipher)),	100,401✔
32	m_block_size(m_cipher->block_size()),	100,401✔
33	m_ctr_size(ctr_size),	100,401✔
34	m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),	100,401✔
35	m_counter(m_cipher->parallel_bytes()),	200,802✔
36	m_pad(m_counter.size()),	100,401✔
37	m_pad_pos(0) {	200,802✔
38	BOTAN_ARG_CHECK(m_ctr_size >= 4 && m_ctr_size <= m_block_size, "Invalid CTR-BE counter size");	100,401✔
39	}	100,401✔
40
41	void CTR_BE::clear() {	7,502✔
42	m_cipher->clear();	7,502✔
43	zeroise(m_pad);	7,502✔
44	zeroise(m_counter);	7,502✔
45	zap(m_iv);	7,502✔
46	m_pad_pos = 0;	7,502✔
47	}	7,502✔
48
49	size_t CTR_BE::default_iv_length() const {	3,276✔
50	return m_block_size;	3,276✔
51	}
52
53	bool CTR_BE::valid_iv_length(size_t iv_len) const {	326,519✔
54	return (iv_len <= m_block_size);	326,519✔
55	}
56
57	size_t CTR_BE::buffer_size() const {	1,659✔
58	return m_pad.size();	1,659✔
59	}
60
61	Key_Length_Specification CTR_BE::key_spec() const {	207,279✔
62	return m_cipher->key_spec();	207,279✔
63	}
64
65	std::unique_ptr<StreamCipher> CTR_BE::new_object() const {	1,638✔
66	return std::make_unique<CTR_BE>(m_cipher->new_object(), m_ctr_size);	1,638✔
67	}
68
69	bool CTR_BE::has_keying_material() const {	20,957,771✔
70	return m_cipher->has_keying_material();	20,957,771✔
71	}
72
73	void CTR_BE::key_schedule(std::span<const uint8_t> key) {	104,458✔
74	m_cipher->set_key(key);	104,458✔
75
76	// Set a default all-zeros IV
77	set_iv(nullptr, 0);	104,458✔
78	}	104,458✔
79
80	std::string CTR_BE::name() const {	14,103✔
81	if(m_ctr_size == m_block_size) {	14,103✔
82	return fmt("CTR-BE({})", m_cipher->name());	13,065✔
83	} else {
84	return fmt("CTR-BE({},{})", m_cipher->name(), m_ctr_size);	1,038✔
85	}
86	}
87
88	void CTR_BE::cipher_bytes(const uint8_t in[], uint8_t out[], size_t length) {	131,834✔
89	assert_key_material_set();	131,834✔
90
91	const uint8_t* pad_bits = m_pad.data();	127,882✔
92	const size_t pad_size = m_pad.size();	127,882✔
93
94	/* Consume any already computed keystream in m_pad */
95
96	if(m_pad_pos > 0) {	127,882✔
97	const size_t avail = pad_size - m_pad_pos;	106,371✔
98	const size_t take = std::min(length, avail);	106,371✔
99	xor_buf(out, in, pad_bits + m_pad_pos, take);	106,371✔
100	length -= take;	106,371✔
101	in += take;	106,371✔
102	out += take;	106,371✔
103	m_pad_pos += take;	106,371✔
104
105	if(take == avail) {	106,371✔
106	add_counter(m_ctr_blocks);	2,806✔
107	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	2,806✔
108	m_pad_pos = 0;	2,806✔
109	}
110	}
111
112	/* Bulk processing */
113
114	#if defined(BOTAN_HAS_CTR_BE_AVX2)
115	if(length >= pad_size && m_block_size == 16 && m_ctr_size == 4 && pad_size % 64 == 0 &&	127,882✔
116	CPUID::has(CPUID::Feature::AVX2)) {	1,638✔
117	const size_t consumed = ctr_proc_bs16_ctr4_avx2(in, out, length);	1,635✔
118	in += consumed;	1,635✔
119	out += consumed;	1,635✔
120	length -= consumed;	1,635✔
121	}
122	#endif
123
124	while(length >= pad_size) {	128,040✔
125	xor_buf(out, in, pad_bits, pad_size);	158✔
126	length -= pad_size;	158✔
127	in += pad_size;	158✔
128	out += pad_size;	158✔
129
130	add_counter(m_ctr_blocks);	158✔
131	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	158✔
132	}
133
134	/* Now if length > 0 then we have some remaining text, and m_pad is full - consume as required */
135	if(length > 0) {	127,882✔
136	xor_buf(out, in, pad_bits, length);	23,562✔
137	m_pad_pos = length;	23,562✔
138	}
139	}	127,882✔
140
141	void CTR_BE::generate_keystream(uint8_t out[], size_t length) {	20,583,568✔
142	assert_key_material_set();	20,583,568✔
143
144	const size_t avail = m_pad.size() - m_pad_pos;	20,583,568✔
145	const size_t take = std::min(length, avail);	20,583,568✔
146	copy_mem(out, &m_pad[m_pad_pos], take);	20,583,568✔
147	length -= take;	20,583,568✔
148	out += take;	20,583,568✔
149	m_pad_pos += take;	20,583,568✔
150
151	while(length >= m_pad.size()) {	20,583,568✔
152	add_counter(m_ctr_blocks);	×
153	m_cipher->encrypt_n(m_counter.data(), out, m_ctr_blocks);	×
154
155	length -= m_pad.size();	×
156	out += m_pad.size();	×
157	}
158
159	if(m_pad_pos == m_pad.size()) {	20,583,568✔
160	add_counter(m_ctr_blocks);	84,459✔
161	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	84,459✔
162	m_pad_pos = 0;	84,459✔
163	}
164
165	copy_mem(out, m_pad.data(), length);	20,583,568✔
166	m_pad_pos += length;	20,583,568✔
167	BOTAN_ASSERT_NOMSG(m_pad_pos < m_pad.size());	20,583,568✔
168	}	20,583,568✔
169
170	void CTR_BE::set_iv_bytes(const uint8_t iv[], size_t iv_len) {	227,478✔
171	if(!valid_iv_length(iv_len)) {	227,478✔
172	throw Invalid_IV_Length(name(), iv_len);	3,284✔
173	}
174
175	m_iv.resize(m_block_size);	225,836✔
176	zeroise(m_iv);	225,836✔
177	copy_mem(m_iv.data(), iv, iv_len);	225,836✔
178
179	seek(0);	225,836✔
180	}	223,884✔
181
182	void CTR_BE::add_counter(const uint64_t counter) {	87,447✔
183	const size_t ctr_size = m_ctr_size;	87,447✔
184	const size_t ctr_blocks = m_ctr_blocks;	87,447✔
185	const size_t BS = m_block_size;	87,447✔
186
187	if(ctr_size == 4) {	87,447✔
188	const size_t off = (BS - 4);	2,791✔
189	const uint32_t low32 = static_cast<uint32_t>(counter + load_be<uint32_t>(&m_counter[off], 0));	2,791✔
190
191	for(size_t i = 0; i != ctr_blocks; ++i) {	66,343✔
192	store_be(uint32_t(low32 + i), &m_counter[i * BS + off]);	63,552✔
193	}
194	} else if(ctr_size == 8) {	84,656✔
195	const size_t off = (BS - 8);	38✔
196	const uint64_t low64 = counter + load_be<uint64_t>(&m_counter[off], 0);	38✔
197
198	for(size_t i = 0; i != ctr_blocks; ++i) {	1,574✔
199	store_be(uint64_t(low64 + i), &m_counter[i * BS + off]);	1,536✔
200	}
201	} else if(ctr_size == 16) {	84,618✔
202	const size_t off = (BS - 16);	84,618✔
203	uint64_t b0 = load_be<uint64_t>(&m_counter[off], 0);	84,618✔
204	uint64_t b1 = load_be<uint64_t>(&m_counter[off], 1);	84,618✔
205	b1 += counter;	84,618✔
206	b0 += (b1 < counter) ? 1 : 0; // carry	84,618✔
207
208	for(size_t i = 0; i != ctr_blocks; ++i) {	2,741,490✔
209	store_be(b0, &m_counter[i * BS + off]);	2,656,872✔
210	store_be(b1, &m_counter[i * BS + off + 8]);	2,656,872✔
211	b1 += 1;	2,656,872✔
212	if(b1 == 0) {	2,656,872✔
213	b0 += 1; // carry	×
214	}
215	}
216	} else {
217	for(size_t i = 0; i != ctr_blocks; ++i) {	×
218	uint64_t local_counter = counter;	×
219	uint16_t carry = static_cast<uint8_t>(local_counter);	×
220	for(size_t j = 0; (carry > 0 \|\| local_counter > 0) && j != ctr_size; ++j) {	×
221	const size_t off = i * BS + (BS - 1 - j);	×
222	const uint16_t cnt = static_cast<uint16_t>(m_counter[off]) + carry;	×
223	m_counter[off] = static_cast<uint8_t>(cnt);	×
224	local_counter = (local_counter >> 8);	×
225	carry = (cnt >> 8) + static_cast<uint8_t>(local_counter);	×
226	}
227	}
228	}
229	}	87,447✔
230
231	void CTR_BE::seek(uint64_t offset) {	230,674✔
232	assert_key_material_set();	230,674✔
233
234	const uint64_t base_counter = m_ctr_blocks * (offset / m_counter.size());	227,084✔
235
236	zeroise(m_counter);	227,084✔
237	BOTAN_ASSERT_NOMSG(m_counter.size() >= m_iv.size());	227,084✔
238	copy_mem(m_counter.data(), m_iv.data(), m_iv.size());	227,084✔
239
240	const size_t BS = m_block_size;	227,084✔
241
242	// Set m_counter blocks to IV, IV + 1, ... IV + n
243
244	if(m_ctr_size == 4 && BS >= 8) {	227,084✔
245	const uint32_t low32 = load_be<uint32_t>(&m_counter[BS - 4], 0);	17,593✔
246
247	if(m_ctr_blocks >= 4 && is_power_of_2(m_ctr_blocks)) {	17,593✔
248	size_t written = 1;
249	while(written < m_ctr_blocks) {	103,086✔
250	copy_mem(&m_counter[written * BS], &m_counter[0], BS * written); // NOLINT(*container-data-pointer)	85,493✔
251	written *= 2;	85,493✔
252	}
253	} else {
254	for(size_t i = 1; i != m_ctr_blocks; ++i) {	×
255	copy_mem(&m_counter[i * BS], &m_counter[0], BS - 4); // NOLINT(*container-data-pointer)	×
256	}
257	}
258
259	for(size_t i = 1; i != m_ctr_blocks; ++i) {	539,864✔
260	const uint32_t c = static_cast<uint32_t>(low32 + i);	522,271✔
261	store_be(c, &m_counter[(BS - 4) + i * BS]);	522,271✔
262	}
263	} else {
264	// do everything sequentially:
265	for(size_t i = 1; i != m_ctr_blocks; ++i) {	7,050,164✔
266	copy_mem(&m_counter[i * BS], &m_counter[(i - 1) * BS], BS);	6,840,673✔
267
268	for(size_t j = 0; j != m_ctr_size; ++j) {	6,846,850✔
269	uint8_t& c = m_counter[i * BS + (BS - 1 - j)];	6,846,793✔
270	c += 1;	6,846,793✔
271	if(c > 0) {	6,846,793✔
272	break;
273	}
274	}
275	}
276	}
277
278	if(base_counter > 0) {	227,084✔
279	add_counter(base_counter);	24✔
280	}
281
282	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	227,084✔
283	m_pad_pos = offset % m_counter.size();	227,084✔
284	}	227,084✔
285	} // namespace Botan

randombit / botan / 23389682302

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous