23093753908

Committed 14 Mar 2026 06:26PM UTC coverage: 89.727% (-0.4%) from 90.107%

Build # 23093753908

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #5442 from randombit/jack/camellia-avx512-gfni

Camellia GFNI: new AVX-512 implementation and improved AVX2 implementation

Run Details

104206 of 116137 relevant lines covered (89.73%)

11503294.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.3

/src/lib/stream/ctr/ctr.cpp

/*
* Counter mode
* (C) 1999-2011,2014 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/ctr.h>

#include <botan/exceptn.h>
#include <botan/internal/bit_ops.h>
#include <botan/internal/fmt.h>
#include <botan/internal/loadstor.h>

namespace Botan {

CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher) :
      m_cipher(std::move(cipher)),
      m_block_size(m_cipher->block_size()),
      m_ctr_size(m_block_size),
      m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),
      m_counter(m_cipher->parallel_bytes()),
      m_pad(m_counter.size()),
      m_pad_pos(0) {}

CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher, size_t ctr_size) :
      m_cipher(std::move(cipher)),
      m_block_size(m_cipher->block_size()),
      m_ctr_size(ctr_size),
      m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),
      m_counter(m_cipher->parallel_bytes()),
      m_pad(m_counter.size()),
      m_pad_pos(0) {
   BOTAN_ARG_CHECK(m_ctr_size >= 4 && m_ctr_size <= m_block_size, "Invalid CTR-BE counter size");
}

void CTR_BE::clear() {
   m_cipher->clear();
   zeroise(m_pad);
   zeroise(m_counter);
   zap(m_iv);
   m_pad_pos = 0;
}

size_t CTR_BE::default_iv_length() const {
   return m_block_size;
}

bool CTR_BE::valid_iv_length(size_t iv_len) const {
   return (iv_len <= m_block_size);
}

size_t CTR_BE::buffer_size() const {
   return m_pad.size();
}

Key_Length_Specification CTR_BE::key_spec() const {
   return m_cipher->key_spec();
}

std::unique_ptr<StreamCipher> CTR_BE::new_object() const {
   return std::make_unique<CTR_BE>(m_cipher->new_object(), m_ctr_size);
}

bool CTR_BE::has_keying_material() const {
   return m_cipher->has_keying_material();
}

void CTR_BE::key_schedule(std::span<const uint8_t> key) {
   m_cipher->set_key(key);

   // Set a default all-zeros IV
   set_iv(nullptr, 0);
}

std::string CTR_BE::name() const {
   if(m_ctr_size == m_block_size) {
      return fmt("CTR-BE({})", m_cipher->name());
   } else {
      return fmt("CTR-BE({},{})", m_cipher->name(), m_ctr_size);
   }
}

void CTR_BE::cipher_bytes(const uint8_t in[], uint8_t out[], size_t length) {
   assert_key_material_set();

   const uint8_t* pad_bits = m_pad.data();
   const size_t pad_size = m_pad.size();

   if(m_pad_pos > 0) {
      const size_t avail = pad_size - m_pad_pos;
      const size_t take = std::min(length, avail);
      xor_buf(out, in, pad_bits + m_pad_pos, take);
      length -= take;
      in += take;
      out += take;
      m_pad_pos += take;

      if(take == avail) {
         add_counter(m_ctr_blocks);
         m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
         m_pad_pos = 0;
      }
   }

   while(length >= pad_size) {
      xor_buf(out, in, pad_bits, pad_size);
      length -= pad_size;
      in += pad_size;
      out += pad_size;

      add_counter(m_ctr_blocks);
      m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
   }

   xor_buf(out, in, pad_bits, length);
   m_pad_pos += length;
}

void CTR_BE::generate_keystream(uint8_t out[], size_t length) {
   assert_key_material_set();

   const size_t avail = m_pad.size() - m_pad_pos;
   const size_t take = std::min(length, avail);
   copy_mem(out, &m_pad[m_pad_pos], take);
   length -= take;
   out += take;
   m_pad_pos += take;

   while(length >= m_pad.size()) {
      add_counter(m_ctr_blocks);
      m_cipher->encrypt_n(m_counter.data(), out, m_ctr_blocks);

      length -= m_pad.size();
      out += m_pad.size();
   }

   if(m_pad_pos == m_pad.size()) {
      add_counter(m_ctr_blocks);
      m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
      m_pad_pos = 0;
   }

   copy_mem(out, m_pad.data(), length);
   m_pad_pos += length;
   BOTAN_ASSERT_NOMSG(m_pad_pos < m_pad.size());
}

void CTR_BE::set_iv_bytes(const uint8_t iv[], size_t iv_len) {
   if(!valid_iv_length(iv_len)) {
      throw Invalid_IV_Length(name(), iv_len);
   }

   m_iv.resize(m_block_size);
   zeroise(m_iv);
   copy_mem(m_iv.data(), iv, iv_len);

   seek(0);
}

void CTR_BE::add_counter(const uint64_t counter) {
   const size_t ctr_size = m_ctr_size;
   const size_t ctr_blocks = m_ctr_blocks;
   const size_t BS = m_block_size;

   if(ctr_size == 4) {
      const size_t off = (BS - 4);
      const uint32_t low32 = static_cast<uint32_t>(counter + load_be<uint32_t>(&m_counter[off], 0));

      for(size_t i = 0; i != ctr_blocks; ++i) {
         store_be(uint32_t(low32 + i), &m_counter[i * BS + off]);
      }
   } else if(ctr_size == 8) {
      const size_t off = (BS - 8);
      const uint64_t low64 = counter + load_be<uint64_t>(&m_counter[off], 0);

      for(size_t i = 0; i != ctr_blocks; ++i) {
         store_be(uint64_t(low64 + i), &m_counter[i * BS + off]);
      }
   } else if(ctr_size == 16) {
      const size_t off = (BS - 16);
      uint64_t b0 = load_be<uint64_t>(&m_counter[off], 0);
      uint64_t b1 = load_be<uint64_t>(&m_counter[off], 1);
      b1 += counter;
      b0 += (b1 < counter) ? 1 : 0;  // carry

      for(size_t i = 0; i != ctr_blocks; ++i) {
         store_be(b0, &m_counter[i * BS + off]);
         store_be(b1, &m_counter[i * BS + off + 8]);
         b1 += 1;
         if(b1 == 0) {
            b0 += 1;  // carry
         }
      }
   } else {
      for(size_t i = 0; i != ctr_blocks; ++i) {
         uint64_t local_counter = counter;
         uint16_t carry = static_cast<uint8_t>(local_counter);
         for(size_t j = 0; (carry > 0 || local_counter > 0) && j != ctr_size; ++j) {
            const size_t off = i * BS + (BS - 1 - j);
            const uint16_t cnt = static_cast<uint16_t>(m_counter[off]) + carry;
            m_counter[off] = static_cast<uint8_t>(cnt);
            local_counter = (local_counter >> 8);
            carry = (cnt >> 8) + static_cast<uint8_t>(local_counter);
         }
      }
   }
}

void CTR_BE::seek(uint64_t offset) {
   assert_key_material_set();

   const uint64_t base_counter = m_ctr_blocks * (offset / m_counter.size());

   zeroise(m_counter);
   BOTAN_ASSERT_NOMSG(m_counter.size() >= m_iv.size());
   copy_mem(m_counter.data(), m_iv.data(), m_iv.size());

   const size_t BS = m_block_size;

   // Set m_counter blocks to IV, IV + 1, ... IV + n

   if(m_ctr_size == 4 && BS >= 8) {
      const uint32_t low32 = load_be<uint32_t>(&m_counter[BS - 4], 0);

      if(m_ctr_blocks >= 4 && is_power_of_2(m_ctr_blocks)) {
         size_t written = 1;
         while(written < m_ctr_blocks) {
            copy_mem(&m_counter[written * BS], &m_counter[0], BS * written);  // NOLINT(*container-data-pointer)
            written *= 2;
         }
      } else {
         for(size_t i = 1; i != m_ctr_blocks; ++i) {
            copy_mem(&m_counter[i * BS], &m_counter[0], BS - 4);  // NOLINT(*container-data-pointer)
         }
      }

      for(size_t i = 1; i != m_ctr_blocks; ++i) {
         const uint32_t c = static_cast<uint32_t>(low32 + i);
         store_be(c, &m_counter[(BS - 4) + i * BS]);
      }
   } else {
      // do everything sequentially:
      for(size_t i = 1; i != m_ctr_blocks; ++i) {
         copy_mem(&m_counter[i * BS], &m_counter[(i - 1) * BS], BS);

         for(size_t j = 0; j != m_ctr_size; ++j) {
            uint8_t& c = m_counter[i * BS + (BS - 1 - j)];
            c += 1;
            if(c > 0) {
               break;
            }
         }
      }
   }

   if(base_counter > 0) {
      add_counter(base_counter);
   }

   m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
   m_pad_pos = offset % m_counter.size();
}
}  // namespace Botan

1	/*
2	* Counter mode
3	* (C) 1999-2011,2014 Jack Lloyd
4	*
5	* Botan is released under the Simplified BSD License (see license.txt)
6	*/
7
8	#include <botan/internal/ctr.h>
9
10	#include <botan/exceptn.h>
11	#include <botan/internal/bit_ops.h>
12	#include <botan/internal/fmt.h>
13	#include <botan/internal/loadstor.h>
14
15	namespace Botan {
16
17	CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher) :	1,023✔
18	m_cipher(std::move(cipher)),	1,023✔
19	m_block_size(m_cipher->block_size()),	1,023✔
20	m_ctr_size(m_block_size),	1,023✔
21	m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),	1,023✔
22	m_counter(m_cipher->parallel_bytes()),	2,046✔
23	m_pad(m_counter.size()),	1,023✔
24	m_pad_pos(0) {}	3,069✔
25
26	CTR_BE::CTR_BE(std::unique_ptr<BlockCipher> cipher, size_t ctr_size) :	98,079✔
27	m_cipher(std::move(cipher)),	98,079✔
28	m_block_size(m_cipher->block_size()),	98,079✔
29	m_ctr_size(ctr_size),	98,079✔
30	m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),	98,079✔
31	m_counter(m_cipher->parallel_bytes()),	196,158✔
32	m_pad(m_counter.size()),	98,079✔
33	m_pad_pos(0) {	196,158✔
34	BOTAN_ARG_CHECK(m_ctr_size >= 4 && m_ctr_size <= m_block_size, "Invalid CTR-BE counter size");	98,079✔
35	}	98,079✔
36
37	void CTR_BE::clear() {	6,627✔
38	m_cipher->clear();	6,627✔
39	zeroise(m_pad);	6,627✔
40	zeroise(m_counter);	6,627✔
41	zap(m_iv);	6,627✔
42	m_pad_pos = 0;	6,627✔
43	}	6,627✔
44
45	size_t CTR_BE::default_iv_length() const {	1,638✔
46	return m_block_size;	1,638✔
47	}
48
49	bool CTR_BE::valid_iv_length(size_t iv_len) const {	316,731✔
50	return (iv_len <= m_block_size);	316,731✔
51	}
52
53	size_t CTR_BE::buffer_size() const {	840✔
54	return m_pad.size();	840✔
55	}
56
57	Key_Length_Specification CTR_BE::key_spec() const {	203,550✔
58	return m_cipher->key_spec();	203,550✔
59	}
60
61	std::unique_ptr<StreamCipher> CTR_BE::new_object() const {	819✔
62	return std::make_unique<CTR_BE>(m_cipher->new_object(), m_ctr_size);	819✔
63	}
64
65	bool CTR_BE::has_keying_material() const {	20,911,511✔
66	return m_cipher->has_keying_material();	20,911,511✔
67	}
68
69	void CTR_BE::key_schedule(std::span<const uint8_t> key) {	102,184✔
70	m_cipher->set_key(key);	102,184✔
71
72	// Set a default all-zeros IV
73	set_iv(nullptr, 0);	102,184✔
74	}	102,184✔
75
76	std::string CTR_BE::name() const {	7,467✔
77	if(m_ctr_size == m_block_size) {	7,467✔
78	return fmt("CTR-BE({})", m_cipher->name());	6,545✔
79	} else {
80	return fmt("CTR-BE({},{})", m_cipher->name(), m_ctr_size);	922✔
81	}
82	}
83
84	void CTR_BE::cipher_bytes(const uint8_t in[], uint8_t out[], size_t length) {	69,087✔
85	assert_key_material_set();	69,087✔
86
87	const uint8_t* pad_bits = m_pad.data();	66,857✔
88	const size_t pad_size = m_pad.size();	66,857✔
89
90	if(m_pad_pos > 0) {	66,857✔
91	const size_t avail = pad_size - m_pad_pos;	47,829✔
92	const size_t take = std::min(length, avail);	47,829✔
93	xor_buf(out, in, pad_bits + m_pad_pos, take);	47,829✔
94	length -= take;	47,829✔
95	in += take;	47,829✔
96	out += take;	47,829✔
97	m_pad_pos += take;	47,829✔
98
99	if(take == avail) {	47,829✔
100	add_counter(m_ctr_blocks);	1,871✔
101	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	1,871✔
102	m_pad_pos = 0;	1,871✔
103	}
104	}
105
106	while(length >= pad_size) {	71,999✔
107	xor_buf(out, in, pad_bits, pad_size);	5,142✔
108	length -= pad_size;	5,142✔
109	in += pad_size;	5,142✔
110	out += pad_size;	5,142✔
111
112	add_counter(m_ctr_blocks);	5,142✔
113	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	5,142✔
114	}
115
116	xor_buf(out, in, pad_bits, length);	66,857✔
117	m_pad_pos += length;	66,857✔
118	}	66,857✔
119
120	void CTR_BE::generate_keystream(uint8_t out[], size_t length) {	20,612,719✔
121	assert_key_material_set();	20,612,719✔
122
123	const size_t avail = m_pad.size() - m_pad_pos;	20,612,719✔
124	const size_t take = std::min(length, avail);	20,612,719✔
125	copy_mem(out, &m_pad[m_pad_pos], take);	20,612,719✔
126	length -= take;	20,612,719✔
127	out += take;	20,612,719✔
128	m_pad_pos += take;	20,612,719✔
129
130	while(length >= m_pad.size()) {	20,612,719✔
131	add_counter(m_ctr_blocks);	×
132	m_cipher->encrypt_n(m_counter.data(), out, m_ctr_blocks);	×
133
134	length -= m_pad.size();	×
135	out += m_pad.size();	×
136	}
137
138	if(m_pad_pos == m_pad.size()) {	20,612,719✔
139	add_counter(m_ctr_blocks);	84,918✔
140	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	84,918✔
141	m_pad_pos = 0;	84,918✔
142	}
143
144	copy_mem(out, m_pad.data(), length);	20,612,719✔
145	m_pad_pos += length;	20,612,719✔
146	BOTAN_ASSERT_NOMSG(m_pad_pos < m_pad.size());	20,612,719✔
147	}	20,612,719✔
148
149	void CTR_BE::set_iv_bytes(const uint8_t iv[], size_t iv_len) {	219,886✔
150	if(!valid_iv_length(iv_len)) {	219,886✔
151	throw Invalid_IV_Length(name(), iv_len);	1,646✔
152	}
153
154	m_iv.resize(m_block_size);	219,063✔
155	zeroise(m_iv);	219,063✔
156	copy_mem(m_iv.data(), iv, iv_len);	219,063✔
157
158	seek(0);	219,063✔
159	}	217,930✔
160
161	void CTR_BE::add_counter(const uint64_t counter) {	91,943✔
162	const size_t ctr_size = m_ctr_size;	91,943✔
163	const size_t ctr_blocks = m_ctr_blocks;	91,943✔
164	const size_t BS = m_block_size;	91,943✔
165
166	if(ctr_size == 4) {	91,943✔
167	const size_t off = (BS - 4);	6,912✔
168	const uint32_t low32 = static_cast<uint32_t>(counter + load_be<uint32_t>(&m_counter[off], 0));	6,912✔
169
170	for(size_t i = 0; i != ctr_blocks; ++i) {	185,872✔
171	store_be(uint32_t(low32 + i), &m_counter[i * BS + off]);	178,960✔
172	}
173	} else if(ctr_size == 8) {	85,031✔
174	const size_t off = (BS - 8);	24✔
175	const uint64_t low64 = counter + load_be<uint64_t>(&m_counter[off], 0);	24✔
176
177	for(size_t i = 0; i != ctr_blocks; ++i) {	952✔
178	store_be(uint64_t(low64 + i), &m_counter[i * BS + off]);	928✔
179	}
180	} else if(ctr_size == 16) {	85,007✔
181	const size_t off = (BS - 16);	85,007✔
182	uint64_t b0 = load_be<uint64_t>(&m_counter[off], 0);	85,007✔
183	uint64_t b1 = load_be<uint64_t>(&m_counter[off], 1);	85,007✔
184	b1 += counter;	85,007✔
185	b0 += (b1 < counter) ? 1 : 0; // carry	85,007✔
186
187	for(size_t i = 0; i != ctr_blocks; ++i) {	2,747,607✔
188	store_be(b0, &m_counter[i * BS + off]);	2,662,600✔
189	store_be(b1, &m_counter[i * BS + off + 8]);	2,662,600✔
190	b1 += 1;	2,662,600✔
191	if(b1 == 0) {	2,662,600✔
192	b0 += 1; // carry	×
193	}
194	}
195	} else {
196	for(size_t i = 0; i != ctr_blocks; ++i) {	×
197	uint64_t local_counter = counter;	×
198	uint16_t carry = static_cast<uint8_t>(local_counter);	×
199	for(size_t j = 0; (carry > 0 \|\| local_counter > 0) && j != ctr_size; ++j) {	×
200	const size_t off = i * BS + (BS - 1 - j);	×
201	const uint16_t cnt = static_cast<uint16_t>(m_counter[off]) + carry;	×
202	m_counter[off] = static_cast<uint8_t>(cnt);	×
203	local_counter = (local_counter >> 8);	×
204	carry = (cnt >> 8) + static_cast<uint8_t>(local_counter);	×
205	}
206	}
207	}
208	}	91,943✔
209
210	void CTR_BE::seek(uint64_t offset) {	221,482✔
211	assert_key_material_set();	221,482✔
212
213	const uint64_t base_counter = m_ctr_blocks * (offset / m_counter.size());	219,530✔
214
215	zeroise(m_counter);	219,530✔
216	BOTAN_ASSERT_NOMSG(m_counter.size() >= m_iv.size());	219,530✔
217	copy_mem(m_counter.data(), m_iv.data(), m_iv.size());	219,530✔
218
219	const size_t BS = m_block_size;	219,530✔
220
221	// Set m_counter blocks to IV, IV + 1, ... IV + n
222
223	if(m_ctr_size == 4 && BS >= 8) {	219,530✔
224	const uint32_t low32 = load_be<uint32_t>(&m_counter[BS - 4], 0);	16,844✔
225
226	if(m_ctr_blocks >= 4 && is_power_of_2(m_ctr_blocks)) {	16,844✔
227	size_t written = 1;
228	while(written < m_ctr_blocks) {	99,498✔
229	copy_mem(&m_counter[written * BS], &m_counter[0], BS * written); // NOLINT(*container-data-pointer)	82,654✔
230	written *= 2;	82,654✔
231	}
232	} else {
233	for(size_t i = 1; i != m_ctr_blocks; ++i) {	×
234	copy_mem(&m_counter[i * BS], &m_counter[0], BS - 4); // NOLINT(*container-data-pointer)	×
235	}
236	}
237
238	for(size_t i = 1; i != m_ctr_blocks; ++i) {	524,352✔
239	const uint32_t c = static_cast<uint32_t>(low32 + i);	507,508✔
240	store_be(c, &m_counter[(BS - 4) + i * BS]);	507,508✔
241	}
242	} else {
243	// do everything sequentially:
244	for(size_t i = 1; i != m_ctr_blocks; ++i) {	6,665,384✔
245	copy_mem(&m_counter[i * BS], &m_counter[(i - 1) * BS], BS);	6,462,698✔
246
247	for(size_t j = 0; j != m_ctr_size; ++j) {	6,466,440✔
248	uint8_t& c = m_counter[i * BS + (BS - 1 - j)];	6,466,403✔
249	c += 1;	6,466,403✔
250	if(c > 0) {	6,466,403✔
251	break;
252	}
253	}
254	}
255	}
256
257	if(base_counter > 0) {	219,530✔
258	add_counter(base_counter);	12✔
259	}
260
261	m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);	219,530✔
262	m_pad_pos = offset % m_counter.size();	219,530✔
263	}	219,530✔
264	} // namespace Botan

randombit / botan / 23093753908

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous