29630228535

Committed 18 Jul 2026 02:46AM UTC coverage: 89.407% (-2.3%) from 91.66%

Build # 29630228535

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #5740 from randombit/jack/x509-serial-number

Add an explicit type for X509 certificate serial numbers

Coverage Stats

114089 of 127606 relevant lines covered (89.41%)

10801388.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.9

/src/lib/mac/poly1305/poly1305.cpp

/*
* Derived from poly1305-donna-64.h by Andrew Moon <liquidsun@gmail.com>
* in https://github.com/floodyberry/poly1305-donna
*
* (C) 2014 Andrew Moon
* (C) 2014,2025,2026 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/poly1305.h>

#include <botan/exceptn.h>
#include <botan/internal/buffer_slicer.h>
#include <botan/internal/ct_utils.h>
#include <botan/internal/donna128.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_POLY1305_AVX2) || defined(BOTAN_HAS_POLY1305_AVX512)
   #include <botan/internal/cpuid.h>
#endif

namespace Botan {

namespace {

// State layout: pad || accum || r || r^2 || r^3 || ... || r^n
// This ordering allows extending with more powers of r at the end
constexpr size_t PAD_BASE = 0;  // pad[0..1]
constexpr size_t H_BASE = 2;    // h[0..2] (accumulator)
constexpr size_t R_BASE = 5;    // r^1[0..2], r^2[3..5], r^3[6..8], etc.

// Multiply two values in radix 2^44 representation mod (2^130 - 5)
// h = a * b mod p
BOTAN_FORCE_INLINE void poly1305_mul_44(uint64_t& h0,
                                        uint64_t& h1,
                                        uint64_t& h2,
                                        uint64_t a0,
                                        uint64_t a1,
                                        uint64_t a2,
                                        uint64_t b0,
                                        uint64_t b1,
                                        uint64_t b2) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
   typedef donna128 uint128_t;
#endif

   const uint64_t s1 = b1 * 20;
   const uint64_t s2 = b2 * 20;

   const uint128_t d0 = uint128_t(a0) * b0 + uint128_t(a1) * s2 + uint128_t(a2) * s1;
   const uint64_t c0 = carry_shift(d0, 44);

   const uint128_t d1 = uint128_t(a0) * b1 + uint128_t(a1) * b0 + uint128_t(a2) * s2 + c0;
   const uint64_t c1 = carry_shift(d1, 44);

   const uint128_t d2 = uint128_t(a0) * b2 + uint128_t(a1) * b1 + uint128_t(a2) * b0 + c1;
   const uint64_t c2 = carry_shift(d2, 42);

   h0 = (d0 & M44) + c2 * 5;
   h1 = (d1 & M44) + (h0 >> 44);
   h0 &= M44;
   h2 = d2 & M42;
}

// Extend powers of r from current max to target
void poly1305_extend_powers(secure_vector<uint64_t>& X, size_t target_powers) {
   const size_t current_powers = (X.size() - 5) / 3;

   if(current_powers >= target_powers) {
      return;
   }

   // Load r^1 for multiplication
   const uint64_t r0 = X[R_BASE + 0];
   const uint64_t r1 = X[R_BASE + 1];
   const uint64_t r2 = X[R_BASE + 2];

   X.resize(5 + target_powers * 3);

   // Compute r^(current+1) through r^target
   for(size_t i = current_powers + 1; i <= target_powers; ++i) {
      const size_t offset = R_BASE + (i - 1) * 3;
      poly1305_mul_44(
         X[offset + 0], X[offset + 1], X[offset + 2], X[offset - 3], X[offset - 2], X[offset - 1], r0, r1, r2);
   }
}

// Initialize Poly1305 state and precompute powers of r
void poly1305_init(secure_vector<uint64_t>& X, const uint8_t key[32]) {
   X.clear();
   X.reserve(2 + 3 + 2 * 3);
   X.resize(2 + 3 + 3);

   /* Save pad for later (first 2 slots) */
   X[PAD_BASE + 0] = load_le<uint64_t>(key, 2);
   X[PAD_BASE + 1] = load_le<uint64_t>(key, 3);

   /* h = 0 (accumulator, next 3 slots) */
   X[H_BASE + 0] = 0;
   X[H_BASE + 1] = 0;
   X[H_BASE + 2] = 0;

   /* r &= 0xffffffc0ffffffc0ffffffc0fffffff (clamping) */
   const uint64_t t0 = load_le<uint64_t>(key, 0);
   const uint64_t t1 = load_le<uint64_t>(key, 1);

   const uint64_t r0 = (t0) & 0xffc0fffffff;
   const uint64_t r1 = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff;
   const uint64_t r2 = ((t1 >> 24)) & 0x00ffffffc0f;

   // Store r^1
   X[R_BASE + 0] = r0;
   X[R_BASE + 1] = r1;
   X[R_BASE + 2] = r2;

   poly1305_extend_powers(X, 2);
}

// Process a single block: h = (h + m) * r mod p
BOTAN_FORCE_INLINE void poly1305_block_single(uint64_t& h0,
                                              uint64_t& h1,
                                              uint64_t& h2,
                                              uint64_t r0,
                                              uint64_t r1,
                                              uint64_t r2,
                                              uint64_t s1,
                                              uint64_t s2,
                                              const uint8_t* m,
                                              uint64_t hibit) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
   typedef donna128 uint128_t;
#endif

   const uint64_t t0 = load_le<uint64_t>(m, 0);
   const uint64_t t1 = load_le<uint64_t>(m, 1);

   h0 += (t0 & M44);
   h1 += ((t0 >> 44) | (t1 << 20)) & M44;
   h2 += ((t1 >> 24) & M42) | hibit;

   const uint128_t d0 = uint128_t(h0) * r0 + uint128_t(h1) * s2 + uint128_t(h2) * s1;
   const uint64_t c0 = carry_shift(d0, 44);

   const uint128_t d1 = uint128_t(h0) * r1 + uint128_t(h1) * r0 + uint128_t(h2) * s2 + c0;
   const uint64_t c1 = carry_shift(d1, 44);

   const uint128_t d2 = uint128_t(h0) * r2 + uint128_t(h1) * r1 + uint128_t(h2) * r0 + c1;
   const uint64_t c2 = carry_shift(d2, 42);

   h0 = (d0 & M44) + c2 * 5;
   h1 = (d1 & M44) + (h0 >> 44);
   h0 &= M44;
   h2 = d2 & M42;
}

// Process two blocks in parallel: h = ((h + m0) * r + m1) * r = (h + m0) * r^2 + m1 * r
// The multiplications by r^2 and r are independent, enabling ILP
BOTAN_FORCE_INLINE void poly1305_block_pair(uint64_t& h0,
                                            uint64_t& h1,
                                            uint64_t& h2,
                                            uint64_t r0,
                                            uint64_t r1,
                                            uint64_t r2,
                                            uint64_t s1,
                                            uint64_t s2,
                                            uint64_t rr0,
                                            uint64_t rr1,
                                            uint64_t rr2,
                                            uint64_t ss1,
                                            uint64_t ss2,
                                            const uint8_t* m,
                                            uint64_t hibit) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
   typedef donna128 uint128_t;
#endif

   // Load first block (will be multiplied by r^2)
   const uint64_t m0_t0 = load_le<uint64_t>(m, 0);
   const uint64_t m0_t1 = load_le<uint64_t>(m, 1);

   // Load second block (will be multiplied by r)
   const uint64_t m1_t0 = load_le<uint64_t>(m + 16, 0);
   const uint64_t m1_t1 = load_le<uint64_t>(m + 16, 1);

   // Add first block to h
   h0 += (m0_t0 & M44);
   h1 += ((m0_t0 >> 44) | (m0_t1 << 20)) & M44;
   h2 += ((m0_t1 >> 24) & M42) | hibit;

   // Convert second block to limbs
   const uint64_t b0 = (m1_t0 & M44);
   const uint64_t b1 = ((m1_t0 >> 44) | (m1_t1 << 20)) & M44;
   const uint64_t b2 = ((m1_t1 >> 24) & M42) | hibit;

   // Compute (h + m0) * r^2 + m1 * r
   const uint128_t d0 = uint128_t(h0) * rr0 + uint128_t(h1) * ss2 + uint128_t(h2) * ss1 + uint128_t(b0) * r0 +
                        uint128_t(b1) * s2 + uint128_t(b2) * s1;
   const uint64_t c0 = carry_shift(d0, 44);

   const uint128_t d1 = uint128_t(h0) * rr1 + uint128_t(h1) * rr0 + uint128_t(h2) * ss2 + uint128_t(b0) * r1 +
                        uint128_t(b1) * r0 + uint128_t(b2) * s2 + c0;
   const uint64_t c1 = carry_shift(d1, 44);

   const uint128_t d2 = uint128_t(h0) * rr2 + uint128_t(h1) * rr1 + uint128_t(h2) * rr0 + uint128_t(b0) * r2 +
                        uint128_t(b1) * r1 + uint128_t(b2) * r0 + c1;
   const uint64_t c2 = carry_shift(d2, 42);

   h0 = (d0 & M44) + c2 * 5;
   h1 = (d1 & M44) + (h0 >> 44);
   h0 &= M44;
   h2 = d2 & M42;
}

void poly1305_blocks(secure_vector<uint64_t>& X, const uint8_t* m, size_t blocks, bool is_final = false) {
   const uint64_t hibit = is_final ? 0 : (static_cast<uint64_t>(1) << 40);

   // Load r (at R_BASE + 0)
   const uint64_t r0 = X[R_BASE + 0];
   const uint64_t r1 = X[R_BASE + 1];
   const uint64_t r2 = X[R_BASE + 2];
   const uint64_t s1 = r1 * 20;
   const uint64_t s2 = r2 * 20;

   // Load r^2 (at R_BASE + 3)
   const uint64_t rr0 = X[R_BASE + 3];
   const uint64_t rr1 = X[R_BASE + 4];
   const uint64_t rr2 = X[R_BASE + 5];

   // Precompute
   const uint64_t ss1 = rr1 * 20;
   const uint64_t ss2 = rr2 * 20;

   // Load accumulator
   uint64_t h0 = X[H_BASE + 0];
   uint64_t h1 = X[H_BASE + 1];
   uint64_t h2 = X[H_BASE + 2];

   while(blocks >= 2) {
      poly1305_block_pair(h0, h1, h2, r0, r1, r2, s1, s2, rr0, rr1, rr2, ss1, ss2, m, hibit);
      m += 32;
      blocks -= 2;
   }

   // Final block?
   if(blocks > 0) {
      poly1305_block_single(h0, h1, h2, r0, r1, r2, s1, s2, m, hibit);
   }

   // Store accumulator
   X[H_BASE + 0] = h0;
   X[H_BASE + 1] = h1;
   X[H_BASE + 2] = h2;
}

void poly1305_finish(secure_vector<uint64_t>& X, uint8_t mac[16]) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

   /* fully carry h */
   uint64_t h0 = X[H_BASE + 0];
   uint64_t h1 = X[H_BASE + 1];
   uint64_t h2 = X[H_BASE + 2];

   uint64_t c = (h1 >> 44);
   h1 &= M44;
   h2 += c;
   c = (h2 >> 42);
   h2 &= M42;
   h0 += c * 5;
   c = (h0 >> 44);
   h0 &= M44;
   h1 += c;
   c = (h1 >> 44);
   h1 &= M44;
   h2 += c;
   c = (h2 >> 42);
   h2 &= M42;
   h0 += c * 5;
   c = (h0 >> 44);
   h0 &= M44;
   h1 += c;

   /* compute h + -p */
   uint64_t g0 = h0 + 5;
   c = (g0 >> 44);
   g0 &= M44;
   uint64_t g1 = h1 + c;
   c = (g1 >> 44);
   g1 &= M44;
   const uint64_t g2 = h2 + c - (static_cast<uint64_t>(1) << 42);

   /* select h if h < p, or h + -p if h >= p */
   const auto h_mask = CT::Mask<uint64_t>::expand_top_bit(g2);
   h0 = h_mask.select(h0, g0);
   h1 = h_mask.select(h1, g1);
   h2 = h_mask.select(h2, g2);

   /* h = (h + pad) */
   const uint64_t t0 = X[PAD_BASE + 0];
   const uint64_t t1 = X[PAD_BASE + 1];

   h0 += ((t0)&M44);
   c = (h0 >> 44);
   h0 &= M44;
   h1 += (((t0 >> 44) | (t1 << 20)) & M44) + c;
   c = (h1 >> 44);
   h1 &= M44;
   h2 += (((t1 >> 24)) & M42) + c;
   h2 &= M42;

   /* mac = h % (2^128) */
   h0 = ((h0) | (h1 << 44));
   h1 = ((h1 >> 20) | (h2 << 24));

   store_le(mac, h0, h1);

   /* zero out the state */
   clear_mem(X.data(), X.size());
}

}  // namespace

void Poly1305::clear() {
   zap(m_poly);
   m_buffer.clear();
}

bool Poly1305::has_keying_material() const {
   // Minimum size: pad(2) + accum(3) + r(3) + r^2(3) = 11
   return m_poly.size() >= 11;
}

void Poly1305::start_msg(std::span<const uint8_t> nonce) {
   if(!nonce.empty()) {
      throw Invalid_IV_Length(name(), nonce.size());
   }
   assert_key_material_set();

   m_buffer.clear();
   m_poly[H_BASE + 0] = 0;
   m_poly[H_BASE + 1] = 0;
   m_poly[H_BASE + 2] = 0;
}

void Poly1305::key_schedule(std::span<const uint8_t> key) {
   m_buffer.clear();

   poly1305_init(m_poly, key.data());
}

std::string Poly1305::provider() const {
#if defined(BOTAN_HAS_POLY1305_AVX512)
   if(auto feat = CPUID::check(CPUID::Feature::AVX512)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_POLY1305_AVX2)
   if(auto feat = CPUID::check(CPUID::Feature::AVX2)) {
      return *feat;
   }
#endif

   return "base";
}

void Poly1305::add_data(std::span<const uint8_t> input) {
   assert_key_material_set();

   BufferSlicer in(input);

   while(!in.empty()) {
      if(const auto one_block = m_buffer.handle_unaligned_data(in)) {
         poly1305_blocks(m_poly, one_block->data(), 1);
      }

      if(m_buffer.in_alignment()) {
         const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);
         if(full_blocks > 0) {
            const uint8_t* data_ptr = aligned_data.data();
            size_t blocks_remaining = full_blocks;

#if defined(BOTAN_HAS_POLY1305_AVX512)
            if(blocks_remaining >= 8 * 3 && CPUID::has(CPUID::Feature::AVX512)) {
               // Lazily compute r^3 through r^8 on first AVX512 use
               poly1305_extend_powers(m_poly, 8);
               const size_t processed = poly1305_avx512_blocks(m_poly, data_ptr, blocks_remaining);
               data_ptr += processed * 16;
               blocks_remaining -= processed;
            }
#endif

#if defined(BOTAN_HAS_POLY1305_AVX2)
            if(blocks_remaining >= 4 * 6 && CPUID::has(CPUID::Feature::AVX2)) {
               // Lazily compute r^3 and r^4 on first AVX2 use
               poly1305_extend_powers(m_poly, 4);
               const size_t processed = poly1305_avx2_blocks(m_poly, data_ptr, blocks_remaining);
               data_ptr += processed * 16;
               blocks_remaining -= processed;
            }
#endif

            if(blocks_remaining > 0) {
               poly1305_blocks(m_poly, data_ptr, blocks_remaining);
            }
         }
      }
   }
}

void Poly1305::final_result(std::span<uint8_t> out) {
   assert_key_material_set();

   if(!m_buffer.in_alignment()) {
      const uint8_t final_byte = 0x01;
      m_buffer.append({&final_byte, 1});
      m_buffer.fill_up_with_zeros();
      poly1305_blocks(m_poly, m_buffer.consume().data(), 1, true);
   }

   poly1305_finish(m_poly, out.data());

   m_poly.clear();
   m_buffer.clear();
}

}  // namespace Botan

1	/*
2	* Derived from poly1305-donna-64.h by Andrew Moon <liquidsun@gmail.com>
3	* in https://github.com/floodyberry/poly1305-donna
4	*
5	* (C) 2014 Andrew Moon
6	* (C) 2014,2025,2026 Jack Lloyd
7	*
8	* Botan is released under the Simplified BSD License (see license.txt)
9	*/
10
11	#include <botan/internal/poly1305.h>
12
13	#include <botan/exceptn.h>
14	#include <botan/internal/buffer_slicer.h>
15	#include <botan/internal/ct_utils.h>
16	#include <botan/internal/donna128.h>
17	#include <botan/internal/loadstor.h>
18
19	#if defined(BOTAN_HAS_POLY1305_AVX2) \|\| defined(BOTAN_HAS_POLY1305_AVX512)
20	#include <botan/internal/cpuid.h>
21	#endif
22
23	namespace Botan {
24
25	namespace {
26
27	// State layout: pad \|\| accum \|\| r \|\| r^2 \|\| r^3 \|\| ... \|\| r^n
28	// This ordering allows extending with more powers of r at the end
29	constexpr size_t PAD_BASE = 0; // pad[0..1]
30	constexpr size_t H_BASE = 2; // h[0..2] (accumulator)
31	constexpr size_t R_BASE = 5; // r^1[0..2], r^2[3..5], r^3[6..8], etc.
32
33	// Multiply two values in radix 2^44 representation mod (2^130 - 5)
34	// h = a * b mod p
35	BOTAN_FORCE_INLINE void poly1305_mul_44(uint64_t& h0,	164,677✔
36	uint64_t& h1,
37	uint64_t& h2,
38	uint64_t a0,
39	uint64_t a1,
40	uint64_t a2,
41	uint64_t b0,
42	uint64_t b1,
43	uint64_t b2) {
44	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	164,677✔
45	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	164,677✔
46
47	#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
48	typedef donna128 uint128_t;
49	#endif
50
51	const uint64_t s1 = b1 * 20;	164,677✔
52	const uint64_t s2 = b2 * 20;	164,677✔
53
54	const uint128_t d0 = uint128_t(a0) * b0 + uint128_t(a1) * s2 + uint128_t(a2) * s1;	164,677✔
55	const uint64_t c0 = carry_shift(d0, 44);	164,677✔
56
57	const uint128_t d1 = uint128_t(a0) * b1 + uint128_t(a1) * b0 + uint128_t(a2) * s2 + c0;	164,677✔
58	const uint64_t c1 = carry_shift(d1, 44);	164,677✔
59
60	const uint128_t d2 = uint128_t(a0) * b2 + uint128_t(a1) * b1 + uint128_t(a2) * b0 + c1;	164,677✔
61	const uint64_t c2 = carry_shift(d2, 42);	164,677✔
62
63	h0 = (d0 & M44) + c2 * 5;	164,677✔
64	h1 = (d1 & M44) + (h0 >> 44);	164,677✔
65	h0 &= M44;	164,677✔
66	h2 = d2 & M42;	164,677✔
67	}
68
69	// Extend powers of r from current max to target
70	void poly1305_extend_powers(secure_vector<uint64_t>& X, size_t target_powers) {	160,469✔
71	const size_t current_powers = (X.size() - 5) / 3;	160,469✔
72
73	if(current_powers >= target_powers) {	160,469✔
74	return;
75	}
76
77	// Load r^1 for multiplication
78	const uint64_t r0 = X[R_BASE + 0];	160,417✔
79	const uint64_t r1 = X[R_BASE + 1];	160,417✔
80	const uint64_t r2 = X[R_BASE + 2];	160,417✔
81
82	X.resize(5 + target_powers * 3);	160,417✔
83
84	// Compute r^(current+1) through r^target
85	for(size_t i = current_powers + 1; i <= target_powers; ++i) {	325,094✔
86	const size_t offset = R_BASE + (i - 1) * 3;	164,677✔
87	poly1305_mul_44(	164,677✔
88	X[offset + 0], X[offset + 1], X[offset + 2], X[offset - 3], X[offset - 2], X[offset - 1], r0, r1, r2);	164,677✔
89	}
90	}
91
92	// Initialize Poly1305 state and precompute powers of r
93	void poly1305_init(secure_vector<uint64_t>& X, const uint8_t key[32]) {	156,157✔
94	X.clear();	156,157✔
95	X.reserve(2 + 3 + 2 * 3);	156,157✔
96	X.resize(2 + 3 + 3);	156,157✔
97
98	/* Save pad for later (first 2 slots) */
99	X[PAD_BASE + 0] = load_le<uint64_t>(key, 2);	156,157✔
100	X[PAD_BASE + 1] = load_le<uint64_t>(key, 3);	156,157✔
101
102	/* h = 0 (accumulator, next 3 slots) */
103	X[H_BASE + 0] = 0;	156,157✔
104	X[H_BASE + 1] = 0;	156,157✔
105	X[H_BASE + 2] = 0;	156,157✔
106
107	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff (clamping) */
108	const uint64_t t0 = load_le<uint64_t>(key, 0);	156,157✔
109	const uint64_t t1 = load_le<uint64_t>(key, 1);	156,157✔
110
111	const uint64_t r0 = (t0) & 0xffc0fffffff;	156,157✔
112	const uint64_t r1 = ((t0 >> 44) \| (t1 << 20)) & 0xfffffc0ffff;	156,157✔
113	const uint64_t r2 = ((t1 >> 24)) & 0x00ffffffc0f;	156,157✔
114
115	// Store r^1
116	X[R_BASE + 0] = r0;	156,157✔
117	X[R_BASE + 1] = r1;	156,157✔
118	X[R_BASE + 2] = r2;	156,157✔
119
120	poly1305_extend_powers(X, 2);	156,157✔
121	}	156,157✔
122
123	// Process a single block: h = (h + m) * r mod p
124	BOTAN_FORCE_INLINE void poly1305_block_single(uint64_t& h0,	441,027✔
125	uint64_t& h1,
126	uint64_t& h2,
127	uint64_t r0,
128	uint64_t r1,
129	uint64_t r2,
130	uint64_t s1,
131	uint64_t s2,
132	const uint8_t* m,
133	uint64_t hibit) {
134	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	441,027✔
135	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	441,027✔
136
137	#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
138	typedef donna128 uint128_t;
139	#endif
140
141	const uint64_t t0 = load_le<uint64_t>(m, 0);	882,054✔
142	const uint64_t t1 = load_le<uint64_t>(m, 1);	441,027✔
143
144	h0 += (t0 & M44);	441,027✔
145	h1 += ((t0 >> 44) \| (t1 << 20)) & M44;	441,027✔
146	h2 += ((t1 >> 24) & M42) \| hibit;	441,027✔
147
148	const uint128_t d0 = uint128_t(h0) * r0 + uint128_t(h1) * s2 + uint128_t(h2) * s1;	441,027✔
149	const uint64_t c0 = carry_shift(d0, 44);	441,027✔
150
151	const uint128_t d1 = uint128_t(h0) * r1 + uint128_t(h1) * r0 + uint128_t(h2) * s2 + c0;	441,027✔
152	const uint64_t c1 = carry_shift(d1, 44);	441,027✔
153
154	const uint128_t d2 = uint128_t(h0) * r2 + uint128_t(h1) * r1 + uint128_t(h2) * r0 + c1;	441,027✔
155	const uint64_t c2 = carry_shift(d2, 42);	441,027✔
156
157	h0 = (d0 & M44) + c2 * 5;	441,027✔
158	h1 = (d1 & M44) + (h0 >> 44);	441,027✔
159	h0 &= M44;	441,027✔
160	h2 = d2 & M42;	441,027✔
161	}
162
163	// Process two blocks in parallel: h = ((h + m0) * r + m1) * r = (h + m0) * r^2 + m1 * r
164	// The multiplications by r^2 and r are independent, enabling ILP
165	BOTAN_FORCE_INLINE void poly1305_block_pair(uint64_t& h0,	145,399✔
166	uint64_t& h1,
167	uint64_t& h2,
168	uint64_t r0,
169	uint64_t r1,
170	uint64_t r2,
171	uint64_t s1,
172	uint64_t s2,
173	uint64_t rr0,
174	uint64_t rr1,
175	uint64_t rr2,
176	uint64_t ss1,
177	uint64_t ss2,
178	const uint8_t* m,
179	uint64_t hibit) {
180	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	145,399✔
181	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	145,399✔
182
183	#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
184	typedef donna128 uint128_t;
185	#endif
186
187	// Load first block (will be multiplied by r^2)
188	const uint64_t m0_t0 = load_le<uint64_t>(m, 0);	290,798✔
189	const uint64_t m0_t1 = load_le<uint64_t>(m, 1);	145,399✔
190
191	// Load second block (will be multiplied by r)
192	const uint64_t m1_t0 = load_le<uint64_t>(m + 16, 0);	145,399✔
193	const uint64_t m1_t1 = load_le<uint64_t>(m + 16, 1);	145,399✔
194
195	// Add first block to h
196	h0 += (m0_t0 & M44);	145,399✔
197	h1 += ((m0_t0 >> 44) \| (m0_t1 << 20)) & M44;	145,399✔
198	h2 += ((m0_t1 >> 24) & M42) \| hibit;	145,399✔
199
200	// Convert second block to limbs
201	const uint64_t b0 = (m1_t0 & M44);	145,399✔
202	const uint64_t b1 = ((m1_t0 >> 44) \| (m1_t1 << 20)) & M44;	145,399✔
203	const uint64_t b2 = ((m1_t1 >> 24) & M42) \| hibit;	145,399✔
204
205	// Compute (h + m0) * r^2 + m1 * r
206	const uint128_t d0 = uint128_t(h0) * rr0 + uint128_t(h1) * ss2 + uint128_t(h2) * ss1 + uint128_t(b0) * r0 +	145,399✔
207	uint128_t(b1) * s2 + uint128_t(b2) * s1;	145,399✔
208	const uint64_t c0 = carry_shift(d0, 44);	145,399✔
209
210	const uint128_t d1 = uint128_t(h0) * rr1 + uint128_t(h1) * rr0 + uint128_t(h2) * ss2 + uint128_t(b0) * r1 +	145,399✔
211	uint128_t(b1) * r0 + uint128_t(b2) * s2 + c0;	145,399✔
212	const uint64_t c1 = carry_shift(d1, 44);	145,399✔
213
214	const uint128_t d2 = uint128_t(h0) * rr2 + uint128_t(h1) * rr1 + uint128_t(h2) * rr0 + uint128_t(b0) * r2 +	145,399✔
215	uint128_t(b1) * r1 + uint128_t(b2) * r0 + c1;	145,399✔
216	const uint64_t c2 = carry_shift(d2, 42);	145,399✔
217
218	h0 = (d0 & M44) + c2 * 5;	145,399✔
219	h1 = (d1 & M44) + (h0 >> 44);	145,399✔
220	h0 &= M44;	145,399✔
221	h2 = d2 & M42;	145,399✔
222	}
223
224	void poly1305_blocks(secure_vector<uint64_t>& X, const uint8_t* m, size_t blocks, bool is_final = false) {	478,837✔
225	const uint64_t hibit = is_final ? 0 : (static_cast<uint64_t>(1) << 40);	478,837✔
226
227	// Load r (at R_BASE + 0)
228	const uint64_t r0 = X[R_BASE + 0];	478,837✔
229	const uint64_t r1 = X[R_BASE + 1];	478,837✔
230	const uint64_t r2 = X[R_BASE + 2];	478,837✔
231	const uint64_t s1 = r1 * 20;	478,837✔
232	const uint64_t s2 = r2 * 20;	478,837✔
233
234	// Load r^2 (at R_BASE + 3)
235	const uint64_t rr0 = X[R_BASE + 3];	478,837✔
236	const uint64_t rr1 = X[R_BASE + 4];	478,837✔
237	const uint64_t rr2 = X[R_BASE + 5];	478,837✔
238
239	// Precompute
240	const uint64_t ss1 = rr1 * 20;	478,837✔
241	const uint64_t ss2 = rr2 * 20;	478,837✔
242
243	// Load accumulator
244	uint64_t h0 = X[H_BASE + 0];	478,837✔
245	uint64_t h1 = X[H_BASE + 1];	478,837✔
246	uint64_t h2 = X[H_BASE + 2];	478,837✔
247
248	while(blocks >= 2) {	624,236✔
249	poly1305_block_pair(h0, h1, h2, r0, r1, r2, s1, s2, rr0, rr1, rr2, ss1, ss2, m, hibit);	145,399✔
250	m += 32;	145,399✔
251	blocks -= 2;	145,399✔
252	}
253
254	// Final block?
255	if(blocks > 0) {	478,837✔
256	poly1305_block_single(h0, h1, h2, r0, r1, r2, s1, s2, m, hibit);	882,054✔
257	}
258
259	// Store accumulator
260	X[H_BASE + 0] = h0;	478,837✔
261	X[H_BASE + 1] = h1;	478,837✔
262	X[H_BASE + 2] = h2;	478,837✔
263	}	478,837✔
264
265	void poly1305_finish(secure_vector<uint64_t>& X, uint8_t mac[16]) {	128,055✔
266	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	128,055✔
267	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	128,055✔
268
269	/* fully carry h */
270	uint64_t h0 = X[H_BASE + 0];	128,055✔
271	uint64_t h1 = X[H_BASE + 1];	128,055✔
272	uint64_t h2 = X[H_BASE + 2];	128,055✔
273
274	uint64_t c = (h1 >> 44);	128,055✔
275	h1 &= M44;	128,055✔
276	h2 += c;	128,055✔
277	c = (h2 >> 42);	128,055✔
278	h2 &= M42;	128,055✔
279	h0 += c * 5;	128,055✔
280	c = (h0 >> 44);	128,055✔
281	h0 &= M44;	128,055✔
282	h1 += c;	128,055✔
283	c = (h1 >> 44);	128,055✔
284	h1 &= M44;	128,055✔
285	h2 += c;	128,055✔
286	c = (h2 >> 42);	128,055✔
287	h2 &= M42;	128,055✔
288	h0 += c * 5;	128,055✔
289	c = (h0 >> 44);	128,055✔
290	h0 &= M44;	128,055✔
291	h1 += c;	128,055✔
292
293	/* compute h + -p */
294	uint64_t g0 = h0 + 5;	128,055✔
295	c = (g0 >> 44);	128,055✔
296	g0 &= M44;	128,055✔
297	uint64_t g1 = h1 + c;	128,055✔
298	c = (g1 >> 44);	128,055✔
299	g1 &= M44;	128,055✔
300	const uint64_t g2 = h2 + c - (static_cast<uint64_t>(1) << 42);	128,055✔
301
302	/* select h if h < p, or h + -p if h >= p */
303	const auto h_mask = CT::Mask<uint64_t>::expand_top_bit(g2);	128,055✔
304	h0 = h_mask.select(h0, g0);	128,055✔
305	h1 = h_mask.select(h1, g1);	128,055✔
306	h2 = h_mask.select(h2, g2);	128,055✔
307
308	/* h = (h + pad) */
309	const uint64_t t0 = X[PAD_BASE + 0];	128,055✔
310	const uint64_t t1 = X[PAD_BASE + 1];	128,055✔
311
312	h0 += ((t0)&M44);	128,055✔
313	c = (h0 >> 44);	128,055✔
314	h0 &= M44;	128,055✔
315	h1 += (((t0 >> 44) \| (t1 << 20)) & M44) + c;	128,055✔
316	c = (h1 >> 44);	128,055✔
317	h1 &= M44;	128,055✔
318	h2 += (((t1 >> 24)) & M42) + c;	128,055✔
319	h2 &= M42;	128,055✔
320
321	/* mac = h % (2^128) */
322	h0 = ((h0) \| (h1 << 44));	128,055✔
323	h1 = ((h1 >> 20) \| (h2 << 24));	128,055✔
324
325	store_le(mac, h0, h1);	128,055✔
326
327	/* zero out the state */
328	clear_mem(X.data(), X.size());	128,055✔
329	}	128,055✔
330
331	} // namespace
332
333	void Poly1305::clear() {	7,334✔
334	zap(m_poly);	7,334✔
335	m_buffer.clear();	7,334✔
336	}	7,334✔
337
338	bool Poly1305::has_keying_material() const {	1,319,869✔
339	// Minimum size: pad(2) + accum(3) + r(3) + r^2(3) = 11
340	return m_poly.size() >= 11;	1,319,869✔
341	}
342
343	void Poly1305::start_msg(std::span<const uint8_t> nonce) {	2,448✔
344	if(!nonce.empty()) {	2,448✔
345	throw Invalid_IV_Length(name(), nonce.size());	×
346	}
347	assert_key_material_set();	2,448✔
348
349	m_buffer.clear();	2,448✔
350	m_poly[H_BASE + 0] = 0;	2,448✔
351	m_poly[H_BASE + 1] = 0;	2,448✔
352	m_poly[H_BASE + 2] = 0;	2,448✔
353	}	2,448✔
354
355	void Poly1305::key_schedule(std::span<const uint8_t> key) {	156,157✔
356	m_buffer.clear();	156,157✔
357
358	poly1305_init(m_poly, key.data());	156,157✔
359	}	156,157✔
360
361	std::string Poly1305::provider() const {	246✔
362	#if defined(BOTAN_HAS_POLY1305_AVX512)
363	if(auto feat = CPUID::check(CPUID::Feature::AVX512)) {	246✔
364	return *feat;	×
365	}	×
366	#endif
367
368	#if defined(BOTAN_HAS_POLY1305_AVX2)
369	if(auto feat = CPUID::check(CPUID::Feature::AVX2)) {	246✔
370	return *feat;	246✔
371	}	123✔
372	#endif
373
374	return "base";	123✔
375	}
376
377	void Poly1305::add_data(std::span<const uint8_t> input) {	1,188,382✔
378	assert_key_material_set();	1,188,382✔
379
380	BufferSlicer in(input);	1,187,890✔
381
382	while(!in.empty()) {	3,614,265✔
383	if(const auto one_block = m_buffer.handle_unaligned_data(in)) {	1,238,485✔
384	poly1305_blocks(m_poly, one_block->data(), 1);	400,994✔
385	}
386
387	if(m_buffer.in_alignment()) {	1,238,485✔
388	const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);	481,025✔
389	if(full_blocks > 0) {	481,025✔
390	const uint8_t* data_ptr = aligned_data.data();	80,347✔
391	size_t blocks_remaining = full_blocks;	80,347✔
392
393	#if defined(BOTAN_HAS_POLY1305_AVX512)
394	if(blocks_remaining >= 8 * 3 && CPUID::has(CPUID::Feature::AVX512)) {	80,347✔
395	// Lazily compute r^3 through r^8 on first AVX512 use
396	poly1305_extend_powers(m_poly, 8);	×
397	const size_t processed = poly1305_avx512_blocks(m_poly, data_ptr, blocks_remaining);	×
398	data_ptr += processed * 16;	×
399	blocks_remaining -= processed;	×
400	}
401	#endif
402
403	#if defined(BOTAN_HAS_POLY1305_AVX2)
404	if(blocks_remaining >= 4 * 6 && CPUID::has(CPUID::Feature::AVX2)) {	80,347✔
405	// Lazily compute r^3 and r^4 on first AVX2 use
406	poly1305_extend_powers(m_poly, 4);	4,312✔
407	const size_t processed = poly1305_avx2_blocks(m_poly, data_ptr, blocks_remaining);	4,312✔
408	data_ptr += processed * 16;	4,312✔
409	blocks_remaining -= processed;	4,312✔
410	}
411	#endif
412
413	if(blocks_remaining > 0) {	80,347✔
414	poly1305_blocks(m_poly, data_ptr, blocks_remaining);	76,619✔
415	}
416	}
417	}
418	}
419	}	1,187,890✔
420
421	void Poly1305::final_result(std::span<uint8_t> out) {	128,301✔
422	assert_key_material_set();	128,301✔
423
424	if(!m_buffer.in_alignment()) {	128,055✔
425	const uint8_t final_byte = 0x01;	1,224✔
426	m_buffer.append({&final_byte, 1});	1,224✔
427	m_buffer.fill_up_with_zeros();	1,224✔
428	poly1305_blocks(m_poly, m_buffer.consume().data(), 1, true);	1,224✔
429	}
430
431	poly1305_finish(m_poly, out.data());	128,055✔
432
433	m_poly.clear();	128,055✔
434	m_buffer.clear();	128,055✔
435	}	128,055✔
436
437	} // namespace Botan

randombit / botan / 29630228535

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous