25457312714

Committed 06 May 2026 07:43PM UTC coverage: 89.331% (-2.3%) from 91.667%

Build # 25457312714

Build Type

push

github

Committed by

randombit

Commit Message

In TLS 1.3 verification of client certs, check the correct extension for OCSP

This was checking if the client asked us (the server) for OCSP, instead of
checking if we asked the client for OCSP when we sent the CertificateRequest.

Coverage Stats

107574 of 120422 relevant lines covered (89.33%)

11482758.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.24

/src/lib/mac/poly1305/poly1305.cpp

/*
* Derived from poly1305-donna-64.h by Andrew Moon <liquidsun@gmail.com>
* in https://github.com/floodyberry/poly1305-donna
*
* (C) 2014 Andrew Moon
* (C) 2014,2025,2026 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/poly1305.h>

#include <botan/internal/buffer_slicer.h>
#include <botan/internal/ct_utils.h>
#include <botan/internal/donna128.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_HAS_POLY1305_AVX2) || defined(BOTAN_HAS_POLY1305_AVX512)
   #include <botan/internal/cpuid.h>
#endif

namespace Botan {

namespace {

// State layout: pad || accum || r || r^2 || r^3 || ... || r^n
// This ordering allows extending with more powers of r at the end
constexpr size_t PAD_BASE = 0;  // pad[0..1]
constexpr size_t H_BASE = 2;    // h[0..2] (accumulator)
constexpr size_t R_BASE = 5;    // r^1[0..2], r^2[3..5], r^3[6..8], etc.

// Multiply two values in radix 2^44 representation mod (2^130 - 5)
// h = a * b mod p
BOTAN_FORCE_INLINE void poly1305_mul_44(uint64_t& h0,
                                        uint64_t& h1,
                                        uint64_t& h2,
                                        uint64_t a0,
                                        uint64_t a1,
                                        uint64_t a2,
                                        uint64_t b0,
                                        uint64_t b1,
                                        uint64_t b2) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
   typedef donna128 uint128_t;
#endif

   const uint64_t s1 = b1 * 20;
   const uint64_t s2 = b2 * 20;

   const uint128_t d0 = uint128_t(a0) * b0 + uint128_t(a1) * s2 + uint128_t(a2) * s1;
   const uint64_t c0 = carry_shift(d0, 44);

   const uint128_t d1 = uint128_t(a0) * b1 + uint128_t(a1) * b0 + uint128_t(a2) * s2 + c0;
   const uint64_t c1 = carry_shift(d1, 44);

   const uint128_t d2 = uint128_t(a0) * b2 + uint128_t(a1) * b1 + uint128_t(a2) * b0 + c1;
   const uint64_t c2 = carry_shift(d2, 42);

   h0 = (d0 & M44) + c2 * 5;
   h1 = (d1 & M44) + (h0 >> 44);
   h0 &= M44;
   h2 = d2 & M42;
}

// Extend powers of r from current max to target
void poly1305_extend_powers(secure_vector<uint64_t>& X, size_t target_powers) {
   const size_t current_powers = (X.size() - 5) / 3;

   if(current_powers >= target_powers) {
      return;
   }

   // Load r^1 for multiplication
   const uint64_t r0 = X[R_BASE + 0];
   const uint64_t r1 = X[R_BASE + 1];
   const uint64_t r2 = X[R_BASE + 2];

   X.resize(5 + target_powers * 3);

   // Compute r^(current+1) through r^target
   for(size_t i = current_powers + 1; i <= target_powers; ++i) {
      const size_t offset = R_BASE + (i - 1) * 3;
      poly1305_mul_44(
         X[offset + 0], X[offset + 1], X[offset + 2], X[offset - 3], X[offset - 2], X[offset - 1], r0, r1, r2);
   }
}

// Initialize Poly1305 state and precompute powers of r
void poly1305_init(secure_vector<uint64_t>& X, const uint8_t key[32]) {
   X.clear();
   X.reserve(2 + 3 + 2 * 3);
   X.resize(2 + 3 + 3);

   /* Save pad for later (first 2 slots) */
   X[PAD_BASE + 0] = load_le<uint64_t>(key, 2);
   X[PAD_BASE + 1] = load_le<uint64_t>(key, 3);

   /* h = 0 (accumulator, next 3 slots) */
   X[H_BASE + 0] = 0;
   X[H_BASE + 1] = 0;
   X[H_BASE + 2] = 0;

   /* r &= 0xffffffc0ffffffc0ffffffc0fffffff (clamping) */
   const uint64_t t0 = load_le<uint64_t>(key, 0);
   const uint64_t t1 = load_le<uint64_t>(key, 1);

   const uint64_t r0 = (t0) & 0xffc0fffffff;
   const uint64_t r1 = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff;
   const uint64_t r2 = ((t1 >> 24)) & 0x00ffffffc0f;

   // Store r^1
   X[R_BASE + 0] = r0;
   X[R_BASE + 1] = r1;
   X[R_BASE + 2] = r2;

   poly1305_extend_powers(X, 2);
}

// Process a single block: h = (h + m) * r mod p
BOTAN_FORCE_INLINE void poly1305_block_single(uint64_t& h0,
                                              uint64_t& h1,
                                              uint64_t& h2,
                                              uint64_t r0,
                                              uint64_t r1,
                                              uint64_t r2,
                                              uint64_t s1,
                                              uint64_t s2,
                                              const uint8_t* m,
                                              uint64_t hibit) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
   typedef donna128 uint128_t;
#endif

   const uint64_t t0 = load_le<uint64_t>(m, 0);
   const uint64_t t1 = load_le<uint64_t>(m, 1);

   h0 += (t0 & M44);
   h1 += ((t0 >> 44) | (t1 << 20)) & M44;
   h2 += ((t1 >> 24) & M42) | hibit;

   const uint128_t d0 = uint128_t(h0) * r0 + uint128_t(h1) * s2 + uint128_t(h2) * s1;
   const uint64_t c0 = carry_shift(d0, 44);

   const uint128_t d1 = uint128_t(h0) * r1 + uint128_t(h1) * r0 + uint128_t(h2) * s2 + c0;
   const uint64_t c1 = carry_shift(d1, 44);

   const uint128_t d2 = uint128_t(h0) * r2 + uint128_t(h1) * r1 + uint128_t(h2) * r0 + c1;
   const uint64_t c2 = carry_shift(d2, 42);

   h0 = (d0 & M44) + c2 * 5;
   h1 = (d1 & M44) + (h0 >> 44);
   h0 &= M44;
   h2 = d2 & M42;
}

// Process two blocks in parallel: h = ((h + m0) * r + m1) * r = (h + m0) * r^2 + m1 * r
// The multiplications by r^2 and r are independent, enabling ILP
BOTAN_FORCE_INLINE void poly1305_block_pair(uint64_t& h0,
                                            uint64_t& h1,
                                            uint64_t& h2,
                                            uint64_t r0,
                                            uint64_t r1,
                                            uint64_t r2,
                                            uint64_t s1,
                                            uint64_t s2,
                                            uint64_t rr0,
                                            uint64_t rr1,
                                            uint64_t rr2,
                                            uint64_t ss1,
                                            uint64_t ss2,
                                            const uint8_t* m,
                                            uint64_t hibit) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
   typedef donna128 uint128_t;
#endif

   // Load first block (will be multiplied by r^2)
   const uint64_t m0_t0 = load_le<uint64_t>(m, 0);
   const uint64_t m0_t1 = load_le<uint64_t>(m, 1);

   // Load second block (will be multiplied by r)
   const uint64_t m1_t0 = load_le<uint64_t>(m + 16, 0);
   const uint64_t m1_t1 = load_le<uint64_t>(m + 16, 1);

   // Add first block to h
   h0 += (m0_t0 & M44);
   h1 += ((m0_t0 >> 44) | (m0_t1 << 20)) & M44;
   h2 += ((m0_t1 >> 24) & M42) | hibit;

   // Convert second block to limbs
   const uint64_t b0 = (m1_t0 & M44);
   const uint64_t b1 = ((m1_t0 >> 44) | (m1_t1 << 20)) & M44;
   const uint64_t b2 = ((m1_t1 >> 24) & M42) | hibit;

   // Compute (h + m0) * r^2 + m1 * r
   const uint128_t d0 = uint128_t(h0) * rr0 + uint128_t(h1) * ss2 + uint128_t(h2) * ss1 + uint128_t(b0) * r0 +
                        uint128_t(b1) * s2 + uint128_t(b2) * s1;
   const uint64_t c0 = carry_shift(d0, 44);

   const uint128_t d1 = uint128_t(h0) * rr1 + uint128_t(h1) * rr0 + uint128_t(h2) * ss2 + uint128_t(b0) * r1 +
                        uint128_t(b1) * r0 + uint128_t(b2) * s2 + c0;
   const uint64_t c1 = carry_shift(d1, 44);

   const uint128_t d2 = uint128_t(h0) * rr2 + uint128_t(h1) * rr1 + uint128_t(h2) * rr0 + uint128_t(b0) * r2 +
                        uint128_t(b1) * r1 + uint128_t(b2) * r0 + c1;
   const uint64_t c2 = carry_shift(d2, 42);

   h0 = (d0 & M44) + c2 * 5;
   h1 = (d1 & M44) + (h0 >> 44);
   h0 &= M44;
   h2 = d2 & M42;
}

void poly1305_blocks(secure_vector<uint64_t>& X, const uint8_t* m, size_t blocks, bool is_final = false) {
   const uint64_t hibit = is_final ? 0 : (static_cast<uint64_t>(1) << 40);

   // Load r (at R_BASE + 0)
   const uint64_t r0 = X[R_BASE + 0];
   const uint64_t r1 = X[R_BASE + 1];
   const uint64_t r2 = X[R_BASE + 2];
   const uint64_t s1 = r1 * 20;
   const uint64_t s2 = r2 * 20;

   // Load r^2 (at R_BASE + 3)
   const uint64_t rr0 = X[R_BASE + 3];
   const uint64_t rr1 = X[R_BASE + 4];
   const uint64_t rr2 = X[R_BASE + 5];

   // Precompute
   const uint64_t ss1 = rr1 * 20;
   const uint64_t ss2 = rr2 * 20;

   // Load accumulator
   uint64_t h0 = X[H_BASE + 0];
   uint64_t h1 = X[H_BASE + 1];
   uint64_t h2 = X[H_BASE + 2];

   while(blocks >= 2) {
      poly1305_block_pair(h0, h1, h2, r0, r1, r2, s1, s2, rr0, rr1, rr2, ss1, ss2, m, hibit);
      m += 32;
      blocks -= 2;
   }

   // Final block?
   if(blocks > 0) {
      poly1305_block_single(h0, h1, h2, r0, r1, r2, s1, s2, m, hibit);
   }

   // Store accumulator
   X[H_BASE + 0] = h0;
   X[H_BASE + 1] = h1;
   X[H_BASE + 2] = h2;
}

void poly1305_finish(secure_vector<uint64_t>& X, uint8_t mac[16]) {
   constexpr uint64_t M44 = 0xFFFFFFFFFFF;
   constexpr uint64_t M42 = 0x3FFFFFFFFFF;

   /* fully carry h */
   uint64_t h0 = X[H_BASE + 0];
   uint64_t h1 = X[H_BASE + 1];
   uint64_t h2 = X[H_BASE + 2];

   uint64_t c = (h1 >> 44);
   h1 &= M44;
   h2 += c;
   c = (h2 >> 42);
   h2 &= M42;
   h0 += c * 5;
   c = (h0 >> 44);
   h0 &= M44;
   h1 += c;
   c = (h1 >> 44);
   h1 &= M44;
   h2 += c;
   c = (h2 >> 42);
   h2 &= M42;
   h0 += c * 5;
   c = (h0 >> 44);
   h0 &= M44;
   h1 += c;

   /* compute h + -p */
   uint64_t g0 = h0 + 5;
   c = (g0 >> 44);
   g0 &= M44;
   uint64_t g1 = h1 + c;
   c = (g1 >> 44);
   g1 &= M44;
   const uint64_t g2 = h2 + c - (static_cast<uint64_t>(1) << 42);

   /* select h if h < p, or h + -p if h >= p */
   const auto h_mask = CT::Mask<uint64_t>::expand_top_bit(g2);
   h0 = h_mask.select(h0, g0);
   h1 = h_mask.select(h1, g1);
   h2 = h_mask.select(h2, g2);

   /* h = (h + pad) */
   const uint64_t t0 = X[PAD_BASE + 0];
   const uint64_t t1 = X[PAD_BASE + 1];

   h0 += ((t0)&M44);
   c = (h0 >> 44);
   h0 &= M44;
   h1 += (((t0 >> 44) | (t1 << 20)) & M44) + c;
   c = (h1 >> 44);
   h1 &= M44;
   h2 += (((t1 >> 24)) & M42) + c;
   h2 &= M42;

   /* mac = h % (2^128) */
   h0 = ((h0) | (h1 << 44));
   h1 = ((h1 >> 20) | (h2 << 24));

   store_le(mac, h0, h1);

   /* zero out the state */
   clear_mem(X.data(), X.size());
}

}  // namespace

void Poly1305::clear() {
   zap(m_poly);
   m_buffer.clear();
}

bool Poly1305::has_keying_material() const {
   // Minimum size: pad(2) + accum(3) + r(3) + r^2(3) = 11
   return m_poly.size() >= 11;
}

void Poly1305::key_schedule(std::span<const uint8_t> key) {
   m_buffer.clear();

   poly1305_init(m_poly, key.data());
}

std::string Poly1305::provider() const {
#if defined(BOTAN_HAS_POLY1305_AVX512)
   if(auto feat = CPUID::check(CPUID::Feature::AVX512)) {
      return *feat;
   }
#endif

#if defined(BOTAN_HAS_POLY1305_AVX2)
   if(auto feat = CPUID::check(CPUID::Feature::AVX2)) {
      return *feat;
   }
#endif

   return "base";
}

void Poly1305::add_data(std::span<const uint8_t> input) {
   assert_key_material_set();

   BufferSlicer in(input);

   while(!in.empty()) {
      if(const auto one_block = m_buffer.handle_unaligned_data(in)) {
         poly1305_blocks(m_poly, one_block->data(), 1);
      }

      if(m_buffer.in_alignment()) {
         const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);
         if(full_blocks > 0) {
            const uint8_t* data_ptr = aligned_data.data();
            size_t blocks_remaining = full_blocks;

#if defined(BOTAN_HAS_POLY1305_AVX512)
            if(blocks_remaining >= 8 * 3 && CPUID::has(CPUID::Feature::AVX512)) {
               // Lazily compute r^3 through r^8 on first AVX512 use
               poly1305_extend_powers(m_poly, 8);
               const size_t processed = poly1305_avx512_blocks(m_poly, data_ptr, blocks_remaining);
               data_ptr += processed * 16;
               blocks_remaining -= processed;
            }
#endif

#if defined(BOTAN_HAS_POLY1305_AVX2)
            if(blocks_remaining >= 4 * 6 && CPUID::has(CPUID::Feature::AVX2)) {
               // Lazily compute r^3 and r^4 on first AVX2 use
               poly1305_extend_powers(m_poly, 4);
               const size_t processed = poly1305_avx2_blocks(m_poly, data_ptr, blocks_remaining);
               data_ptr += processed * 16;
               blocks_remaining -= processed;
            }
#endif

            if(blocks_remaining > 0) {
               poly1305_blocks(m_poly, data_ptr, blocks_remaining);
            }
         }
      }
   }
}

void Poly1305::final_result(std::span<uint8_t> out) {
   assert_key_material_set();

   if(!m_buffer.in_alignment()) {
      const uint8_t final_byte = 0x01;
      m_buffer.append({&final_byte, 1});
      m_buffer.fill_up_with_zeros();
      poly1305_blocks(m_poly, m_buffer.consume().data(), 1, true);
   }

   poly1305_finish(m_poly, out.data());

   m_poly.clear();
   m_buffer.clear();
}

}  // namespace Botan

1	/*
2	* Derived from poly1305-donna-64.h by Andrew Moon <liquidsun@gmail.com>
3	* in https://github.com/floodyberry/poly1305-donna
4	*
5	* (C) 2014 Andrew Moon
6	* (C) 2014,2025,2026 Jack Lloyd
7	*
8	* Botan is released under the Simplified BSD License (see license.txt)
9	*/
10
11	#include <botan/internal/poly1305.h>
12
13	#include <botan/internal/buffer_slicer.h>
14	#include <botan/internal/ct_utils.h>
15	#include <botan/internal/donna128.h>
16	#include <botan/internal/loadstor.h>
17
18	#if defined(BOTAN_HAS_POLY1305_AVX2) \|\| defined(BOTAN_HAS_POLY1305_AVX512)
19	#include <botan/internal/cpuid.h>
20	#endif
21
22	namespace Botan {
23
24	namespace {
25
26	// State layout: pad \|\| accum \|\| r \|\| r^2 \|\| r^3 \|\| ... \|\| r^n
27	// This ordering allows extending with more powers of r at the end
28	constexpr size_t PAD_BASE = 0; // pad[0..1]
29	constexpr size_t H_BASE = 2; // h[0..2] (accumulator)
30	constexpr size_t R_BASE = 5; // r^1[0..2], r^2[3..5], r^3[6..8], etc.
31
32	// Multiply two values in radix 2^44 representation mod (2^130 - 5)
33	// h = a * b mod p
34	BOTAN_FORCE_INLINE void poly1305_mul_44(uint64_t& h0,	126,055✔
35	uint64_t& h1,
36	uint64_t& h2,
37	uint64_t a0,
38	uint64_t a1,
39	uint64_t a2,
40	uint64_t b0,
41	uint64_t b1,
42	uint64_t b2) {
43	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	126,055✔
44	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	126,055✔
45
46	#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
47	typedef donna128 uint128_t;
48	#endif
49
50	const uint64_t s1 = b1 * 20;	126,055✔
51	const uint64_t s2 = b2 * 20;	126,055✔
52
53	const uint128_t d0 = uint128_t(a0) * b0 + uint128_t(a1) * s2 + uint128_t(a2) * s1;	126,055✔
54	const uint64_t c0 = carry_shift(d0, 44);	126,055✔
55
56	const uint128_t d1 = uint128_t(a0) * b1 + uint128_t(a1) * b0 + uint128_t(a2) * s2 + c0;	126,055✔
57	const uint64_t c1 = carry_shift(d1, 44);	126,055✔
58
59	const uint128_t d2 = uint128_t(a0) * b2 + uint128_t(a1) * b1 + uint128_t(a2) * b0 + c1;	126,055✔
60	const uint64_t c2 = carry_shift(d2, 42);	126,055✔
61
62	h0 = (d0 & M44) + c2 * 5;	126,055✔
63	h1 = (d1 & M44) + (h0 >> 44);	126,055✔
64	h0 &= M44;	126,055✔
65	h2 = d2 & M42;	126,055✔
66	}
67
68	// Extend powers of r from current max to target
69	void poly1305_extend_powers(secure_vector<uint64_t>& X, size_t target_powers) {	123,653✔
70	const size_t current_powers = (X.size() - 5) / 3;	123,653✔
71
72	if(current_powers >= target_powers) {	123,653✔
73	return;
74	}
75
76	// Load r^1 for multiplication
77	const uint64_t r0 = X[R_BASE + 0];	123,625✔
78	const uint64_t r1 = X[R_BASE + 1];	123,625✔
79	const uint64_t r2 = X[R_BASE + 2];	123,625✔
80
81	X.resize(5 + target_powers * 3);	123,625✔
82
83	// Compute r^(current+1) through r^target
84	for(size_t i = current_powers + 1; i <= target_powers; ++i) {	249,680✔
85	const size_t offset = R_BASE + (i - 1) * 3;	126,055✔
86	poly1305_mul_44(	126,055✔
87	X[offset + 0], X[offset + 1], X[offset + 2], X[offset - 3], X[offset - 2], X[offset - 1], r0, r1, r2);	126,055✔
88	}
89	}
90
91	// Initialize Poly1305 state and precompute powers of r
92	void poly1305_init(secure_vector<uint64_t>& X, const uint8_t key[32]) {	121,195✔
93	X.clear();	121,195✔
94	X.reserve(2 + 3 + 2 * 3);	121,195✔
95	X.resize(2 + 3 + 3);	121,195✔
96
97	/* Save pad for later (first 2 slots) */
98	X[PAD_BASE + 0] = load_le<uint64_t>(key, 2);	121,195✔
99	X[PAD_BASE + 1] = load_le<uint64_t>(key, 3);	121,195✔
100
101	/* h = 0 (accumulator, next 3 slots) */
102	X[H_BASE + 0] = 0;	121,195✔
103	X[H_BASE + 1] = 0;	121,195✔
104	X[H_BASE + 2] = 0;	121,195✔
105
106	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff (clamping) */
107	const uint64_t t0 = load_le<uint64_t>(key, 0);	121,195✔
108	const uint64_t t1 = load_le<uint64_t>(key, 1);	121,195✔
109
110	const uint64_t r0 = (t0) & 0xffc0fffffff;	121,195✔
111	const uint64_t r1 = ((t0 >> 44) \| (t1 << 20)) & 0xfffffc0ffff;	121,195✔
112	const uint64_t r2 = ((t1 >> 24)) & 0x00ffffffc0f;	121,195✔
113
114	// Store r^1
115	X[R_BASE + 0] = r0;	121,195✔
116	X[R_BASE + 1] = r1;	121,195✔
117	X[R_BASE + 2] = r2;	121,195✔
118
119	poly1305_extend_powers(X, 2);	121,195✔
120	}	121,195✔
121
122	// Process a single block: h = (h + m) * r mod p
123	BOTAN_FORCE_INLINE void poly1305_block_single(uint64_t& h0,	352,462✔
124	uint64_t& h1,
125	uint64_t& h2,
126	uint64_t r0,
127	uint64_t r1,
128	uint64_t r2,
129	uint64_t s1,
130	uint64_t s2,
131	const uint8_t* m,
132	uint64_t hibit) {
133	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	352,462✔
134	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	352,462✔
135
136	#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
137	typedef donna128 uint128_t;
138	#endif
139
140	const uint64_t t0 = load_le<uint64_t>(m, 0);	704,924✔
141	const uint64_t t1 = load_le<uint64_t>(m, 1);	352,462✔
142
143	h0 += (t0 & M44);	352,462✔
144	h1 += ((t0 >> 44) \| (t1 << 20)) & M44;	352,462✔
145	h2 += ((t1 >> 24) & M42) \| hibit;	352,462✔
146
147	const uint128_t d0 = uint128_t(h0) * r0 + uint128_t(h1) * s2 + uint128_t(h2) * s1;	352,462✔
148	const uint64_t c0 = carry_shift(d0, 44);	352,462✔
149
150	const uint128_t d1 = uint128_t(h0) * r1 + uint128_t(h1) * r0 + uint128_t(h2) * s2 + c0;	352,462✔
151	const uint64_t c1 = carry_shift(d1, 44);	352,462✔
152
153	const uint128_t d2 = uint128_t(h0) * r2 + uint128_t(h1) * r1 + uint128_t(h2) * r0 + c1;	352,462✔
154	const uint64_t c2 = carry_shift(d2, 42);	352,462✔
155
156	h0 = (d0 & M44) + c2 * 5;	352,462✔
157	h1 = (d1 & M44) + (h0 >> 44);	352,462✔
158	h0 &= M44;	352,462✔
159	h2 = d2 & M42;	352,462✔
160	}
161
162	// Process two blocks in parallel: h = ((h + m0) * r + m1) * r = (h + m0) * r^2 + m1 * r
163	// The multiplications by r^2 and r are independent, enabling ILP
164	BOTAN_FORCE_INLINE void poly1305_block_pair(uint64_t& h0,	75,678✔
165	uint64_t& h1,
166	uint64_t& h2,
167	uint64_t r0,
168	uint64_t r1,
169	uint64_t r2,
170	uint64_t s1,
171	uint64_t s2,
172	uint64_t rr0,
173	uint64_t rr1,
174	uint64_t rr2,
175	uint64_t ss1,
176	uint64_t ss2,
177	const uint8_t* m,
178	uint64_t hibit) {
179	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	75,678✔
180	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	75,678✔
181
182	#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128)
183	typedef donna128 uint128_t;
184	#endif
185
186	// Load first block (will be multiplied by r^2)
187	const uint64_t m0_t0 = load_le<uint64_t>(m, 0);	151,356✔
188	const uint64_t m0_t1 = load_le<uint64_t>(m, 1);	75,678✔
189
190	// Load second block (will be multiplied by r)
191	const uint64_t m1_t0 = load_le<uint64_t>(m + 16, 0);	75,678✔
192	const uint64_t m1_t1 = load_le<uint64_t>(m + 16, 1);	75,678✔
193
194	// Add first block to h
195	h0 += (m0_t0 & M44);	75,678✔
196	h1 += ((m0_t0 >> 44) \| (m0_t1 << 20)) & M44;	75,678✔
197	h2 += ((m0_t1 >> 24) & M42) \| hibit;	75,678✔
198
199	// Convert second block to limbs
200	const uint64_t b0 = (m1_t0 & M44);	75,678✔
201	const uint64_t b1 = ((m1_t0 >> 44) \| (m1_t1 << 20)) & M44;	75,678✔
202	const uint64_t b2 = ((m1_t1 >> 24) & M42) \| hibit;	75,678✔
203
204	// Compute (h + m0) * r^2 + m1 * r
205	const uint128_t d0 = uint128_t(h0) * rr0 + uint128_t(h1) * ss2 + uint128_t(h2) * ss1 + uint128_t(b0) * r0 +	75,678✔
206	uint128_t(b1) * s2 + uint128_t(b2) * s1;	75,678✔
207	const uint64_t c0 = carry_shift(d0, 44);	75,678✔
208
209	const uint128_t d1 = uint128_t(h0) * rr1 + uint128_t(h1) * rr0 + uint128_t(h2) * ss2 + uint128_t(b0) * r1 +	75,678✔
210	uint128_t(b1) * r0 + uint128_t(b2) * s2 + c0;	75,678✔
211	const uint64_t c1 = carry_shift(d1, 44);	75,678✔
212
213	const uint128_t d2 = uint128_t(h0) * rr2 + uint128_t(h1) * rr1 + uint128_t(h2) * rr0 + uint128_t(b0) * r2 +	75,678✔
214	uint128_t(b1) * r1 + uint128_t(b2) * r0 + c1;	75,678✔
215	const uint64_t c2 = carry_shift(d2, 42);	75,678✔
216
217	h0 = (d0 & M44) + c2 * 5;	75,678✔
218	h1 = (d1 & M44) + (h0 >> 44);	75,678✔
219	h0 &= M44;	75,678✔
220	h2 = d2 & M42;	75,678✔
221	}
222
223	void poly1305_blocks(secure_vector<uint64_t>& X, const uint8_t* m, size_t blocks, bool is_final = false) {	375,333✔
224	const uint64_t hibit = is_final ? 0 : (static_cast<uint64_t>(1) << 40);	375,333✔
225
226	// Load r (at R_BASE + 0)
227	const uint64_t r0 = X[R_BASE + 0];	375,333✔
228	const uint64_t r1 = X[R_BASE + 1];	375,333✔
229	const uint64_t r2 = X[R_BASE + 2];	375,333✔
230	const uint64_t s1 = r1 * 20;	375,333✔
231	const uint64_t s2 = r2 * 20;	375,333✔
232
233	// Load r^2 (at R_BASE + 3)
234	const uint64_t rr0 = X[R_BASE + 3];	375,333✔
235	const uint64_t rr1 = X[R_BASE + 4];	375,333✔
236	const uint64_t rr2 = X[R_BASE + 5];	375,333✔
237
238	// Precompute
239	const uint64_t ss1 = rr1 * 20;	375,333✔
240	const uint64_t ss2 = rr2 * 20;	375,333✔
241
242	// Load accumulator
243	uint64_t h0 = X[H_BASE + 0];	375,333✔
244	uint64_t h1 = X[H_BASE + 1];	375,333✔
245	uint64_t h2 = X[H_BASE + 2];	375,333✔
246
247	while(blocks >= 2) {	451,011✔
248	poly1305_block_pair(h0, h1, h2, r0, r1, r2, s1, s2, rr0, rr1, rr2, ss1, ss2, m, hibit);	75,678✔
249	m += 32;	75,678✔
250	blocks -= 2;	75,678✔
251	}
252
253	// Final block?
254	if(blocks > 0) {	375,333✔
255	poly1305_block_single(h0, h1, h2, r0, r1, r2, s1, s2, m, hibit);	704,924✔
256	}
257
258	// Store accumulator
259	X[H_BASE + 0] = h0;	375,333✔
260	X[H_BASE + 1] = h1;	375,333✔
261	X[H_BASE + 2] = h2;	375,333✔
262	}	375,333✔
263
264	void poly1305_finish(secure_vector<uint64_t>& X, uint8_t mac[16]) {	107,167✔
265	constexpr uint64_t M44 = 0xFFFFFFFFFFF;	107,167✔
266	constexpr uint64_t M42 = 0x3FFFFFFFFFF;	107,167✔
267
268	/* fully carry h */
269	uint64_t h0 = X[H_BASE + 0];	107,167✔
270	uint64_t h1 = X[H_BASE + 1];	107,167✔
271	uint64_t h2 = X[H_BASE + 2];	107,167✔
272
273	uint64_t c = (h1 >> 44);	107,167✔
274	h1 &= M44;	107,167✔
275	h2 += c;	107,167✔
276	c = (h2 >> 42);	107,167✔
277	h2 &= M42;	107,167✔
278	h0 += c * 5;	107,167✔
279	c = (h0 >> 44);	107,167✔
280	h0 &= M44;	107,167✔
281	h1 += c;	107,167✔
282	c = (h1 >> 44);	107,167✔
283	h1 &= M44;	107,167✔
284	h2 += c;	107,167✔
285	c = (h2 >> 42);	107,167✔
286	h2 &= M42;	107,167✔
287	h0 += c * 5;	107,167✔
288	c = (h0 >> 44);	107,167✔
289	h0 &= M44;	107,167✔
290	h1 += c;	107,167✔
291
292	/* compute h + -p */
293	uint64_t g0 = h0 + 5;	107,167✔
294	c = (g0 >> 44);	107,167✔
295	g0 &= M44;	107,167✔
296	uint64_t g1 = h1 + c;	107,167✔
297	c = (g1 >> 44);	107,167✔
298	g1 &= M44;	107,167✔
299	const uint64_t g2 = h2 + c - (static_cast<uint64_t>(1) << 42);	107,167✔
300
301	/* select h if h < p, or h + -p if h >= p */
302	const auto h_mask = CT::Mask<uint64_t>::expand_top_bit(g2);	107,167✔
303	h0 = h_mask.select(h0, g0);	107,167✔
304	h1 = h_mask.select(h1, g1);	107,167✔
305	h2 = h_mask.select(h2, g2);	107,167✔
306
307	/* h = (h + pad) */
308	const uint64_t t0 = X[PAD_BASE + 0];	107,167✔
309	const uint64_t t1 = X[PAD_BASE + 1];	107,167✔
310
311	h0 += ((t0)&M44);	107,167✔
312	c = (h0 >> 44);	107,167✔
313	h0 &= M44;	107,167✔
314	h1 += (((t0 >> 44) \| (t1 << 20)) & M44) + c;	107,167✔
315	c = (h1 >> 44);	107,167✔
316	h1 &= M44;	107,167✔
317	h2 += (((t1 >> 24)) & M42) + c;	107,167✔
318	h2 &= M42;	107,167✔
319
320	/* mac = h % (2^128) */
321	h0 = ((h0) \| (h1 << 44));	107,167✔
322	h1 = ((h1 >> 20) \| (h2 << 24));	107,167✔
323
324	store_le(mac, h0, h1);	107,167✔
325
326	/* zero out the state */
327	clear_mem(X.data(), X.size());	107,167✔
328	}	107,167✔
329
330	} // namespace
331
332	void Poly1305::clear() {	7,186✔
333	zap(m_poly);	7,186✔
334	m_buffer.clear();	7,186✔
335	}	7,186✔
336
337	bool Poly1305::has_keying_material() const {	1,174,856✔
338	// Minimum size: pad(2) + accum(3) + r(3) + r^2(3) = 11
339	return m_poly.size() >= 11;	1,174,856✔
340	}
341
342	void Poly1305::key_schedule(std::span<const uint8_t> key) {	121,195✔
343	m_buffer.clear();	121,195✔
344
345	poly1305_init(m_poly, key.data());	121,195✔
346	}	121,195✔
347
348	std::string Poly1305::provider() const {	172✔
349	#if defined(BOTAN_HAS_POLY1305_AVX512)
350	if(auto feat = CPUID::check(CPUID::Feature::AVX512)) {	172✔
351	return *feat;	×
352	}	×
353	#endif
354
355	#if defined(BOTAN_HAS_POLY1305_AVX2)
356	if(auto feat = CPUID::check(CPUID::Feature::AVX2)) {	172✔
357	return *feat;	172✔
358	}	86✔
359	#endif
360
361	return "base";	86✔
362	}
363
364	void Poly1305::add_data(std::span<const uint8_t> input) {	1,067,001✔
365	assert_key_material_set();	1,067,001✔
366
367	BufferSlicer in(input);	1,042,717✔
368
369	while(!in.empty()) {	3,148,015✔
370	if(const auto one_block = m_buffer.handle_unaligned_data(in)) {	1,062,581✔
371	poly1305_blocks(m_poly, one_block->data(), 1);	330,782✔
372	}
373
374	if(m_buffer.in_alignment()) {	1,062,581✔
375	const auto [aligned_data, full_blocks] = m_buffer.aligned_data_to_process(in);	376,345✔
376	if(full_blocks > 0) {	376,345✔
377	const uint8_t* data_ptr = aligned_data.data();	45,747✔
378	size_t blocks_remaining = full_blocks;	45,747✔
379
380	#if defined(BOTAN_HAS_POLY1305_AVX512)
381	if(blocks_remaining >= 8 * 3 && CPUID::has(CPUID::Feature::AVX512)) {	45,747✔
382	// Lazily compute r^3 through r^8 on first AVX512 use
383	poly1305_extend_powers(m_poly, 8);	×
384	const size_t processed = poly1305_avx512_blocks(m_poly, data_ptr, blocks_remaining);	×
385	data_ptr += processed * 16;	×
386	blocks_remaining -= processed;	×
387	}
388	#endif
389
390	#if defined(BOTAN_HAS_POLY1305_AVX2)
391	if(blocks_remaining >= 4 * 6 && CPUID::has(CPUID::Feature::AVX2)) {	45,747✔
392	// Lazily compute r^3 and r^4 on first AVX2 use
393	poly1305_extend_powers(m_poly, 4);	2,458✔
394	const size_t processed = poly1305_avx2_blocks(m_poly, data_ptr, blocks_remaining);	2,458✔
395	data_ptr += processed * 16;	2,458✔
396	blocks_remaining -= processed;	2,458✔
397	}
398	#endif
399
400	if(blocks_remaining > 0) {	45,747✔
401	poly1305_blocks(m_poly, data_ptr, blocks_remaining);	43,731✔
402	}
403	}
404	}
405	}
406	}	1,042,717✔
407
408	void Poly1305::final_result(std::span<uint8_t> out) {	107,339✔
409	assert_key_material_set();	107,339✔
410
411	if(!m_buffer.in_alignment()) {	107,167✔
412	const uint8_t final_byte = 0x01;	820✔
413	m_buffer.append({&final_byte, 1});	820✔
414	m_buffer.fill_up_with_zeros();	820✔
415	poly1305_blocks(m_poly, m_buffer.consume().data(), 1, true);	820✔
416	}
417
418	poly1305_finish(m_poly, out.data());	107,167✔
419
420	m_poly.clear();	107,167✔
421	m_buffer.clear();	107,167✔
422	}	107,167✔
423
424	} // namespace Botan

randombit / botan / 25457312714

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous