4834822395

Committed 28 Apr 2023 09:55PM CUT coverage: 92.132% (-0.01%) from 92.146%

Build # 4834822395

Build Type

push

github

Run Details

77584 of 84210 relevant lines covered (92.13%)

12129586.55 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.61

/src/lib/utils/cpuid/cpuid_x86.cpp

/*
* Runtime CPU detection for x86
* (C) 2009,2010,2013,2017,2023 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/cpuid.h>

#include <botan/mem_ops.h>
#include <botan/internal/loadstor.h>

#if defined(BOTAN_TARGET_CPU_IS_X86_FAMILY)

#include <immintrin.h>

#if defined(BOTAN_BUILD_COMPILER_IS_MSVC)
  #include <intrin.h>
#elif defined(BOTAN_BUILD_COMPILER_IS_INTEL)
  #include <ia32intrin.h>
#elif defined(BOTAN_BUILD_COMPILER_IS_GCC) || defined(BOTAN_BUILD_COMPILER_IS_CLANG)
  #include <cpuid.h>
#endif

#endif

namespace Botan {

#if defined(BOTAN_TARGET_CPU_IS_X86_FAMILY)

namespace {

void invoke_cpuid(uint32_t type, uint32_t out[4])
   {
#if defined(BOTAN_BUILD_COMPILER_IS_MSVC) || defined(BOTAN_BUILD_COMPILER_IS_INTEL)
   __cpuid((int*)out, type);

#elif defined(BOTAN_BUILD_COMPILER_IS_GCC) || defined(BOTAN_BUILD_COMPILER_IS_CLANG)
   __get_cpuid(type, out, out+1, out+2, out+3);

#elif defined(BOTAN_USE_GCC_INLINE_ASM)
   asm("cpuid\n\t"
       : "=a" (out[0]), "=b" (out[1]), "=c" (out[2]), "=d" (out[3])
       : "0" (type));

#else
   #warning "No way of calling x86 cpuid instruction for this compiler"
   clear_mem(out, 4);
#endif
   }

BOTAN_FUNC_ISA("xsave")
uint64_t xgetbv()
   {
   return _xgetbv(0);
   }

void invoke_cpuid_sublevel(uint32_t type, uint32_t level, uint32_t out[4])
   {
#if defined(BOTAN_BUILD_COMPILER_IS_MSVC)
   __cpuidex((int*)out, type, level);

#elif defined(BOTAN_BUILD_COMPILER_IS_GCC) || defined(BOTAN_BUILD_COMPILER_IS_CLANG)
   __cpuid_count(type, level, out[0], out[1], out[2], out[3]);

#elif defined(BOTAN_USE_GCC_INLINE_ASM)
   asm("cpuid\n\t"
       : "=a" (out[0]), "=b" (out[1]), "=c" (out[2]), "=d" (out[3])     \
       : "0" (type), "2" (level));

#else
   #warning "No way of calling x86 cpuid instruction for this compiler"
   clear_mem(out, 4);
#endif
   }

}

uint32_t CPUID::CPUID_Data::detect_cpu_features()
   {
   uint32_t features_detected = 0;
   uint32_t cpuid[4] = { 0 };
   bool has_os_ymm_support = false;
   bool has_os_zmm_support = false;

   // CPUID 0: vendor identification, max sublevel
   invoke_cpuid(0, cpuid);

   const uint32_t max_supported_sublevel = cpuid[0];

   if(max_supported_sublevel >= 1)
      {
      // CPUID 1: feature bits
      invoke_cpuid(1, cpuid);
      const uint64_t flags0 = (static_cast<uint64_t>(cpuid[2]) << 32) | cpuid[3];

      enum x86_CPUID_1_bits : uint64_t {
         RDTSC = (1ULL << 4),
         SSE2 = (1ULL << 26),
         CLMUL = (1ULL << 33),
         SSSE3 = (1ULL << 41),
         AESNI = (1ULL << 57),
         OSXSAVE = (1ULL << 59),
         AVX = (1ULL << 60),
         RDRAND = (1ULL << 62)
      };

      if(flags0 & x86_CPUID_1_bits::RDTSC)
         features_detected |= CPUID::CPUID_RDTSC_BIT;
      if(flags0 & x86_CPUID_1_bits::SSE2)
         features_detected |= CPUID::CPUID_SSE2_BIT;
      if(flags0 & x86_CPUID_1_bits::CLMUL)
         features_detected |= CPUID::CPUID_CLMUL_BIT;
      if(flags0 & x86_CPUID_1_bits::SSSE3)
         features_detected |= CPUID::CPUID_SSSE3_BIT;
      if(flags0 & x86_CPUID_1_bits::AESNI)
         features_detected |= CPUID::CPUID_AESNI_BIT;
      if(flags0 & x86_CPUID_1_bits::RDRAND)
         features_detected |= CPUID::CPUID_RDRAND_BIT;

      if((flags0 & x86_CPUID_1_bits::AVX) &&
         (flags0 & x86_CPUID_1_bits::OSXSAVE))
         {
         const uint64_t xcr_flags = xgetbv();
         if((xcr_flags & 0x6) == 0x6)
            {
            has_os_ymm_support = true;
            has_os_zmm_support = (xcr_flags & 0xE0) == 0xE0;
            }
         }
      }

   if(max_supported_sublevel >= 7)
      {
      clear_mem(cpuid, 4);
      invoke_cpuid_sublevel(7, 0, cpuid);

      enum x86_CPUID_7_bits : uint64_t {
         BMI1 = (1ULL << 3),
         AVX2 = (1ULL << 5),
         BMI2 = (1ULL << 8),
         AVX512_F = (1ULL << 16),
         AVX512_DQ = (1ULL << 17),
         RDSEED = (1ULL << 18),
         ADX = (1ULL << 19),
         AVX512_IFMA = (1ULL << 21),
         SHA = (1ULL << 29),
         AVX512_BW = (1ULL << 30),
         AVX512_VL = (1ULL << 31),
         AVX512_VBMI = (1ULL << 33),
         AVX512_VBMI2 = (1ULL << 38),
         AVX512_VAES = (1ULL << 41),
         AVX512_VCLMUL = (1ULL << 42),
         AVX512_VBITALG = (1ULL << 44),
      };

      const uint64_t flags7 = (static_cast<uint64_t>(cpuid[2]) << 32) | cpuid[1];

      if((flags7 & x86_CPUID_7_bits::AVX2) && has_os_ymm_support)
         features_detected |= CPUID::CPUID_AVX2_BIT;
      if(flags7 & x86_CPUID_7_bits::RDSEED)
         features_detected |= CPUID::CPUID_RDSEED_BIT;
      if(flags7 & x86_CPUID_7_bits::ADX)
         features_detected |= CPUID::CPUID_ADX_BIT;
      if(flags7 & x86_CPUID_7_bits::SHA)
         features_detected |= CPUID::CPUID_SHA_BIT;

      /*
      We only set the BMI bit if both BMI1 and BMI2 are supported, since
      typically we want to use both extensions in the same code.
      */
      if((flags7 & x86_CPUID_7_bits::BMI1) && (flags7 & x86_CPUID_7_bits::BMI2))
         {
         features_detected |= CPUID::CPUID_BMI_BIT;
         }

      if((flags7 & x86_CPUID_7_bits::AVX512_F) && has_os_zmm_support)
         {
         const uint64_t AVX512_PROFILE_FLAGS =
            x86_CPUID_7_bits::AVX512_F |
            x86_CPUID_7_bits::AVX512_DQ |
            x86_CPUID_7_bits::AVX512_IFMA |
            x86_CPUID_7_bits::AVX512_BW |
            x86_CPUID_7_bits::AVX512_VL |
            x86_CPUID_7_bits::AVX512_VBMI |
            x86_CPUID_7_bits::AVX512_VBMI2 |
            x86_CPUID_7_bits::AVX512_VBITALG;

         /*
         We only enable AVX512 support if all of the above flags are available

         This is more than we strictly need for most uses, however it also has
         the effect of preventing execution of AVX512 codepaths on cores that
         have serious downclocking problems when AVX512 code executes,
         especially Intel Skylake.

         VBMI2/VBITALG are the key flags here as they restrict us to Intel Ice
         Lake/Rocket Lake, or AMD Zen4, all of which do not have penalties for
         executing AVX512.

         There is nothing stopping some future processor from supporting the
         above flags and having AVX512 penalties, but maybe you should not have
         bought such a processor.
         */
         if((flags7 & AVX512_PROFILE_FLAGS) == AVX512_PROFILE_FLAGS)
            features_detected |= CPUID::CPUID_AVX512_BIT;

         if(flags7 & x86_CPUID_7_bits::AVX512_VAES)
            features_detected |= CPUID::CPUID_AVX512_AES_BIT;
         if(flags7 & x86_CPUID_7_bits::AVX512_VCLMUL)
            features_detected |= CPUID::CPUID_AVX512_CLMUL_BIT;
         }
      }

   /*
   * If we don't have access to CPUID, we can still safely assume that
   * any x86-64 processor has SSE2 and RDTSC
   */
#if defined(BOTAN_TARGET_ARCH_IS_X86_64)
   if(features_detected == 0)
      {
      features_detected |= CPUID::CPUID_SSE2_BIT;
      features_detected |= CPUID::CPUID_RDTSC_BIT;
      }
#endif

   return features_detected;
   }

#endif

}

1	/*
2	* Runtime CPU detection for x86
3	* (C) 2009,2010,2013,2017,2023 Jack Lloyd
4	*
5	* Botan is released under the Simplified BSD License (see license.txt)
6	*/
7
8	#include <botan/internal/cpuid.h>
9
10	#include <botan/mem_ops.h>
11	#include <botan/internal/loadstor.h>
12
13	#if defined(BOTAN_TARGET_CPU_IS_X86_FAMILY)
14
15	#include <immintrin.h>
16
17	#if defined(BOTAN_BUILD_COMPILER_IS_MSVC)
18	#include <intrin.h>
19	#elif defined(BOTAN_BUILD_COMPILER_IS_INTEL)
20	#include <ia32intrin.h>
21	#elif defined(BOTAN_BUILD_COMPILER_IS_GCC) \|\| defined(BOTAN_BUILD_COMPILER_IS_CLANG)
22	#include <cpuid.h>
23	#endif
24
25	#endif
26
27	namespace Botan {
28
29	#if defined(BOTAN_TARGET_CPU_IS_X86_FAMILY)
30
31	namespace {
32
33	void invoke_cpuid(uint32_t type, uint32_t out[4])	25,658✔
34	{
35	#if defined(BOTAN_BUILD_COMPILER_IS_MSVC) \|\| defined(BOTAN_BUILD_COMPILER_IS_INTEL)
36	__cpuid((int*)out, type);
37
38	#elif defined(BOTAN_BUILD_COMPILER_IS_GCC) \|\| defined(BOTAN_BUILD_COMPILER_IS_CLANG)
39	__get_cpuid(type, out, out+1, out+2, out+3);	51,316✔
40
41	#elif defined(BOTAN_USE_GCC_INLINE_ASM)
42	asm("cpuid\n\t"
43	: "=a" (out[0]), "=b" (out[1]), "=c" (out[2]), "=d" (out[3])
44	: "0" (type));
45
46	#else
47	#warning "No way of calling x86 cpuid instruction for this compiler"
48	clear_mem(out, 4);
49	#endif
50	}
51
52	BOTAN_FUNC_ISA("xsave")
53	uint64_t xgetbv()	12,829✔
54	{
55	return _xgetbv(0);	12,829✔
56	}
57
58	void invoke_cpuid_sublevel(uint32_t type, uint32_t level, uint32_t out[4])	12,829✔
59	{
60	#if defined(BOTAN_BUILD_COMPILER_IS_MSVC)
61	__cpuidex((int*)out, type, level);
62
63	#elif defined(BOTAN_BUILD_COMPILER_IS_GCC) \|\| defined(BOTAN_BUILD_COMPILER_IS_CLANG)
64	__cpuid_count(type, level, out[0], out[1], out[2], out[3]);	12,829✔
65
66	#elif defined(BOTAN_USE_GCC_INLINE_ASM)
67	asm("cpuid\n\t"
68	: "=a" (out[0]), "=b" (out[1]), "=c" (out[2]), "=d" (out[3]) \
69	: "0" (type), "2" (level));
70
71	#else
72	#warning "No way of calling x86 cpuid instruction for this compiler"
73	clear_mem(out, 4);
74	#endif
75	}
76
77	}
78
79	uint32_t CPUID::CPUID_Data::detect_cpu_features()	12,829✔
80	{
81	uint32_t features_detected = 0;	12,829✔
82	uint32_t cpuid[4] = { 0 };	12,829✔
83	bool has_os_ymm_support = false;	12,829✔
84	bool has_os_zmm_support = false;	12,829✔
85
86	// CPUID 0: vendor identification, max sublevel
87	invoke_cpuid(0, cpuid);	12,829✔
88
89	const uint32_t max_supported_sublevel = cpuid[0];	12,829✔
90
91	if(max_supported_sublevel >= 1)	12,829✔
92	{
93	// CPUID 1: feature bits
94	invoke_cpuid(1, cpuid);	12,829✔
95	const uint64_t flags0 = (static_cast<uint64_t>(cpuid[2]) << 32) \| cpuid[3];	12,829✔
96
97	enum x86_CPUID_1_bits : uint64_t {	12,829✔
98	RDTSC = (1ULL << 4),
99	SSE2 = (1ULL << 26),
100	CLMUL = (1ULL << 33),
101	SSSE3 = (1ULL << 41),
102	AESNI = (1ULL << 57),
103	OSXSAVE = (1ULL << 59),
104	AVX = (1ULL << 60),
105	RDRAND = (1ULL << 62)
106	};
107
108	if(flags0 & x86_CPUID_1_bits::RDTSC)	12,829✔
109	features_detected \|= CPUID::CPUID_RDTSC_BIT;	12,829✔
110	if(flags0 & x86_CPUID_1_bits::SSE2)	12,829✔
111	features_detected \|= CPUID::CPUID_SSE2_BIT;	12,829✔
112	if(flags0 & x86_CPUID_1_bits::CLMUL)	12,829✔
113	features_detected \|= CPUID::CPUID_CLMUL_BIT;	12,829✔
114	if(flags0 & x86_CPUID_1_bits::SSSE3)	12,829✔
115	features_detected \|= CPUID::CPUID_SSSE3_BIT;	12,829✔
116	if(flags0 & x86_CPUID_1_bits::AESNI)	12,829✔
117	features_detected \|= CPUID::CPUID_AESNI_BIT;	12,829✔
118	if(flags0 & x86_CPUID_1_bits::RDRAND)	12,829✔
119	features_detected \|= CPUID::CPUID_RDRAND_BIT;	12,829✔
120
121	if((flags0 & x86_CPUID_1_bits::AVX) &&	12,829✔
122	(flags0 & x86_CPUID_1_bits::OSXSAVE))
123	{
124	const uint64_t xcr_flags = xgetbv();	12,829✔
125	if((xcr_flags & 0x6) == 0x6)	12,829✔
126	{
127	has_os_ymm_support = true;	12,829✔
128	has_os_zmm_support = (xcr_flags & 0xE0) == 0xE0;	12,829✔
129	}
130	}
131	}
132
133	if(max_supported_sublevel >= 7)	12,829✔
134	{
135	clear_mem(cpuid, 4);	12,829✔
136	invoke_cpuid_sublevel(7, 0, cpuid);	12,829✔
137
138	enum x86_CPUID_7_bits : uint64_t {	12,829✔
139	BMI1 = (1ULL << 3),
140	AVX2 = (1ULL << 5),
141	BMI2 = (1ULL << 8),
142	AVX512_F = (1ULL << 16),
143	AVX512_DQ = (1ULL << 17),
144	RDSEED = (1ULL << 18),
145	ADX = (1ULL << 19),
146	AVX512_IFMA = (1ULL << 21),
147	SHA = (1ULL << 29),
148	AVX512_BW = (1ULL << 30),
149	AVX512_VL = (1ULL << 31),
150	AVX512_VBMI = (1ULL << 33),
151	AVX512_VBMI2 = (1ULL << 38),
152	AVX512_VAES = (1ULL << 41),
153	AVX512_VCLMUL = (1ULL << 42),
154	AVX512_VBITALG = (1ULL << 44),
155	};
156
157	const uint64_t flags7 = (static_cast<uint64_t>(cpuid[2]) << 32) \| cpuid[1];	12,829✔
158
159	if((flags7 & x86_CPUID_7_bits::AVX2) && has_os_ymm_support)	12,829✔
160	features_detected \|= CPUID::CPUID_AVX2_BIT;	12,829✔
161	if(flags7 & x86_CPUID_7_bits::RDSEED)	12,829✔
162	features_detected \|= CPUID::CPUID_RDSEED_BIT;	12,829✔
163	if(flags7 & x86_CPUID_7_bits::ADX)	12,829✔
164	features_detected \|= CPUID::CPUID_ADX_BIT;	12,829✔
165	if(flags7 & x86_CPUID_7_bits::SHA)	12,829✔
166	features_detected \|= CPUID::CPUID_SHA_BIT;	×
167
168	/*
169	We only set the BMI bit if both BMI1 and BMI2 are supported, since
170	typically we want to use both extensions in the same code.
171	*/
172	if((flags7 & x86_CPUID_7_bits::BMI1) && (flags7 & x86_CPUID_7_bits::BMI2))	12,829✔
173	{
174	features_detected \|= CPUID::CPUID_BMI_BIT;	12,829✔
175	}
176
177	if((flags7 & x86_CPUID_7_bits::AVX512_F) && has_os_zmm_support)	12,829✔
178	{
179	const uint64_t AVX512_PROFILE_FLAGS =	×
180	x86_CPUID_7_bits::AVX512_F \|
181	x86_CPUID_7_bits::AVX512_DQ \|
182	x86_CPUID_7_bits::AVX512_IFMA \|
183	x86_CPUID_7_bits::AVX512_BW \|
184	x86_CPUID_7_bits::AVX512_VL \|
185	x86_CPUID_7_bits::AVX512_VBMI \|
186	x86_CPUID_7_bits::AVX512_VBMI2 \|
187	x86_CPUID_7_bits::AVX512_VBITALG;
188
189	/*
190	We only enable AVX512 support if all of the above flags are available
191
192	This is more than we strictly need for most uses, however it also has
193	the effect of preventing execution of AVX512 codepaths on cores that
194	have serious downclocking problems when AVX512 code executes,
195	especially Intel Skylake.
196
197	VBMI2/VBITALG are the key flags here as they restrict us to Intel Ice
198	Lake/Rocket Lake, or AMD Zen4, all of which do not have penalties for
199	executing AVX512.
200
201	There is nothing stopping some future processor from supporting the
202	above flags and having AVX512 penalties, but maybe you should not have
203	bought such a processor.
204	*/
205	if((flags7 & AVX512_PROFILE_FLAGS) == AVX512_PROFILE_FLAGS)	×
206	features_detected \|= CPUID::CPUID_AVX512_BIT;	×
207
208	if(flags7 & x86_CPUID_7_bits::AVX512_VAES)	×
209	features_detected \|= CPUID::CPUID_AVX512_AES_BIT;	×
210	if(flags7 & x86_CPUID_7_bits::AVX512_VCLMUL)	×
211	features_detected \|= CPUID::CPUID_AVX512_CLMUL_BIT;	×
212	}
213	}
214
215	/*
216	* If we don't have access to CPUID, we can still safely assume that
217	* any x86-64 processor has SSE2 and RDTSC
218	*/
219	#if defined(BOTAN_TARGET_ARCH_IS_X86_64)
220	if(features_detected == 0)	12,829✔
221	{
222	features_detected \|= CPUID::CPUID_SSE2_BIT;	×
223	features_detected \|= CPUID::CPUID_RDTSC_BIT;	×
224	}
225	#endif
226
227	return features_detected;	12,829✔
228	}
229
230	#endif
231
232	}

randombit / botan / 4834822395

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous