randombit/botan | Build 26013989535 | source/src/lib/utils/charset.cpp | Coveralls

18

void append_utf8_for(std::string& s, uint32_t c) {

28,077✔

19

   if(c >= 0xD800 && c < 0xE000) {

28,077✔

20

      throw Decoding_Error("Invalid Unicode character");

2✔

23

   if(c <= 0x7F) {

28,075✔

24

      const uint8_t b0 = static_cast<uint8_t>(c);

27,563✔

25

      s.push_back(static_cast<char>(b0));

27,563✔

26

   } else if(c <= 0x7FF) {

512✔

27

      const uint8_t b0 = 0xC0 | static_cast<uint8_t>(c >> 6);

198✔

28

      const uint8_t b1 = 0x80 | static_cast<uint8_t>(c & 0x3F);

198✔

29

      s.push_back(static_cast<char>(b0));

198✔

30

      s.push_back(static_cast<char>(b1));

198✔

31

   } else if(c <= 0xFFFF) {

314✔

32

      const uint8_t b0 = 0xE0 | static_cast<uint8_t>(c >> 12);

292✔

33

      const uint8_t b1 = 0x80 | static_cast<uint8_t>((c >> 6) & 0x3F);

292✔

34

      const uint8_t b2 = 0x80 | static_cast<uint8_t>(c & 0x3F);

292✔

35

      s.push_back(static_cast<char>(b0));

292✔

36

      s.push_back(static_cast<char>(b1));

292✔

37

      s.push_back(static_cast<char>(b2));

292✔

38

   } else if(c <= 0x10FFFF) {

22✔

39

      const uint8_t b0 = 0xF0 | static_cast<uint8_t>(c >> 18);

1✔

40

      const uint8_t b1 = 0x80 | static_cast<uint8_t>((c >> 12) & 0x3F);

1✔

41

      const uint8_t b2 = 0x80 | static_cast<uint8_t>((c >> 6) & 0x3F);

1✔

42

      const uint8_t b3 = 0x80 | static_cast<uint8_t>(c & 0x3F);

1✔

43

      s.push_back(static_cast<char>(b0));

1✔

44

      s.push_back(static_cast<char>(b1));

1✔

45

      s.push_back(static_cast<char>(b2));

1✔

46

      s.push_back(static_cast<char>(b3));

1✔

48

      throw Decoding_Error("Invalid Unicode character");

21✔

50

28,054✔

52

uint32_t next_utf8_codepoint(std::string_view utf8, size_t& pos) {

552,786✔

53

   auto read_continuation = [&]() -> uint32_t {

555,562✔

54

      if(pos >= utf8.size()) {

2,776✔

55

         throw Decoding_Error("Invalid UTF-8 sequence");

5✔

57

      const uint8_t b = static_cast<uint8_t>(utf8[pos++]);

2,771✔

58

      if((b & 0xC0) != 0x80) {

2,771✔

59

         throw Decoding_Error("Invalid UTF-8 sequence");

8✔

61

      return b & 0x3F;

2,763✔

62

};

552,786✔

64

   const uint8_t lead = static_cast<uint8_t>(utf8[pos++]);

552,786✔

65

   uint32_t c = 0;

552,786✔

67

   if(lead <= 0x7F) {

552,786✔

68

      c = lead;

550,074✔

69

   } else if((lead & 0xE0) == 0xC0) {

2,712✔

70

      c = (lead & 0x1F) << 6;

2,631✔

71

      c |= read_continuation();

2,631✔

72

      if(c < 0x80) {

2,625✔

73

         throw Decoding_Error("Overlong UTF-8 sequence");

1✔

75

   } else if((lead & 0xF0) == 0xE0) {

81✔

76

      c = (lead & 0x0F) << 12;

53✔

77

      c |= read_continuation() << 6;

53✔

78

      c |= read_continuation();

53✔

79

      if(c < 0x800) {

50✔

80

         throw Decoding_Error("Overlong UTF-8 sequence");

3✔

82

   } else if((lead & 0xF8) == 0xF0) {

28✔

83

      c = (lead & 0x07) << 18;

14✔

84

      c |= read_continuation() << 12;

14✔

85

      c |= read_continuation() << 6;

13✔

86

      c |= read_continuation();

12✔

87

      if(c < 0x10000) {

10✔

88

         throw Decoding_Error("Overlong UTF-8 sequence");

3✔

91

      throw Decoding_Error("Invalid UTF-8 sequence");

14✔

94

   if(c > 0x10FFFF) {

552,752✔

95

      throw Decoding_Error("UTF-8 sequence encodes value outside Unicode range");

3✔

97

   if(c >= 0xD800 && c < 0xE000) {

552,749✔

98

      throw Decoding_Error("UTF-8 sequence encodes surrogate code point");

3✔

101

   return c;

552,746✔

106

bool is_valid_utf8(std::string_view utf8) {

33,594✔

107

   try {

33,594✔

108

      size_t pos = 0;

33,594✔

109

      while(pos < utf8.size()) {

586,238✔

110

         const uint32_t c = next_utf8_codepoint(utf8, pos);

552,662✔

113

   } catch(Decoding_Error&) {

18✔

114

      return false;

18✔

115

18✔

116

   return true;

33,576✔

119

std::string ucs2_to_utf8(std::span<const uint8_t> ucs2) {

951✔

120

   if(ucs2.size() % 2 != 0) {

951✔

121

      throw Decoding_Error("Invalid length for UCS-2 string");

8✔

124

   const size_t chars = ucs2.size() / 2;

943✔

126

   std::string s;

943✔

127

   for(size_t i = 0; i != chars; ++i) {

2,438✔

128

      const uint32_t c = load_be<uint16_t>(ucs2.data(), i);

1,496✔

129

      append_utf8_for(s, c);

1,496✔

132

   return s;

942✔

133

1✔

135

std::vector<uint8_t> utf8_to_ucs2(std::string_view utf8) {

24✔

136

   std::vector<uint8_t> out;

24✔

137

   out.reserve(utf8.size() * 2);

24✔

139

   size_t pos = 0;

24✔

140

   while(pos < utf8.size()) {

92✔

141

      const uint32_t c = next_utf8_codepoint(utf8, pos);

81✔

142

      if(c > 0xFFFF) {

69✔

143

         throw Decoding_Error("Cannot encode character in UCS-2");

1✔

145

      const uint16_t val = static_cast<uint16_t>(c);

68✔

146

      out.push_back(get_byte<0>(val));

68✔

147

      out.push_back(get_byte<1>(val));

68✔

150

   return out;

11✔

151

13✔

153

std::string ucs4_to_utf8(std::span<const uint8_t> ucs4) {

70✔

154

   if(ucs4.size() % 4 != 0) {

70✔

155

      throw Decoding_Error("Invalid length for UCS-4 string");

5✔

158

   const size_t chars = ucs4.size() / 4;

65✔

160

   std::string s;

65✔

161

   for(size_t i = 0; i != chars; ++i) {

108✔

162

      const uint32_t c = load_be<uint32_t>(ucs4.data(), i);

65✔

163

      append_utf8_for(s, c);

65✔

166

   return s;

43✔

167

22✔

169

std::vector<uint8_t> utf8_to_ucs4(std::string_view utf8) {

21✔

170

   std::vector<uint8_t> out;

21✔

171

   out.reserve(utf8.size() * 4);

21✔

173

   size_t pos = 0;

21✔

174

   while(pos < utf8.size()) {

54✔

175

      const uint32_t val = next_utf8_codepoint(utf8, pos);

43✔

176

      out.push_back(get_byte<0>(val));

33✔

177

      out.push_back(get_byte<1>(val));

33✔

178

      out.push_back(get_byte<2>(val));

33✔

179

      out.push_back(get_byte<3>(val));

33✔

182

   return out;

11✔

183

10✔

188

std::string latin1_to_utf8(std::span<const uint8_t> chars) {

1,197✔

189

   std::string s;

1,197✔

190

   for(const uint8_t b : chars) {

27,713✔

191

      append_utf8_for(s, static_cast<uint32_t>(b));

26,516✔

193

   return s;

1,197✔

196

std::string format_char_for_display(char c) {

77✔

197

   std::ostringstream oss;

77✔

199

   oss << "'";

77✔

201

   if(c == '\t') {

77✔

202

      oss << "\\t";

14✔

203

   } else if(c == '\n') {

63✔

204

      oss << "\\n";

14✔

205

   } else if(c == '\r') {

49✔

206

      oss << "\\r";

14✔

207

   } else if(static_cast<unsigned char>(c) >= 128) {

35✔

208

      const unsigned char z = static_cast<unsigned char>(c);

6✔

209

      oss << "\\x" << std::hex << std::uppercase << static_cast<int>(z);

6✔

211

      oss << c;

29✔

214

   oss << "'";

77✔

216

   return oss.str();

154✔

217

77✔

randombit / botan / 26013989535

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1	/*
2	* Character Set Handling
3	* (C) 1999-2007,2021 Jack Lloyd
4	*
5	* Botan is released under the Simplified BSD License (see license.txt)
6	*/
7
8	#include <botan/internal/charset.h>
9
10	#include <botan/exceptn.h>
11	#include <botan/internal/loadstor.h>
12	#include <sstream>
13
14	namespace Botan {
15
16	namespace {
17
18	void append_utf8_for(std::string& s, uint32_t c) {	28,077✔
19	if(c >= 0xD800 && c < 0xE000) {	28,077✔
20	throw Decoding_Error("Invalid Unicode character");	2✔
21	}
22
23	if(c <= 0x7F) {	28,075✔
24	const uint8_t b0 = static_cast<uint8_t>(c);	27,563✔
25	s.push_back(static_cast<char>(b0));	27,563✔
26	} else if(c <= 0x7FF) {	512✔
27	const uint8_t b0 = 0xC0 \| static_cast<uint8_t>(c >> 6);	198✔
28	const uint8_t b1 = 0x80 \| static_cast<uint8_t>(c & 0x3F);	198✔
29	s.push_back(static_cast<char>(b0));	198✔
30	s.push_back(static_cast<char>(b1));	198✔
31	} else if(c <= 0xFFFF) {	314✔
32	const uint8_t b0 = 0xE0 \| static_cast<uint8_t>(c >> 12);	292✔
33	const uint8_t b1 = 0x80 \| static_cast<uint8_t>((c >> 6) & 0x3F);	292✔
34	const uint8_t b2 = 0x80 \| static_cast<uint8_t>(c & 0x3F);	292✔
35	s.push_back(static_cast<char>(b0));	292✔
36	s.push_back(static_cast<char>(b1));	292✔
37	s.push_back(static_cast<char>(b2));	292✔
38	} else if(c <= 0x10FFFF) {	22✔
39	const uint8_t b0 = 0xF0 \| static_cast<uint8_t>(c >> 18);	1✔
40	const uint8_t b1 = 0x80 \| static_cast<uint8_t>((c >> 12) & 0x3F);	1✔
41	const uint8_t b2 = 0x80 \| static_cast<uint8_t>((c >> 6) & 0x3F);	1✔
42	const uint8_t b3 = 0x80 \| static_cast<uint8_t>(c & 0x3F);	1✔
43	s.push_back(static_cast<char>(b0));	1✔
44	s.push_back(static_cast<char>(b1));	1✔
45	s.push_back(static_cast<char>(b2));	1✔
46	s.push_back(static_cast<char>(b3));	1✔
47	} else {
48	throw Decoding_Error("Invalid Unicode character");	21✔
49	}
50	}	28,054✔
51
52	uint32_t next_utf8_codepoint(std::string_view utf8, size_t& pos) {	552,786✔
53	auto read_continuation = [&]() -> uint32_t {	555,562✔
54	if(pos >= utf8.size()) {	2,776✔
55	throw Decoding_Error("Invalid UTF-8 sequence");	5✔
56	}
57	const uint8_t b = static_cast<uint8_t>(utf8[pos++]);	2,771✔
58	if((b & 0xC0) != 0x80) {	2,771✔
59	throw Decoding_Error("Invalid UTF-8 sequence");	8✔
60	}
61	return b & 0x3F;	2,763✔
62	};	552,786✔
63
64	const uint8_t lead = static_cast<uint8_t>(utf8[pos++]);	552,786✔
65	uint32_t c = 0;	552,786✔
66
67	if(lead <= 0x7F) {	552,786✔
68	c = lead;	550,074✔
69	} else if((lead & 0xE0) == 0xC0) {	2,712✔
70	c = (lead & 0x1F) << 6;	2,631✔
71	c \|= read_continuation();	2,631✔
72	if(c < 0x80) {	2,625✔
73	throw Decoding_Error("Overlong UTF-8 sequence");	1✔
74	}
75	} else if((lead & 0xF0) == 0xE0) {	81✔
76	c = (lead & 0x0F) << 12;	53✔
77	c \|= read_continuation() << 6;	53✔
78	c \|= read_continuation();	53✔
79	if(c < 0x800) {	50✔
80	throw Decoding_Error("Overlong UTF-8 sequence");	3✔
81	}
82	} else if((lead & 0xF8) == 0xF0) {	28✔
83	c = (lead & 0x07) << 18;	14✔
84	c \|= read_continuation() << 12;	14✔
85	c \|= read_continuation() << 6;	13✔
86	c \|= read_continuation();	12✔
87	if(c < 0x10000) {	10✔
88	throw Decoding_Error("Overlong UTF-8 sequence");	3✔
89	}
90	} else {
91	throw Decoding_Error("Invalid UTF-8 sequence");	14✔
92	}
93
94	if(c > 0x10FFFF) {	552,752✔
95	throw Decoding_Error("UTF-8 sequence encodes value outside Unicode range");	3✔
96	}
97	if(c >= 0xD800 && c < 0xE000) {	552,749✔
98	throw Decoding_Error("UTF-8 sequence encodes surrogate code point");	3✔
99	}
100
101	return c;	552,746✔
102	}
103
104	} // namespace
105
106	bool is_valid_utf8(std::string_view utf8) {	33,594✔
107	try {	33,594✔
108	size_t pos = 0;	33,594✔
109	while(pos < utf8.size()) {	586,238✔
110	const uint32_t c = next_utf8_codepoint(utf8, pos);	552,662✔
111	BOTAN_UNUSED(c);
112	}
113	} catch(Decoding_Error&) {	18✔
114	return false;	18✔
115	}	18✔
116	return true;	33,576✔
117	}
118
119	std::string ucs2_to_utf8(std::span<const uint8_t> ucs2) {	951✔
120	if(ucs2.size() % 2 != 0) {	951✔
121	throw Decoding_Error("Invalid length for UCS-2 string");	8✔
122	}
123
124	const size_t chars = ucs2.size() / 2;	943✔
125
126	std::string s;	943✔
127	for(size_t i = 0; i != chars; ++i) {	2,438✔
128	const uint32_t c = load_be<uint16_t>(ucs2.data(), i);	1,496✔
129	append_utf8_for(s, c);	1,496✔
130	}
131
132	return s;	942✔
133	}	1✔
134
135	std::vector<uint8_t> utf8_to_ucs2(std::string_view utf8) {	24✔
136	std::vector<uint8_t> out;	24✔
137	out.reserve(utf8.size() * 2);	24✔
138
139	size_t pos = 0;	24✔
140	while(pos < utf8.size()) {	92✔
141	const uint32_t c = next_utf8_codepoint(utf8, pos);	81✔
142	if(c > 0xFFFF) {	69✔
143	throw Decoding_Error("Cannot encode character in UCS-2");	1✔
144	}
145	const uint16_t val = static_cast<uint16_t>(c);	68✔
146	out.push_back(get_byte<0>(val));	68✔
147	out.push_back(get_byte<1>(val));	68✔
148	}
149
150	return out;	11✔
151	}	13✔
152
153	std::string ucs4_to_utf8(std::span<const uint8_t> ucs4) {	70✔
154	if(ucs4.size() % 4 != 0) {	70✔
155	throw Decoding_Error("Invalid length for UCS-4 string");	5✔
156	}
157
158	const size_t chars = ucs4.size() / 4;	65✔
159
160	std::string s;	65✔
161	for(size_t i = 0; i != chars; ++i) {	108✔
162	const uint32_t c = load_be<uint32_t>(ucs4.data(), i);	65✔
163	append_utf8_for(s, c);	65✔
164	}
165
166	return s;	43✔
167	}	22✔
168
169	std::vector<uint8_t> utf8_to_ucs4(std::string_view utf8) {	21✔
170	std::vector<uint8_t> out;	21✔
171	out.reserve(utf8.size() * 4);	21✔
172
173	size_t pos = 0;	21✔
174	while(pos < utf8.size()) {	54✔
175	const uint32_t val = next_utf8_codepoint(utf8, pos);	43✔
176	out.push_back(get_byte<0>(val));	33✔
177	out.push_back(get_byte<1>(val));	33✔
178	out.push_back(get_byte<2>(val));	33✔
179	out.push_back(get_byte<3>(val));	33✔
180	}
181
182	return out;	11✔
183	}	10✔
184
185	/*
186	* Convert from ISO 8859-1 to UTF-8
187	*/
188	std::string latin1_to_utf8(std::span<const uint8_t> chars) {	1,197✔
189	std::string s;	1,197✔
190	for(const uint8_t b : chars) {	27,713✔
191	append_utf8_for(s, static_cast<uint32_t>(b));	26,516✔
192	}
193	return s;	1,197✔
194	}	×
195
196	std::string format_char_for_display(char c) {	77✔
197	std::ostringstream oss;	77✔
198
199	oss << "'";	77✔
200
201	if(c == '\t') {	77✔
202	oss << "\\t";	14✔
203	} else if(c == '\n') {	63✔
204	oss << "\\n";	14✔
205	} else if(c == '\r') {	49✔
206	oss << "\\r";	14✔
207	} else if(static_cast<unsigned char>(c) >= 128) {	35✔
208	const unsigned char z = static_cast<unsigned char>(c);	6✔
209	oss << "\\x" << std::hex << std::uppercase << static_cast<int>(z);	6✔
210	} else {
211	oss << c;	29✔
212	}
213
214	oss << "'";	77✔
215
216	return oss.str();	154✔
217	}	77✔
218
219	} // namespace Botan

randombit / botan / 26013989535

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous