#848

Committed 07 Dec 2022 11:00PM UTC coverage: 86.456% (+0.6%) from 85.835%

Build # #848

Build Type

push

Committed by StellarBot

Commit Message

Merge #6096

6096: Forking Boost.Tokenizer r=hkaiser a=hkaiser

- flyby: remove more Boost headers that are not needed anymore

Working towards #3440 

Co-authored-by: Hartmut Kaiser <hartmut.kaiser@gmail.com>

Run Details

525 of 525 new or added lines in 20 files covered. (100.0%)

173087 of 200202 relevant lines covered (86.46%)

1845223.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.91

/libs/core/string_util/tests/unit/tokenizer.cpp

//  Copyright (c) 2022 Hartmut Kaiser
//
//  SPDX-License-Identifier: BSL-1.0
//  Distributed under the Boost Software License, Version 1.0. (See accompanying
//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

// (c) Copyright John R. Bandela 2001.

// See http://www.boost.org for updates, documentation, and revision history.

#include <hpx/modules/string_util.hpp>
#include <hpx/modules/testing.hpp>

#include <algorithm>
#include <array>
#include <iterator>
#include <string>

int main()
{
    using namespace hpx::string_util;

    // Use tokenizer
    {
        std::string const test_string = ";;Hello|world||-foo--bar;yow;baz|";
        std::string answer[] = {"Hello", "world", "foo", "bar", "yow", "baz"};
        using Tok = tokenizer<char_separator<char>>;
        char_separator<char> sep("-;|");
        Tok t(test_string, sep);
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    {
        std::string const test_string = ";;Hello|world||-foo--bar;yow;baz|";
        std::string answer[] = {"", "", "Hello", "|", "world", "|", "", "|", "",
            "foo", "", "bar", "yow", "baz", "|", ""};
        using Tok = tokenizer<char_separator<char>>;
        char_separator<char> sep("-;", "|", empty_token_policy::keep);
        Tok t(test_string, sep);
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    {
        std::string const test_string = "This,,is, a.test..";
        std::string answer[] = {"This", "is", "a", "test"};
        using Tok = tokenizer<char_delimiters_separator<char>>;
        Tok t(test_string);
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    {
        std::string const test_string =
            "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
        std::string answer[] = {
            "Field 1", "embedded,comma", "quote \"", " escape \\"};
        using Tok = tokenizer<escaped_list_separator<char>>;
        Tok t(test_string);
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    {
        std::string const test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';";
        std::string answer[] = {
            "", "1", "", "2\"", "3;", "4", "5\\", "6,7", ""};
        using Tok = tokenizer<escaped_list_separator<char>>;
        escaped_list_separator<char> sep("\\^", ",;", "\"\'");
        Tok t(test_string, sep);
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    {
        std::string const test_string = "12252001";
        std::string answer[] = {"12", "25", "2001"};
        using Tok = tokenizer<offset_separator>;
        std::array<int, 3> offsets = {{2, 2, 4}};
        offset_separator func(offsets.begin(), offsets.end());
        Tok t(test_string, func);
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    // Use token_iterator_generator
    {
        std::string const test_string = "This,,is, a.test..";
        std::string answer[] = {"This", "is", "a", "test"};
        using Iter =
            token_iterator_generator<char_delimiters_separator<char>>::type;
        Iter begin = make_token_iterator<std::string>(test_string.begin(),
            test_string.end(), char_delimiters_separator<char>());
        Iter end;
        HPX_TEST(std::equal(begin, end, answer));
    }

    {
        std::string const test_string =
            "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
        std::string answer[] = {
            "Field 1", "embedded,comma", "quote \"", " escape \\"};
        using Iter =
            token_iterator_generator<escaped_list_separator<char>>::type;
        Iter begin = make_token_iterator<std::string>(test_string.begin(),
            test_string.end(), escaped_list_separator<char>());
        Iter begin_c(begin);
        Iter end;
        HPX_TEST(std::equal(begin, end, answer));

        while (begin_c != end)
        {
            HPX_TEST(begin_c.at_end() == 0);
            ++begin_c;
        }
        HPX_TEST(begin_c.at_end());
    }

    {
        std::string const test_string = "12252001";
        std::string answer[] = {"12", "25", "2001"};
        using Iter = token_iterator_generator<offset_separator>::type;
        std::array<int, 3> offsets = {{2, 2, 4}};
        offset_separator func(offsets.begin(), offsets.end());
        Iter begin = make_token_iterator<std::string>(
            test_string.begin(), test_string.end(), func);
        Iter end = make_token_iterator<std::string>(
            test_string.end(), test_string.end(), func);
        HPX_TEST(std::equal(begin, end, answer));
    }

    // Test copying
    {
        std::string const test_string = "abcdef";
        token_iterator_generator<offset_separator>::type beg, end, other;
        std::array<int, 3> ar = {{1, 2, 3}};
        offset_separator f(ar.begin(), ar.end());
        beg = make_token_iterator<std::string>(
            test_string.begin(), test_string.end(), f);

        ++beg;
        other = beg;
        ++other;

        HPX_TEST(*beg == "bc");
        HPX_TEST(*other == "def");

        other = make_token_iterator<std::string>(
            test_string.begin(), test_string.end(), f);

        HPX_TEST(*other == "a");
    }

    // Test non-default constructed char_separator
    {
        std::string const test_string = "how,are you, doing";
        std::string answer[] = {"how", ",", "are you", ",", " doing"};
        tokenizer t(
            test_string, char_delimiters_separator<char>(true, ",", ""));
        HPX_TEST(std::equal(t.begin(), t.end(), answer));
    }

    return hpx::util::report_errors();
}

1	// Copyright (c) 2022 Hartmut Kaiser
2	//
3	// SPDX-License-Identifier: BSL-1.0
4	// Distributed under the Boost Software License, Version 1.0. (See accompanying
5	// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
7	// (c) Copyright John R. Bandela 2001.
8
9	// See http://www.boost.org for updates, documentation, and revision history.
10
11	#include <hpx/modules/string_util.hpp>
12	#include <hpx/modules/testing.hpp>
13
14	#include <algorithm>
15	#include <array>
16	#include <iterator>
17	#include <string>
18
19	int main()	1✔
20	{
21	using namespace hpx::string_util;
22
23	// Use tokenizer
24	{
25	std::string const test_string = ";;Hello\|world\|\|-foo--bar;yow;baz\|";	1✔
26	std::string answer[] = {"Hello", "world", "foo", "bar", "yow", "baz"};	1✔
27	using Tok = tokenizer<char_separator<char>>;
28	char_separator<char> sep("-;\|");	1✔
29	Tok t(test_string, sep);	1✔
30	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
31	}	6✔
32
33	{
34	std::string const test_string = ";;Hello\|world\|\|-foo--bar;yow;baz\|";	1✔
35	std::string answer[] = {"", "", "Hello", "\|", "world", "\|", "", "\|", "",	1✔
36	"foo", "", "bar", "yow", "baz", "\|", ""};	1✔
37	using Tok = tokenizer<char_separator<char>>;
38	char_separator<char> sep("-;", "\|", empty_token_policy::keep);	1✔
39	Tok t(test_string, sep);	1✔
40	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
41	}	16✔
42
43	{
44	std::string const test_string = "This,,is, a.test..";	1✔
45	std::string answer[] = {"This", "is", "a", "test"};	1✔
46	using Tok = tokenizer<char_delimiters_separator<char>>;
47	Tok t(test_string);	1✔
48	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
49	}	4✔
50
51	{
52	std::string const test_string =
53	"Field 1,\"embedded,comma\",quote \\\", escape \\\\";	1✔
54	std::string answer[] = {	1✔
55	"Field 1", "embedded,comma", "quote \"", " escape \\"};	1✔
56	using Tok = tokenizer<escaped_list_separator<char>>;
57	Tok t(test_string);	1✔
58	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
59	}	4✔
60
61	{
62	std::string const test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';";	1✔
63	std::string answer[] = {	1✔
64	"", "1", "", "2\"", "3;", "4", "5\\", "6,7", ""};	1✔
65	using Tok = tokenizer<escaped_list_separator<char>>;
66	escaped_list_separator<char> sep("\\^", ",;", "\"\'");	1✔
67	Tok t(test_string, sep);	1✔
68	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
69	}	9✔
70
71	{
72	std::string const test_string = "12252001";	1✔
73	std::string answer[] = {"12", "25", "2001"};	1✔
74	using Tok = tokenizer<offset_separator>;
75	std::array<int, 3> offsets = {{2, 2, 4}};	1✔
76	offset_separator func(offsets.begin(), offsets.end());	1✔
77	Tok t(test_string, func);	1✔
78	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
79	}	3✔
80
81	// Use token_iterator_generator
82	{
83	std::string const test_string = "This,,is, a.test..";	1✔
84	std::string answer[] = {"This", "is", "a", "test"};	1✔
85	using Iter =
86	token_iterator_generator<char_delimiters_separator<char>>::type;
87	Iter begin = make_token_iterator<std::string>(test_string.begin(),	1✔
88	test_string.end(), char_delimiters_separator<char>());	1✔
89	Iter end;	1✔
90	HPX_TEST(std::equal(begin, end, answer));	1✔
91	}	4✔
92
93	{
94	std::string const test_string =
95	"Field 1,\"embedded,comma\",quote \\\", escape \\\\";	1✔
96	std::string answer[] = {	1✔
97	"Field 1", "embedded,comma", "quote \"", " escape \\"};	1✔
98	using Iter =
99	token_iterator_generator<escaped_list_separator<char>>::type;
100	Iter begin = make_token_iterator<std::string>(test_string.begin(),	1✔
101	test_string.end(), escaped_list_separator<char>());	1✔
102	Iter begin_c(begin);	1✔
103	Iter end;	1✔
104	HPX_TEST(std::equal(begin, end, answer));	1✔
105
106	while (begin_c != end)	5✔
107	{
108	HPX_TEST(begin_c.at_end() == 0);	4✔
109	++begin_c;	4✔
110	}
111	HPX_TEST(begin_c.at_end());	1✔
112	}	4✔
113
114	{
115	std::string const test_string = "12252001";	1✔
116	std::string answer[] = {"12", "25", "2001"};	1✔
117	using Iter = token_iterator_generator<offset_separator>::type;
118	std::array<int, 3> offsets = {{2, 2, 4}};	1✔
119	offset_separator func(offsets.begin(), offsets.end());	1✔
120	Iter begin = make_token_iterator<std::string>(	1✔
121	test_string.begin(), test_string.end(), func);	1✔
122	Iter end = make_token_iterator<std::string>(	1✔
123	test_string.end(), test_string.end(), func);	1✔
124	HPX_TEST(std::equal(begin, end, answer));	1✔
125	}	3✔
126
127	// Test copying
128	{
129	std::string const test_string = "abcdef";	1✔
130	token_iterator_generator<offset_separator>::type beg, end, other;	1✔
131	std::array<int, 3> ar = {{1, 2, 3}};	1✔
132	offset_separator f(ar.begin(), ar.end());	1✔
133	beg = make_token_iterator<std::string>(	1✔
134	test_string.begin(), test_string.end(), f);	1✔
135
136	++beg;	1✔
137	other = beg;	1✔
138	++other;	1✔
139
140	HPX_TEST(*beg == "bc");	1✔
141	HPX_TEST(*other == "def");	1✔
142
143	other = make_token_iterator<std::string>(	1✔
144	test_string.begin(), test_string.end(), f);	1✔
145
146	HPX_TEST(*other == "a");	1✔
147	}	1✔
148
149	// Test non-default constructed char_separator
150	{
151	std::string const test_string = "how,are you, doing";	1✔
152	std::string answer[] = {"how", ",", "are you", ",", " doing"};	1✔
153	tokenizer t(	1✔
154	test_string, char_delimiters_separator<char>(true, ",", ""));	1✔
155	HPX_TEST(std::equal(t.begin(), t.end(), answer));	1✔
156	}	5✔
157
158	return hpx::util::report_errors();	1✔
159	}	×

STEllAR-GROUP / hpx / #848

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous