#855

Committed 20 Dec 2022 09:59PM UTC coverage: 86.55% (+0.04%) from 86.511%

Build # #855

Build Type

push

Committed by StellarBot

Commit Message

Merge #6112

6112: Modernize modules from levels 9 and 10 r=hkaiser a=hkaiser

working towards https://github.com/STEllAR-GROUP/hpx/issues/5497

Co-authored-by: Hartmut Kaiser <hartmut.kaiser@gmail.com>

Run Details

185 of 185 new or added lines in 30 files covered. (100.0%)

174495 of 201611 relevant lines covered (86.55%)

1898061.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.73

/libs/core/string_util/include/hpx/string_util/token_functions.hpp

//  Copyright (c) 2022 Hartmut Kaiser
//
//  SPDX-License-Identifier: BSL-1.0
//  Distributed under the Boost Software License, Version 1.0. (See accompanying
//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

// Copyright John R. Bandela 2001.

// See http://www.boost.org/libs/tokenizer/ for documentation.

// Revision History:
// 01 Oct 2004   Joaquin M Lopez Munoz
//      Workaround for a problem with string::assign in msvc-stlport
// 06 Apr 2004   John Bandela
//      Fixed a bug involving using char_delimiter with a true input iterator
// 28 Nov 2003   Robert Zeh and John Bandela
//      Converted into "fast" functions that avoid using += when
//      the supplied iterator isn't an input_iterator; based on
//      some work done at Archelon and a version that was checked into
//      the boost CVS for a short period of time.
// 20 Feb 2002   John Maddock
//      Removed using namespace std declarations and added
//      workaround for BOOST_NO_STDC_NAMESPACE (the library
//      can be safely mixed with regex).
// 06 Feb 2002   Jeremy Siek
//      Added char_separator.
// 02 Feb 2002   Jeremy Siek
//      Removed tabs and a little cleanup.

#pragma once

#include <hpx/config.hpp>
#include <hpx/assert.hpp>
#include <hpx/modules/errors.hpp>

#include <algorithm>    // for find_if
#include <cctype>
#include <cwctype>
#include <initializer_list>
#include <iterator>
#include <stdexcept>
#include <string>
#include <vector>

namespace hpx::string_util {

    //=========================================================================
    // The escaped_list_separator class. Which is a model of TokenizerFunction
    // An escaped list is a super-set of what is commonly known as a comma
    // separated value (csv) list.It is separated into fields by a comma or
    // other character. If the delimiting character is inside quotes, then it is
    // counted as a regular character.To allow for embedded quotes in a field,
    // there can be escape sequences using the \ much like C. The role of the
    // comma, the quotation mark, and the escape character (backslash \), can be
    // assigned to other characters.
    template <typename Char,
        typename Traits = typename std::basic_string<Char>::traits_type,
        typename Allocator = typename std::basic_string<Char>::allocator_type>
    class escaped_list_separator
    {
    private:
        using string_type = std::basic_string<Char, Traits, Allocator>;

        struct char_eq
        {
            Char e_;

            explicit char_eq(Char e) noexcept
              : e_(e)
            {
            }

            bool operator()(Char c) noexcept
            {
                return Traits::eq(e_, c);
            }
        };

        string_type escape_;
        string_type c_;
        string_type quote_;
        bool last_ = false;

        bool is_escape(Char e)
        {
            char_eq f(e);
            return std::find_if(escape_.begin(), escape_.end(), f) !=
                escape_.end();
        }

        bool is_c(Char e)
        {
            char_eq f(e);
            return std::find_if(c_.begin(), c_.end(), f) != c_.end();
        }

        bool is_quote(Char e)
        {
            char_eq f(e);
            return std::find_if(quote_.begin(), quote_.end(), f) !=
                quote_.end();
        }

        template <typename Iterator, typename Token>
        void do_escape(Iterator& next, Iterator end, Token& tok)
        {
            if (++next == end)
            {
                HPX_THROW_EXCEPTION(hpx::error::invalid_status,
                    "escaped_list_separator::do_escape",
                    "cannot end with escape");
            }

            if (Traits::eq(*next, 'n'))
            {
                tok += '\n';
                return;
            }
            else if (is_quote(*next) || is_c(*next) || is_escape(*next))
            {
                tok += *next;
                return;
            }
            else
            {
                HPX_THROW_EXCEPTION(hpx::error::invalid_status,
                    "escaped_list_separator::do_escape",
                    "unknown escape sequence");
            }
        }

    public:
        explicit escaped_list_separator(
            Char e = '\\', Char c = ',', Char q = '\"')
          : escape_(1, e)
          , c_(1, c)
          , quote_(1, q)
        {
        }

        escaped_list_separator(
            string_type e, string_type c, string_type q) noexcept
          : escape_(HPX_MOVE(e))
          , c_(HPX_MOVE(c))
          , quote_(HPX_MOVE(q))
        {
        }

        void reset() noexcept
        {
            last_ = false;
        }

        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            bool in_quote = false;
            tok = Token();

            if (next == end)
            {
                if (last_)
                {
                    last_ = false;
                    return true;
                }
                else
                {
                    return false;
                }
            }

            last_ = false;
            for (/**/; next != end; ++next)
            {
                if (is_escape(*next))
                {
                    do_escape(next, end, tok);
                }
                else if (is_c(*next))
                {
                    if (!in_quote)
                    {
                        // If we are not in quote, then we are done
                        ++next;

                        // The last character was a c, that means there is 1
                        // more blank field
                        last_ = true;
                        return true;
                    }
                    else
                    {
                        tok += *next;
                    }
                }
                else if (is_quote(*next))
                {
                    in_quote = !in_quote;
                }
                else
                {
                    tok += *next;
                }
            }
            return true;
        }
    };

    //=========================================================================
    // The classes here are used by offset_separator and char_separator to
    // implement faster assigning of tokens using assign instead of +=

    namespace detail {

        //=====================================================================
        // Tokenizer was broken for wide character separators, at least on
        // Windows, since CRT functions isspace etc only expect values in [0,
        // 0xFF]. Debug build asserts if higher values are passed in. The traits
        // extension class should take care of this. Assuming that the
        // conditional will always get optimized out in the function
        // implementations, argument types are not a problem since both forms of
        // character classifiers expect an int.
        template <typename Traits, int N>
        struct traits_extension_details : public Traits
        {
            using char_type = typename Traits::char_type;

            static bool isspace(char_type c) noexcept
            {
                return std::iswspace(c) != 0;
            }

            static bool ispunct(char_type c) noexcept
            {
                return std::iswpunct(c) != 0;
            }
        };

        template <typename Traits>
        struct traits_extension_details<Traits, 1> : public Traits
        {
            using char_type = typename Traits::char_type;

            static bool isspace(char_type c) noexcept
            {
                return std::isspace(c) != 0;
            }

            static bool ispunct(char_type c) noexcept
            {
                return std::ispunct(c) != 0;
            }
        };

        // In case there is no cwctype header, we implement the checks manually.
        // We make use of the fact that the tested categories should fit in
        // ASCII.
        template <typename Traits>
        struct traits_extension : public Traits
        {
            using char_type = typename Traits::char_type;

            static bool isspace(char_type c) noexcept
            {
                return traits_extension_details<Traits,
                    sizeof(char_type)>::isspace(c);
            }

            static bool ispunct(char_type c) noexcept
            {
                return traits_extension_details<Traits,
                    sizeof(char_type)>::ispunct(c);
            }
        };

        // The assign_or_plus_equal struct contains functions that implement
        // assign, +=, and clearing based on the iterator type. The generic case
        // does nothing for plus_equal and clearing, while passing through the
        // call for assign.
        //
        // When an input iterator is being used, the situation is reversed. The
        // assign method does nothing, plus_equal invokes operator +=, and the
        // clearing method sets the supplied token to the default token
        // constructor's result.
        template <typename IteratorTag>
        struct assign_or_plus_equal
        {
            template <typename Iterator, typename Token>
            static constexpr void assign(Iterator b, Iterator e, Token& t)
            {
                t.assign(b, e);
            }

            template <typename Token, typename Value>
            static constexpr void plus_equal(Token&, Value&&) noexcept
            {
            }

            // If we are doing an assign, there is no need for the the clear.
            template <typename Token>
            static constexpr void clear(Token&) noexcept
            {
            }
        };

        template <>
        struct assign_or_plus_equal<std::input_iterator_tag>
        {
            template <class Iterator, class Token>
            static constexpr void assign(Iterator, Iterator, Token&) noexcept
            {
            }

            template <class Token, class Value>
            static constexpr void plus_equal(Token& t, Value&& v)
            {
                t += HPX_FORWARD(Value, v);
            }

            template <class Token>
            static constexpr void clear(Token& t)
            {
                t = Token();
            }
        };

        template <typename Iterator>
        struct class_iterator_category
        {
            using type = typename Iterator::iterator_category;
        };

        // This portably gets the iterator_tag without partial template
        // specialization
        template <typename Iterator>
        struct get_iterator_category
        {
            using iterator_category =
                std::conditional_t<std::is_pointer_v<Iterator>,
                    std::random_access_iterator_tag,
                    typename class_iterator_category<Iterator>::type>;
        };
    }    // namespace detail

    //===========================================================================
    // The offset_separator class, which is a model of TokenizerFunction. Offset
    // breaks a string into tokens based on a range of offsets
    class offset_separator
    {
    private:
        std::vector<int> offsets_;
        unsigned int current_offset_ = 0;
        bool wrap_offsets_ = true;
        bool return_partial_last_ = true;

    public:
        template <typename Iter>
        offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
            bool return_partial_last = true)
          : offsets_(begin, end)
          , wrap_offsets_(wrap_offsets)
          , return_partial_last_(return_partial_last)
        {
        }

        offset_separator(std::initializer_list<int> init,
            bool wrap_offsets = true, bool return_partial_last = true)
          : offsets_(HPX_MOVE(init))
          , wrap_offsets_(wrap_offsets)
          , return_partial_last_(return_partial_last)
        {
        }

        offset_separator()
          : offsets_(1, 1)
        {
        }

        void reset()
        {
            current_offset_ = 0;
        }

        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            using assigner = detail::assign_or_plus_equal<typename detail::
                    get_iterator_category<InputIterator>::iterator_category>;

            HPX_ASSERT(!offsets_.empty());

            assigner::clear(tok);
            InputIterator start(next);

            if (next == end)
            {
                return false;
            }

            if (current_offset_ == offsets_.size())
            {
                if (wrap_offsets_)
                {
                    current_offset_ = 0;
                }
                else
                {
                    return false;
                }
            }

            int c = offsets_[current_offset_];
            int i = 0;
            for (; i < c; ++i)
            {
                if (next == end)
                {
                    break;
                }
                assigner::plus_equal(tok, *next++);
            }
            assigner::assign(start, next, tok);

            if (!return_partial_last_)
            {
                if (i < (c - 1))
                {
                    return false;
                }
            }

            ++current_offset_;
            return true;
        }
    };

    //=========================================================================
    // The char_separator class breaks a sequence of characters into tokens
    // based on the character delimiters (very much like bad old strtok). A
    // delimiter character can either be kept or dropped. A kept delimiter shows
    // up as an output token, whereas a dropped delimiter does not.

    // This class replaces the char_delimiters_separator class. The constructor
    // for the char_delimiters_separator class was too confusing and needed to
    // be deprecated. However, because of the default arguments to the
    // constructor, adding the new constructor would cause ambiguity, so instead
    // I deprecated the whole class. The implementation of the class was also
    // simplified considerably.
    enum class empty_token_policy
    {
        drop,
        keep
    };

    template <typename Char,
        typename Traits = typename std::basic_string<Char>::traits_type,
        typename Allocator = typename std::basic_string<Char>::allocator_type>
    class char_separator
    {
        using traits_type = detail::traits_extension<Traits>;
        using string_type = std::basic_string<Char, Traits, Allocator>;

    public:
        explicit char_separator(Char const* dropped_delims,
            Char const* kept_delims = nullptr,
            empty_token_policy empty_tokens = empty_token_policy::drop)
          : m_dropped_delims(dropped_delims)
          , m_use_ispunct(false)
          , m_use_isspace(false)
          , m_empty_tokens(empty_tokens)
        {
            if (kept_delims)
                m_kept_delims = kept_delims;
        }

        // use ispunct() for kept delimiters and isspace for dropped.
        char_separator() = default;

        constexpr void reset() noexcept {}

        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            using assigner = detail::assign_or_plus_equal<typename detail::
                    get_iterator_category<InputIterator>::iterator_category>;

            assigner::clear(tok);

            // skip past all dropped_delims
            if (m_empty_tokens == empty_token_policy::drop)
            {
                for (/**/; next != end && is_dropped(*next); ++next)
                {
                }
            }

            InputIterator start(next);

            if (m_empty_tokens == empty_token_policy::drop)
            {
                if (next == end)
                    return false;

                // if we are on a kept_delims move past it and stop
                if (is_kept(*next))
                {
                    assigner::plus_equal(tok, *next);
                    ++next;
                }
                else
                {
                    // append all the non delim characters
                    for (/**/;
                         next != end && !is_dropped(*next) && !is_kept(*next);
                         ++next)
                    {
                        assigner::plus_equal(tok, *next);
                    }
                }
            }
            else
            {
                // m_empty_tokens == empty_token_policy::keep

                // Handle empty token at the end
                if (next == end)
                {
                    if (!m_output_done)
                    {
                        m_output_done = true;
                        assigner::assign(start, next, tok);
                        return true;
                    }
                    else
                    {
                        return false;
                    }
                }

                if (is_kept(*next))
                {
                    if (!m_output_done)
                    {
                        m_output_done = true;
                    }
                    else
                    {
                        assigner::plus_equal(tok, *next);
                        ++next;
                        m_output_done = false;
                    }
                }
                else if (!m_output_done && is_dropped(*next))
                {
                    m_output_done = true;
                }
                else
                {
                    if (is_dropped(*next))
                    {
                        start = ++next;
                    }

                    for (/**/;
                         next != end && !is_dropped(*next) && !is_kept(*next);
                         ++next)
                    {
                        assigner::plus_equal(tok, *next);
                    }

                    m_output_done = true;
                }
            }

            assigner::assign(start, next, tok);
            return true;
        }

    private:
        string_type m_kept_delims;
        string_type m_dropped_delims;
        bool m_use_ispunct = true;
        bool m_use_isspace = true;
        empty_token_policy m_empty_tokens = empty_token_policy::drop;
        bool m_output_done = false;

        bool is_kept(Char E) const
        {
            if (m_kept_delims.length())
            {
                return m_kept_delims.find(E) != string_type::npos;
            }
            else if (m_use_ispunct)
            {
                return traits_type::ispunct(E) != 0;
            }
            return false;
        }

        bool is_dropped(Char E) const
        {
            if (m_dropped_delims.length())
            {
                return m_dropped_delims.find(E) != string_type::npos;
            }
            else if (m_use_isspace)
            {
                return traits_type::isspace(E) != 0;
            }
            return false;
        }
    };

    //===========================================================================
    // The char_delimiters_separator class, which is a model of
    // TokenizerFunction. char_delimiters_separator breaks a string into tokens
    // based on character delimiters. There are 2 types of delimiters.
    // Returnable delimiters can be returned as tokens. These are often
    // punctuation. Nonreturnable delimiters cannot be returned as tokens. These
    // are often whitespace

    template <typename Char,
        typename Traits = typename std::basic_string<Char>::traits_type,
        typename Allocator = typename std::basic_string<Char>::allocator_type>
    class char_delimiters_separator
    {
    private:
        using traits_type = detail::traits_extension<Traits>;
        using string_type = std::basic_string<Char, Traits, Allocator>;

        string_type returnable_;
        string_type nonreturnable_;
        bool return_delims_;
        bool no_ispunct_;
        bool no_isspace_;

        bool is_ret(Char E) const noexcept
        {
            if (returnable_.length())
            {
                return returnable_.find(E) != string_type::npos;
            }
            else
            {
                if (no_ispunct_)
                {
                    return false;
                }
                else
                {
                    int r = traits_type::ispunct(E);
                    return r != 0;
                }
            }
        }

        bool is_nonret(Char E) const noexcept
        {
            if (nonreturnable_.length())
            {
                return nonreturnable_.find(E) != string_type::npos;
            }
            else
            {
                if (no_isspace_)
                {
                    return false;
                }
                else
                {
                    int r = traits_type::isspace(E);
                    return r != 0;
                }
            }
        }

    public:
        explicit char_delimiters_separator(bool return_delims = false,
            Char const* returnable = nullptr,
            Char const* nonreturnable = nullptr)
          : returnable_(returnable ? returnable : string_type().c_str())
          , nonreturnable_(
                nonreturnable ? nonreturnable : string_type().c_str())
          , return_delims_(return_delims)
          , no_ispunct_(returnable != nullptr)
          , no_isspace_(nonreturnable != nullptr)
        {
        }

        constexpr void reset() noexcept {}

    public:
        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            tok = Token();

            // skip past all nonreturnable delims
            // skip past the returnable only if we are not returning delims
            for (/**/; next != end &&
                 (is_nonret(*next) || (is_ret(*next) && !return_delims_));
                 ++next)
            {
            }

            if (next == end)
            {
                return false;
            }

            // if we are to return delims and we are one a returnable one move
            // past it and stop
            if (is_ret(*next) && return_delims_)
            {
                tok += *next;
                ++next;
            }
            else
            {
                // append all the non delim characters
                for (/**/; next != end && !is_nonret(*next) && !is_ret(*next);
                     ++next)
                {
                    tok += *next;
                }
            }

            return true;
        }
    };
}    // namespace hpx::string_util

1	// Copyright (c) 2022 Hartmut Kaiser
2	//
3	// SPDX-License-Identifier: BSL-1.0
4	// Distributed under the Boost Software License, Version 1.0. (See accompanying
5	// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
7	// Copyright John R. Bandela 2001.
8
9	// See http://www.boost.org/libs/tokenizer/ for documentation.
10
11	// Revision History:
12	// 01 Oct 2004 Joaquin M Lopez Munoz
13	// Workaround for a problem with string::assign in msvc-stlport
14	// 06 Apr 2004 John Bandela
15	// Fixed a bug involving using char_delimiter with a true input iterator
16	// 28 Nov 2003 Robert Zeh and John Bandela
17	// Converted into "fast" functions that avoid using += when
18	// the supplied iterator isn't an input_iterator; based on
19	// some work done at Archelon and a version that was checked into
20	// the boost CVS for a short period of time.
21	// 20 Feb 2002 John Maddock
22	// Removed using namespace std declarations and added
23	// workaround for BOOST_NO_STDC_NAMESPACE (the library
24	// can be safely mixed with regex).
25	// 06 Feb 2002 Jeremy Siek
26	// Added char_separator.
27	// 02 Feb 2002 Jeremy Siek
28	// Removed tabs and a little cleanup.
29
30	#pragma once
31
32	#include <hpx/config.hpp>
33	#include <hpx/assert.hpp>
34	#include <hpx/modules/errors.hpp>
35
36	#include <algorithm> // for find_if
37	#include <cctype>
38	#include <cwctype>
39	#include <initializer_list>
40	#include <iterator>
41	#include <stdexcept>
42	#include <string>
43	#include <vector>
44
45	namespace hpx::string_util {
46
47	//=========================================================================
48	// The escaped_list_separator class. Which is a model of TokenizerFunction
49	// An escaped list is a super-set of what is commonly known as a comma
50	// separated value (csv) list.It is separated into fields by a comma or
51	// other character. If the delimiting character is inside quotes, then it is
52	// counted as a regular character.To allow for embedded quotes in a field,
53	// there can be escape sequences using the \ much like C. The role of the
54	// comma, the quotation mark, and the escape character (backslash \), can be
55	// assigned to other characters.
56	template <typename Char,
57	typename Traits = typename std::basic_string<Char>::traits_type,
58	typename Allocator = typename std::basic_string<Char>::allocator_type>
59	class escaped_list_separator	13,909✔
60	{
61	private:
62	using string_type = std::basic_string<Char, Traits, Allocator>;
63
64	struct char_eq
65	{
66	Char e_;
67
68	explicit char_eq(Char e) noexcept	737,110✔
69	: e_(e)	737,110✔
70	{
71	}	737,110✔
72
73	bool operator()(Char c) noexcept	1,219,264✔
74	{
75	return Traits::eq(e_, c);	1,219,264✔
76	}
77	};
78
79	string_type escape_;
80	string_type c_;
81	string_type quote_;
82	bool last_ = false;	1,983✔
83
84	bool is_escape(Char e)	247,907✔
85	{
86	char_eq f(e);	247,907✔
87	return std::find_if(escape_.begin(), escape_.end(), f) !=	495,814✔
88	escape_.end();	247,907✔
89	}
90
91	bool is_c(Char e)	247,893✔
92	{
93	char_eq f(e);	247,893✔
94	return std::find_if(c_.begin(), c_.end(), f) != c_.end();	247,893✔
95	}
96
97	bool is_quote(Char e)	241,310✔
98	{
99	char_eq f(e);	241,310✔
100	return std::find_if(quote_.begin(), quote_.end(), f) !=	482,620✔
101	quote_.end();	241,310✔
102	}
103
104	template <typename Iterator, typename Token>
105	void do_escape(Iterator& next, Iterator end, Token& tok)	17✔
106	{
107	if (++next == end)	17✔
108	{
109	HPX_THROW_EXCEPTION(hpx::error::invalid_status,	×
110	"escaped_list_separator::do_escape",
111	"cannot end with escape");
112	}
113
114	if (Traits::eq(*next, 'n'))	17✔
115	{
116	tok += '\n';	×
117	return;	×
118	}
119	else if (is_quote(next) \|\| is_c(next) \|\| is_escape(*next))	17✔
120	{
121	tok += *next;	17✔
122	return;	17✔
123	}
124	else
125	{
126	HPX_THROW_EXCEPTION(hpx::error::invalid_status,	×
127	"escaped_list_separator::do_escape",
128	"unknown escape sequence");
129	}
130	}	17✔
131
132	public:
133	explicit escaped_list_separator(	4✔
134	Char e = '\\', Char c = ',', Char q = '\"')
135	: escape_(1, e)	4✔
136	, c_(1, c)	4✔
137	, quote_(1, q)	4✔
138	{
139	}	4✔
140
141	escaped_list_separator(	1,979✔
142	string_type e, string_type c, string_type q) noexcept
143	: escape_(HPX_MOVE(e))	1,979✔
144	, c_(HPX_MOVE(c))	1,979✔
145	, quote_(HPX_MOVE(q))	1,979✔
146	{
147	}	1,979✔
148
149	void reset() noexcept	3,963✔
150	{
151	last_ = false;	3,963✔
152	}	3,963✔
153
154	template <typename InputIterator, typename Token>
155	bool operator()(InputIterator& next, InputIterator end, Token& tok)	10,549✔
156	{
157	bool in_quote = false;	10,549✔
158	tok = Token();	10,549✔
159
160	if (next == end)	10,549✔
161	{
162	if (last_)	2,013✔
163	{
164	last_ = false;	30✔
165	return true;	30✔
166	}
167	else
168	{
169	return false;	1,983✔
170	}
171	}
172
173	last_ = false;	8,536✔
174	for (/**/; next != end; ++next)	249,856✔
175	{
176	if (is_escape(*next))	247,903✔
177	{
178	do_escape(next, end, tok);	17✔
179	}	17✔
180	else if (is_c(*next))	247,886✔
181	{
182	if (!in_quote)	6,593✔
183	{
184	// If we are not in quote, then we are done
185	++next;	6,583✔
186
187	// The last character was a c, that means there is 1
188	// more blank field
189	last_ = true;	6,583✔
190	return true;	6,583✔
191	}
192	else
193	{
194	tok += *next;	10✔
195	}
196	}	10✔
197	else if (is_quote(*next))	241,293✔
198	{
199	in_quote = !in_quote;	20✔
200	}	20✔
201	else
202	{
203	tok += *next;	241,273✔
204	}
205	}	241,320✔
206	return true;	1,953✔
207	}	10,549✔
208	};
209
210	//=========================================================================
211	// The classes here are used by offset_separator and char_separator to
212	// implement faster assigning of tokens using assign instead of +=
213
214	namespace detail {
215
216	//=====================================================================
217	// Tokenizer was broken for wide character separators, at least on
218	// Windows, since CRT functions isspace etc only expect values in [0,
219	// 0xFF]. Debug build asserts if higher values are passed in. The traits
220	// extension class should take care of this. Assuming that the
221	// conditional will always get optimized out in the function
222	// implementations, argument types are not a problem since both forms of
223	// character classifiers expect an int.
224	template <typename Traits, int N>
225	struct traits_extension_details : public Traits
226	{
227	using char_type = typename Traits::char_type;
228
229	static bool isspace(char_type c) noexcept
230	{
231	return std::iswspace(c) != 0;
232	}
233
234	static bool ispunct(char_type c) noexcept
235	{
236	return std::iswpunct(c) != 0;
237	}
238	};
239
240	template <typename Traits>
241	struct traits_extension_details<Traits, 1> : public Traits
242	{
243	using char_type = typename Traits::char_type;
244
245	static bool isspace(char_type c) noexcept	98✔
246	{
247	return std::isspace(c) != 0;	98✔
248	}
249
250	static bool ispunct(char_type c) noexcept	92✔
251	{
252	return std::ispunct(c) != 0;	92✔
253	}
254	};
255
256	// In case there is no cwctype header, we implement the checks manually.
257	// We make use of the fact that the tested categories should fit in
258	// ASCII.
259	template <typename Traits>
260	struct traits_extension : public Traits
261	{
262	using char_type = typename Traits::char_type;
263
264	static bool isspace(char_type c) noexcept	98✔
265	{
266	return traits_extension_details<Traits,	98✔
267	sizeof(char_type)>::isspace(c);	98✔
268	}
269
270	static bool ispunct(char_type c) noexcept	92✔
271	{
272	return traits_extension_details<Traits,	92✔
273	sizeof(char_type)>::ispunct(c);	92✔
274	}
275	};
276
277	// The assign_or_plus_equal struct contains functions that implement
278	// assign, +=, and clearing based on the iterator type. The generic case
279	// does nothing for plus_equal and clearing, while passing through the
280	// call for assign.
281	//
282	// When an input iterator is being used, the situation is reversed. The
283	// assign method does nothing, plus_equal invokes operator +=, and the
284	// clearing method sets the supplied token to the default token
285	// constructor's result.
286	template <typename IteratorTag>
287	struct assign_or_plus_equal
288	{
289	template <typename Iterator, typename Token>
290	static constexpr void assign(Iterator b, Iterator e, Token& t)	66,995✔
291	{
292	t.assign(b, e);	66,995✔
293	}	66,995✔
294
295	template <typename Token, typename Value>
296	static constexpr void plus_equal(Token&, Value&&) noexcept	792,422✔
297	{
298	}	792,422✔
299
300	// If we are doing an assign, there is no need for the the clear.
301	template <typename Token>
302	static constexpr void clear(Token&) noexcept	101,108✔
303	{
304	}	101,108✔
305	};
306
307	template <>
308	struct assign_or_plus_equal<std::input_iterator_tag>
309	{
310	template <class Iterator, class Token>
311	static constexpr void assign(Iterator, Iterator, Token&) noexcept
312	{
313	}
314
315	template <class Token, class Value>
316	static constexpr void plus_equal(Token& t, Value&& v)
317	{
318	t += HPX_FORWARD(Value, v);
319	}
320
321	template <class Token>
322	static constexpr void clear(Token& t)
323	{
324	t = Token();
325	}
326	};
327
328	template <typename Iterator>
329	struct class_iterator_category
330	{
331	using type = typename Iterator::iterator_category;
332	};
333
334	// This portably gets the iterator_tag without partial template
335	// specialization
336	template <typename Iterator>
337	struct get_iterator_category
338	{
339	using iterator_category =
340	std::conditional_t<std::is_pointer_v<Iterator>,
341	std::random_access_iterator_tag,
342	typename class_iterator_category<Iterator>::type>;
343	};
344	} // namespace detail
345
346	//===========================================================================
347	// The offset_separator class, which is a model of TokenizerFunction. Offset
348	// breaks a string into tokens based on a range of offsets
349	class offset_separator	63✔
350	{
351	private:
352	std::vector<int> offsets_;
353	unsigned int current_offset_ = 0;	8✔
354	bool wrap_offsets_ = true;	3✔
355	bool return_partial_last_ = true;	3✔
356
357	public:
358	template <typename Iter>
359	offset_separator(Iter begin, Iter end, bool wrap_offsets = true,	4✔
360	bool return_partial_last = true)
361	: offsets_(begin, end)	4✔
362	, wrap_offsets_(wrap_offsets)	4✔
363	, return_partial_last_(return_partial_last)	4✔
364	{
365	}	4✔
366
367	offset_separator(std::initializer_list<int> init,	1✔
368	bool wrap_offsets = true, bool return_partial_last = true)
369	: offsets_(HPX_MOVE(init))	1✔
370	, wrap_offsets_(wrap_offsets)	1✔
371	, return_partial_last_(return_partial_last)	1✔
372	{
373	}	1✔
374
375	offset_separator()	3✔
376	: offsets_(1, 1)	3✔
377	{
378	}	3✔
379
380	void reset()	10✔
381	{
382	current_offset_ = 0;	10✔
383	}	10✔
384
385	template <typename InputIterator, typename Token>
386	bool operator()(InputIterator& next, InputIterator end, Token& tok)	20✔
387	{
388	using assigner = detail::assign_or_plus_equal<typename detail::
389	get_iterator_category<InputIterator>::iterator_category>;
390
391	HPX_ASSERT(!offsets_.empty());	20✔
392
393	assigner::clear(tok);	20✔
394	InputIterator start(next);	20✔
395
396	if (next == end)	20✔
397	{
398	return false;	4✔
399	}
400
401	if (current_offset_ == offsets_.size())	16✔
402	{
403	if (wrap_offsets_)	×
404	{
405	current_offset_ = 0;	×
406	}	×
407	else
408	{
409	return false;	×
410	}
411	}	×
412
413	int c = offsets_[current_offset_];	16✔
414	int i = 0;	16✔
415	for (; i < c; ++i)	55✔
416	{
417	if (next == end)	39✔
418	{
419	break;	×
420	}
421	assigner::plus_equal(tok, *next++);	39✔
422	}	39✔
423	assigner::assign(start, next, tok);	16✔
424
425	if (!return_partial_last_)	16✔
426	{
427	if (i < (c - 1))	×
428	{
429	return false;	×
430	}
431	}	×
432
433	++current_offset_;	16✔
434	return true;	16✔
435	}	20✔
436	};
437
438	//=========================================================================
439	// The char_separator class breaks a sequence of characters into tokens
440	// based on the character delimiters (very much like bad old strtok). A
441	// delimiter character can either be kept or dropped. A kept delimiter shows
442	// up as an output token, whereas a dropped delimiter does not.
443
444	// This class replaces the char_delimiters_separator class. The constructor
445	// for the char_delimiters_separator class was too confusing and needed to
446	// be deprecated. However, because of the default arguments to the
447	// constructor, adding the new constructor would cause ambiguity, so instead
448	// I deprecated the whole class. The implementation of the class was also
449	// simplified considerably.
450	enum class empty_token_policy
451	{
452	drop,
453	keep
454	};
455
456	template <typename Char,
457	typename Traits = typename std::basic_string<Char>::traits_type,
458	typename Allocator = typename std::basic_string<Char>::allocator_type>
459	class char_separator	425,723✔
460	{
461	using traits_type = detail::traits_extension<Traits>;
462	using string_type = std::basic_string<Char, Traits, Allocator>;
463
464	public:
465	explicit char_separator(Char const* dropped_delims,	44,527✔
466	Char const* kept_delims = nullptr,
467	empty_token_policy empty_tokens = empty_token_policy::drop)
468	: m_dropped_delims(dropped_delims)	44,527✔
469	, m_use_ispunct(false)	44,527✔
470	, m_use_isspace(false)	44,527✔
471	, m_empty_tokens(empty_tokens)	44,527✔
472	{
473	if (kept_delims)	44,527✔
474	m_kept_delims = kept_delims;	97✔
475	}	44,527✔
476
477	// use ispunct() for kept delimiters and isspace for dropped.
478	char_separator() = default;	2✔
479
480	constexpr void reset() noexcept {}	138,815✔
481
482	template <typename InputIterator, typename Token>
483	bool operator()(InputIterator& next, InputIterator end, Token& tok)	101,088✔
484	{
485	using assigner = detail::assign_or_plus_equal<typename detail::
486	get_iterator_category<InputIterator>::iterator_category>;
487
488	assigner::clear(tok);	101,088✔
489
490	// skip past all dropped_delims
491	if (m_empty_tokens == empty_token_policy::drop)	101,088✔
492	{
493	for (/*/; next != end && is_dropped(next); ++next)	126,422✔
494	{
495	}	25,552✔
496	}	100,870✔
497
498	InputIterator start(next);	101,088✔
499
500	if (m_empty_tokens == empty_token_policy::drop)	101,088✔
501	{
502	if (next == end)	100,870✔
503	return false;	34,012✔
504
505	// if we are on a kept_delims move past it and stop
506	if (is_kept(*next))	66,858✔
507	{
508	assigner::plus_equal(tok, *next);	2✔
509	++next;	2✔
510	}	2✔
511	else
512	{
513	// append all the non delim characters
514	for (/**/;	1,696,474✔
515	next != end && !is_dropped(next) && !is_kept(next);	848,237✔
516	++next)	781,381✔
517	{
518	assigner::plus_equal(tok, *next);	781,381✔
519	}	781,381✔
520	}
521	}	66,858✔
522	else
523	{
524	// m_empty_tokens == empty_token_policy::keep
525
526	// Handle empty token at the end
527	if (next == end)	218✔
528	{
529	if (!m_output_done)	98✔
530	{
531	m_output_done = true;	1✔
532	assigner::assign(start, next, tok);	1✔
533	return true;	1✔
534	}
535	else
536	{
537	return false;	97✔
538	}
539	}
540
541	if (is_kept(*next))	120✔
542	{
543	if (!m_output_done)	5✔
544	{
545	m_output_done = true;	1✔
546	}	1✔
547	else
548	{
549	assigner::plus_equal(tok, *next);	4✔
550	++next;	4✔
551	m_output_done = false;	4✔
552	}
553	}	5✔
554	else if (!m_output_done && is_dropped(*next))	115✔
555	{
556	m_output_done = true;	2✔
557	}	2✔
558	else
559	{
560	if (is_dropped(*next))	113✔
561	{
562	start = ++next;	16✔
563	}	16✔
564
565	for (/**/;	22,218✔
566	next != end && !is_dropped(next) && !is_kept(next);	11,109✔
567	++next)	10,996✔
568	{
569	assigner::plus_equal(tok, *next);	10,996✔
570	}	10,996✔
571
572	m_output_done = true;	113✔
573	}
574	}
575
576	assigner::assign(start, next, tok);	66,978✔
577	return true;	66,978✔
578	}	101,088✔
579
580	private:
581	string_type m_kept_delims;
582	string_type m_dropped_delims;
583	bool m_use_ispunct = true;	2✔
584	bool m_use_isspace = true;	2✔
585	empty_token_policy m_empty_tokens = empty_token_policy::drop;	2✔
586	bool m_output_done = false;	44,529✔
587
588	bool is_kept(Char E) const	859,360✔
589	{
590	if (m_kept_delims.length())	859,360✔
591	{
592	return m_kept_delims.find(E) != string_type::npos;	40✔
593	}
594	else if (m_use_ispunct)	859,320✔
595	{
596	return traits_type::ispunct(E) != 0;	34✔
597	}
598	return false;	859,286✔
599	}	859,360✔
600
601	bool is_dropped(Char E) const	908,313✔
602	{
603	if (m_dropped_delims.length())	908,313✔
604	{
605	return m_dropped_delims.find(E) != string_type::npos;	908,267✔
606	}
607	else if (m_use_isspace)	46✔
608	{
609	return traits_type::isspace(E) != 0;	46✔
610	}
611	return false;	×
612	}	908,313✔
613	};
614
615	//===========================================================================
616	// The char_delimiters_separator class, which is a model of
617	// TokenizerFunction. char_delimiters_separator breaks a string into tokens
618	// based on character delimiters. There are 2 types of delimiters.
619	// Returnable delimiters can be returned as tokens. These are often
620	// punctuation. Nonreturnable delimiters cannot be returned as tokens. These
621	// are often whitespace
622
623	template <typename Char,
624	typename Traits = typename std::basic_string<Char>::traits_type,
625	typename Allocator = typename std::basic_string<Char>::allocator_type>
626	class char_delimiters_separator	56✔
627	{
628	private:
629	using traits_type = detail::traits_extension<Traits>;
630	using string_type = std::basic_string<Char, Traits, Allocator>;
631
632	string_type returnable_;
633	string_type nonreturnable_;
634	bool return_delims_;
635	bool no_ispunct_;
636	bool no_isspace_;
637
638	bool is_ret(Char E) const noexcept	86✔
639	{
640	if (returnable_.length())	86✔
641	{
642	return returnable_.find(E) != string_type::npos;	28✔
643	}
644	else
645	{
646	if (no_ispunct_)	58✔
647	{
648	return false;	×
649	}
650	else
651	{
652	int r = traits_type::ispunct(E);	58✔
653	return r != 0;	58✔
654	}
655	}
656	}	86✔
657
658	bool is_nonret(Char E) const noexcept	75✔
659	{
660	if (nonreturnable_.length())	75✔
661	{
662	return nonreturnable_.find(E) != string_type::npos;	×
663	}
664	else
665	{
666	if (no_isspace_)	75✔
667	{
668	return false;	23✔
669	}
670	else
671	{
672	int r = traits_type::isspace(E);	52✔
673	return r != 0;	52✔
674	}
675	}
676	}	75✔
677
678	public:
679	explicit char_delimiters_separator(bool return_delims = false,	4✔
680	Char const* returnable = nullptr,
681	Char const* nonreturnable = nullptr)
682	: returnable_(returnable ? returnable : string_type().c_str())	4✔
683	, nonreturnable_(	8✔
684	nonreturnable ? nonreturnable : string_type().c_str())	4✔
685	, return_delims_(return_delims)	4✔
686	, no_ispunct_(returnable != nullptr)	4✔
687	, no_isspace_(nonreturnable != nullptr)	4✔
688	{
689	}	4✔
690
691	constexpr void reset() noexcept {}	5✔
692
693	public:
694	template <typename InputIterator, typename Token>
695	bool operator()(InputIterator& next, InputIterator end, Token& tok)	16✔
696	{
697	tok = Token();	16✔
698
699	// skip past all nonreturnable delims
700	// skip past the returnable only if we are not returning delims
701	for (/**/; next != end &&	57✔
702	(is_nonret(next) \|\| (is_ret(next) && !return_delims_));	27✔
703	++next)	14✔
704	{
705	}	14✔
706
707	if (next == end)	16✔
708	{
709	return false;	3✔
710	}
711
712	// if we are to return delims and we are one a returnable one move
713	// past it and stop
714	if (is_ret(*next) && return_delims_)	13✔
715	{
716	tok += *next;	2✔
717	++next;	2✔
718	}	2✔
719	else
720	{
721	// append all the non delim characters
722	for (/*/; next != end && !is_nonret(next) && !is_ret(*next);	49✔
723	++next)	38✔
724	{
725	tok += *next;	38✔
726	}	38✔
727	}
728
729	return true;	13✔
730	}	16✔
731	};
732	} // namespace hpx::string_util

STEllAR-GROUP / hpx / #855

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous