#862

Committed 10 Jan 2023 05:30PM UTC coverage: 86.582% (-0.05%) from 86.634%

Build # #862

Build Type

push

Committed by StellarBot

Commit Message

Merge #6130

6130: Remove the mutex lock in the critical path of get_partitioner. r=hkaiser a=JiakunYan

Remove the mutex lock in the critical path of hpx::resource::detail::get_partitioner.

The protected variable `partitioner_ref` is only set once during initialization.

Co-authored-by: Jiakun Yan <jiakunyan1998@gmail.com>

Run Details

6 of 6 new or added lines in 1 file covered. (100.0%)

174767 of 201851 relevant lines covered (86.58%)

2069816.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.73

/libs/core/string_util/include/hpx/string_util/token_functions.hpp

//  Copyright (c) 2022 Hartmut Kaiser
//
//  SPDX-License-Identifier: BSL-1.0
//  Distributed under the Boost Software License, Version 1.0. (See accompanying
//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

// Copyright John R. Bandela 2001.

// See http://www.boost.org/libs/tokenizer/ for documentation.

// Revision History:
// 01 Oct 2004   Joaquin M Lopez Munoz
//      Workaround for a problem with string::assign in msvc-stlport
// 06 Apr 2004   John Bandela
//      Fixed a bug involving using char_delimiter with a true input iterator
// 28 Nov 2003   Robert Zeh and John Bandela
//      Converted into "fast" functions that avoid using += when
//      the supplied iterator isn't an input_iterator; based on
//      some work done at Archelon and a version that was checked into
//      the boost CVS for a short period of time.
// 20 Feb 2002   John Maddock
//      Removed using namespace std declarations and added
//      workaround for BOOST_NO_STDC_NAMESPACE (the library
//      can be safely mixed with regex).
// 06 Feb 2002   Jeremy Siek
//      Added char_separator.
// 02 Feb 2002   Jeremy Siek
//      Removed tabs and a little cleanup.

#pragma once

#include <hpx/config.hpp>
#include <hpx/assert.hpp>
#include <hpx/modules/errors.hpp>

#include <algorithm>    // for find_if
#include <cctype>
#include <cstddef>
#include <cwctype>
#include <initializer_list>
#include <iterator>
#include <stdexcept>
#include <string>
#include <vector>

namespace hpx::string_util {

    //=========================================================================
    // The escaped_list_separator class. Which is a model of TokenizerFunction
    // An escaped list is a super-set of what is commonly known as a comma
    // separated value (csv) list.It is separated into fields by a comma or
    // other character. If the delimiting character is inside quotes, then it is
    // counted as a regular character.To allow for embedded quotes in a field,
    // there can be escape sequences using the \ much like C. The role of the
    // comma, the quotation mark, and the escape character (backslash \), can be
    // assigned to other characters.
    template <typename Char,
        typename Traits = typename std::basic_string<Char>::traits_type,
        typename Allocator = typename std::basic_string<Char>::allocator_type>
    class escaped_list_separator
    {
    private:
        using string_type = std::basic_string<Char, Traits, Allocator>;

        struct char_eq
        {
            Char e_;

            explicit char_eq(Char e) noexcept
              : e_(e)
            {
            }

            bool operator()(Char c) noexcept
            {
                return Traits::eq(e_, c);
            }
        };

        string_type escape_;
        string_type c_;
        string_type quote_;
        bool last_ = false;

        bool is_escape(Char e)
        {
            char_eq f(e);
            return std::find_if(escape_.begin(), escape_.end(), f) !=
                escape_.end();
        }

        bool is_c(Char e)
        {
            char_eq f(e);
            return std::find_if(c_.begin(), c_.end(), f) != c_.end();
        }

        bool is_quote(Char e)
        {
            char_eq f(e);
            return std::find_if(quote_.begin(), quote_.end(), f) !=
                quote_.end();
        }

        template <typename Iterator, typename Token>
        void do_escape(Iterator& next, Iterator end, Token& tok)
        {
            if (++next == end)
            {
                HPX_THROW_EXCEPTION(hpx::error::invalid_status,
                    "escaped_list_separator::do_escape",
                    "cannot end with escape");
            }

            if (Traits::eq(*next, 'n'))
            {
                tok += '\n';
                return;
            }
            else if (is_quote(*next) || is_c(*next) || is_escape(*next))
            {
                tok += *next;
                return;
            }
            else
            {
                HPX_THROW_EXCEPTION(hpx::error::invalid_status,
                    "escaped_list_separator::do_escape",
                    "unknown escape sequence");
            }
        }

    public:
        explicit escaped_list_separator(
            Char e = '\\', Char c = ',', Char q = '\"')
          : escape_(1, e)
          , c_(1, c)
          , quote_(1, q)
        {
        }

        escaped_list_separator(
            string_type e, string_type c, string_type q) noexcept
          : escape_(HPX_MOVE(e))
          , c_(HPX_MOVE(c))
          , quote_(HPX_MOVE(q))
        {
        }

        void reset() noexcept
        {
            last_ = false;
        }

        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            bool in_quote = false;
            tok = Token();

            if (next == end)
            {
                if (last_)
                {
                    last_ = false;
                    return true;
                }
                else
                {
                    return false;
                }
            }

            last_ = false;
            for (/**/; next != end; ++next)
            {
                if (is_escape(*next))
                {
                    do_escape(next, end, tok);
                }
                else if (is_c(*next))
                {
                    if (!in_quote)
                    {
                        // If we are not in quote, then we are done
                        ++next;

                        // The last character was a c, that means there is 1
                        // more blank field
                        last_ = true;
                        return true;
                    }
                    else
                    {
                        tok += *next;
                    }
                }
                else if (is_quote(*next))
                {
                    in_quote = !in_quote;
                }
                else
                {
                    tok += *next;
                }
            }
            return true;
        }
    };

    //=========================================================================
    // The classes here are used by offset_separator and char_separator to
    // implement faster assigning of tokens using assign instead of +=

    namespace detail {

        //=====================================================================
        // Tokenizer was broken for wide character separators, at least on
        // Windows, since CRT functions isspace etc only expect values in [0,
        // 0xFF]. Debug build asserts if higher values are passed in. The traits
        // extension class should take care of this. Assuming that the
        // conditional will always get optimized out in the function
        // implementations, argument types are not a problem since both forms of
        // character classifiers expect an int.
        template <typename Traits, int N>
        struct traits_extension_details : public Traits
        {
            using char_type = typename Traits::char_type;

            static bool isspace(char_type c) noexcept
            {
                return std::iswspace(c) != 0;
            }

            static bool ispunct(char_type c) noexcept
            {
                return std::iswpunct(c) != 0;
            }
        };

        template <typename Traits>
        struct traits_extension_details<Traits, 1> : public Traits
        {
            using char_type = typename Traits::char_type;

            static bool isspace(char_type c) noexcept
            {
                return std::isspace(c) != 0;
            }

            static bool ispunct(char_type c) noexcept
            {
                return std::ispunct(c) != 0;
            }
        };

        // In case there is no cwctype header, we implement the checks manually.
        // We make use of the fact that the tested categories should fit in
        // ASCII.
        template <typename Traits>
        struct traits_extension : public Traits
        {
            using char_type = typename Traits::char_type;

            static bool isspace(char_type c) noexcept
            {
                return traits_extension_details<Traits,
                    sizeof(char_type)>::isspace(c);
            }

            static bool ispunct(char_type c) noexcept
            {
                return traits_extension_details<Traits,
                    sizeof(char_type)>::ispunct(c);
            }
        };

        // The assign_or_plus_equal struct contains functions that implement
        // assign, +=, and clearing based on the iterator type. The generic case
        // does nothing for plus_equal and clearing, while passing through the
        // call for assign.
        //
        // When an input iterator is being used, the situation is reversed. The
        // assign method does nothing, plus_equal invokes operator +=, and the
        // clearing method sets the supplied token to the default token
        // constructor's result.
        template <typename IteratorTag>
        struct assign_or_plus_equal
        {
            template <typename Iterator, typename Token>
            static constexpr void assign(Iterator b, Iterator e, Token& t)
            {
                t.assign(b, e);
            }

            template <typename Token, typename Value>
            static constexpr void plus_equal(Token&, Value&&) noexcept
            {
            }

            // If we are doing an assign, there is no need for the the clear.
            template <typename Token>
            static constexpr void clear(Token&) noexcept
            {
            }
        };

        template <>
        struct assign_or_plus_equal<std::input_iterator_tag>
        {
            template <class Iterator, class Token>
            static constexpr void assign(Iterator, Iterator, Token&) noexcept
            {
            }

            template <class Token, class Value>
            static constexpr void plus_equal(Token& t, Value&& v)
            {
                t += HPX_FORWARD(Value, v);
            }

            template <class Token>
            static constexpr void clear(Token& t)
            {
                t = Token();
            }
        };

        template <typename Iterator>
        struct class_iterator_category
        {
            using type = typename Iterator::iterator_category;
        };

        // This portably gets the iterator_tag without partial template
        // specialization
        template <typename Iterator>
        struct get_iterator_category
        {
            using iterator_category =
                std::conditional_t<std::is_pointer_v<Iterator>,
                    std::random_access_iterator_tag,
                    typename class_iterator_category<Iterator>::type>;
        };
    }    // namespace detail

    //===========================================================================
    // The offset_separator class, which is a model of TokenizerFunction. Offset
    // breaks a string into tokens based on a range of offsets
    class offset_separator
    {
    private:
        std::vector<int> offsets_;
        unsigned int current_offset_ = 0;
        bool wrap_offsets_ = true;
        bool return_partial_last_ = true;

    public:
        template <typename Iter>
        offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
            bool return_partial_last = true)
          : offsets_(begin, end)
          , wrap_offsets_(wrap_offsets)
          , return_partial_last_(return_partial_last)
        {
        }

        offset_separator(std::initializer_list<int> init,
            bool wrap_offsets = true, bool return_partial_last = true)
          : offsets_(HPX_MOVE(init))
          , wrap_offsets_(wrap_offsets)
          , return_partial_last_(return_partial_last)
        {
        }

        offset_separator()
          : offsets_(1, 1)
        {
        }

        void reset()
        {
            current_offset_ = 0;
        }

        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            using assigner = detail::assign_or_plus_equal<typename detail::
                    get_iterator_category<InputIterator>::iterator_category>;

            HPX_ASSERT(!offsets_.empty());

            assigner::clear(tok);
            InputIterator start(next);

            if (next == end)
            {
                return false;
            }

            if (static_cast<std::size_t>(current_offset_) == offsets_.size())
            {
                if (wrap_offsets_)
                {
                    current_offset_ = 0;
                }
                else
                {
                    return false;
                }
            }

            int c = offsets_[static_cast<std::size_t>(current_offset_)];
            int i = 0;
            for (; i < c; ++i)
            {
                if (next == end)
                {
                    break;
                }
                assigner::plus_equal(tok, *next++);
            }
            assigner::assign(start, next, tok);

            if (!return_partial_last_)
            {
                if (i < (c - 1))
                {
                    return false;
                }
            }

            ++current_offset_;
            return true;
        }
    };

    //=========================================================================
    // The char_separator class breaks a sequence of characters into tokens
    // based on the character delimiters (very much like bad old strtok). A
    // delimiter character can either be kept or dropped. A kept delimiter shows
    // up as an output token, whereas a dropped delimiter does not.

    // This class replaces the char_delimiters_separator class. The constructor
    // for the char_delimiters_separator class was too confusing and needed to
    // be deprecated. However, because of the default arguments to the
    // constructor, adding the new constructor would cause ambiguity, so instead
    // I deprecated the whole class. The implementation of the class was also
    // simplified considerably.
    enum class empty_token_policy
    {
        drop,
        keep
    };

    template <typename Char,
        typename Traits = typename std::basic_string<Char>::traits_type,
        typename Allocator = typename std::basic_string<Char>::allocator_type>
    class char_separator
    {
        using traits_type = detail::traits_extension<Traits>;
        using string_type = std::basic_string<Char, Traits, Allocator>;

    public:
        explicit char_separator(Char const* dropped_delims,
            Char const* kept_delims = nullptr,
            empty_token_policy empty_tokens = empty_token_policy::drop)
          : m_dropped_delims(dropped_delims)
          , m_use_ispunct(false)
          , m_use_isspace(false)
          , m_empty_tokens(empty_tokens)
        {
            if (kept_delims)
                m_kept_delims = kept_delims;
        }

        // use ispunct() for kept delimiters and isspace for dropped.
        char_separator() = default;

        constexpr void reset() noexcept {}

        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            using assigner = detail::assign_or_plus_equal<typename detail::
                    get_iterator_category<InputIterator>::iterator_category>;

            assigner::clear(tok);

            // skip past all dropped_delims
            if (m_empty_tokens == empty_token_policy::drop)
            {
                for (/**/; next != end && is_dropped(*next); ++next)
                {
                }
            }

            InputIterator start(next);

            if (m_empty_tokens == empty_token_policy::drop)
            {
                if (next == end)
                    return false;

                // if we are on a kept_delims move past it and stop
                if (is_kept(*next))
                {
                    assigner::plus_equal(tok, *next);
                    ++next;
                }
                else
                {
                    // append all the non delim characters
                    for (/**/;
                         next != end && !is_dropped(*next) && !is_kept(*next);
                         ++next)
                    {
                        assigner::plus_equal(tok, *next);
                    }
                }
            }
            else
            {
                // m_empty_tokens == empty_token_policy::keep

                // Handle empty token at the end
                if (next == end)
                {
                    if (!m_output_done)
                    {
                        m_output_done = true;
                        assigner::assign(start, next, tok);
                        return true;
                    }
                    else
                    {
                        return false;
                    }
                }

                if (is_kept(*next))
                {
                    if (!m_output_done)
                    {
                        m_output_done = true;
                    }
                    else
                    {
                        assigner::plus_equal(tok, *next);
                        ++next;
                        m_output_done = false;
                    }
                }
                else if (!m_output_done && is_dropped(*next))
                {
                    m_output_done = true;
                }
                else
                {
                    if (is_dropped(*next))
                    {
                        start = ++next;
                    }

                    for (/**/;
                         next != end && !is_dropped(*next) && !is_kept(*next);
                         ++next)
                    {
                        assigner::plus_equal(tok, *next);
                    }

                    m_output_done = true;
                }
            }

            assigner::assign(start, next, tok);
            return true;
        }

    private:
        string_type m_kept_delims;
        string_type m_dropped_delims;
        bool m_use_ispunct = true;
        bool m_use_isspace = true;
        empty_token_policy m_empty_tokens = empty_token_policy::drop;
        bool m_output_done = false;

        bool is_kept(Char E) const
        {
            if (m_kept_delims.length())
            {
                return m_kept_delims.find(E) != string_type::npos;
            }
            else if (m_use_ispunct)
            {
                return traits_type::ispunct(E) != 0;
            }
            return false;
        }

        bool is_dropped(Char E) const
        {
            if (m_dropped_delims.length())
            {
                return m_dropped_delims.find(E) != string_type::npos;
            }
            else if (m_use_isspace)
            {
                return traits_type::isspace(E) != 0;
            }
            return false;
        }
    };

    //===========================================================================
    // The char_delimiters_separator class, which is a model of
    // TokenizerFunction. char_delimiters_separator breaks a string into tokens
    // based on character delimiters. There are 2 types of delimiters.
    // Returnable delimiters can be returned as tokens. These are often
    // punctuation. Nonreturnable delimiters cannot be returned as tokens. These
    // are often whitespace

    template <typename Char,
        typename Traits = typename std::basic_string<Char>::traits_type,
        typename Allocator = typename std::basic_string<Char>::allocator_type>
    class char_delimiters_separator
    {
    private:
        using traits_type = detail::traits_extension<Traits>;
        using string_type = std::basic_string<Char, Traits, Allocator>;

        string_type returnable_;
        string_type nonreturnable_;
        bool return_delims_;
        bool no_ispunct_;
        bool no_isspace_;

        bool is_ret(Char E) const noexcept
        {
            if (returnable_.length())
            {
                return returnable_.find(E) != string_type::npos;
            }
            else
            {
                if (no_ispunct_)
                {
                    return false;
                }
                else
                {
                    int r = traits_type::ispunct(E);
                    return r != 0;
                }
            }
        }

        bool is_nonret(Char E) const noexcept
        {
            if (nonreturnable_.length())
            {
                return nonreturnable_.find(E) != string_type::npos;
            }
            else
            {
                if (no_isspace_)
                {
                    return false;
                }
                else
                {
                    int r = traits_type::isspace(E);
                    return r != 0;
                }
            }
        }

    public:
        explicit char_delimiters_separator(bool return_delims = false,
            Char const* returnable = nullptr,
            Char const* nonreturnable = nullptr)
          : returnable_(returnable ? returnable : string_type().c_str())
          , nonreturnable_(
                nonreturnable ? nonreturnable : string_type().c_str())
          , return_delims_(return_delims)
          , no_ispunct_(returnable != nullptr)
          , no_isspace_(nonreturnable != nullptr)
        {
        }

        constexpr void reset() noexcept {}

    public:
        template <typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator end, Token& tok)
        {
            tok = Token();

            // skip past all nonreturnable delims
            // skip past the returnable only if we are not returning delims
            for (/**/; next != end &&
                 (is_nonret(*next) || (is_ret(*next) && !return_delims_));
                 ++next)
            {
            }

            if (next == end)
            {
                return false;
            }

            // if we are to return delims and we are one a returnable one move
            // past it and stop
            if (is_ret(*next) && return_delims_)
            {
                tok += *next;
                ++next;
            }
            else
            {
                // append all the non delim characters
                for (/**/; next != end && !is_nonret(*next) && !is_ret(*next);
                     ++next)
                {
                    tok += *next;
                }
            }

            return true;
        }
    };
}    // namespace hpx::string_util

1	// Copyright (c) 2022 Hartmut Kaiser
2	//
3	// SPDX-License-Identifier: BSL-1.0
4	// Distributed under the Boost Software License, Version 1.0. (See accompanying
5	// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
7	// Copyright John R. Bandela 2001.
8
9	// See http://www.boost.org/libs/tokenizer/ for documentation.
10
11	// Revision History:
12	// 01 Oct 2004 Joaquin M Lopez Munoz
13	// Workaround for a problem with string::assign in msvc-stlport
14	// 06 Apr 2004 John Bandela
15	// Fixed a bug involving using char_delimiter with a true input iterator
16	// 28 Nov 2003 Robert Zeh and John Bandela
17	// Converted into "fast" functions that avoid using += when
18	// the supplied iterator isn't an input_iterator; based on
19	// some work done at Archelon and a version that was checked into
20	// the boost CVS for a short period of time.
21	// 20 Feb 2002 John Maddock
22	// Removed using namespace std declarations and added
23	// workaround for BOOST_NO_STDC_NAMESPACE (the library
24	// can be safely mixed with regex).
25	// 06 Feb 2002 Jeremy Siek
26	// Added char_separator.
27	// 02 Feb 2002 Jeremy Siek
28	// Removed tabs and a little cleanup.
29
30	#pragma once
31
32	#include <hpx/config.hpp>
33	#include <hpx/assert.hpp>
34	#include <hpx/modules/errors.hpp>
35
36	#include <algorithm> // for find_if
37	#include <cctype>
38	#include <cstddef>
39	#include <cwctype>
40	#include <initializer_list>
41	#include <iterator>
42	#include <stdexcept>
43	#include <string>
44	#include <vector>
45
46	namespace hpx::string_util {
47
48	//=========================================================================
49	// The escaped_list_separator class. Which is a model of TokenizerFunction
50	// An escaped list is a super-set of what is commonly known as a comma
51	// separated value (csv) list.It is separated into fields by a comma or
52	// other character. If the delimiting character is inside quotes, then it is
53	// counted as a regular character.To allow for embedded quotes in a field,
54	// there can be escape sequences using the \ much like C. The role of the
55	// comma, the quotation mark, and the escape character (backslash \), can be
56	// assigned to other characters.
57	template <typename Char,
58	typename Traits = typename std::basic_string<Char>::traits_type,
59	typename Allocator = typename std::basic_string<Char>::allocator_type>
60	class escaped_list_separator	13,937✔
61	{
62	private:
63	using string_type = std::basic_string<Char, Traits, Allocator>;
64
65	struct char_eq
66	{
67	Char e_;
68
69	explicit char_eq(Char e) noexcept	737,974✔
70	: e_(e)	737,974✔
71	{
72	}	737,974✔
73
74	bool operator()(Char c) noexcept	1,220,696✔
75	{
76	return Traits::eq(e_, c);	1,220,696✔
77	}
78	};
79
80	string_type escape_;
81	string_type c_;
82	string_type quote_;
83	bool last_ = false;	1,987✔
84
85	bool is_escape(Char e)	248,197✔
86	{
87	char_eq f(e);	248,197✔
88	return std::find_if(escape_.begin(), escape_.end(), f) !=	496,394✔
89	escape_.end();	248,197✔
90	}
91
92	bool is_c(Char e)	248,183✔
93	{
94	char_eq f(e);	248,183✔
95	return std::find_if(c_.begin(), c_.end(), f) != c_.end();	248,183✔
96	}
97
98	bool is_quote(Char e)	241,594✔
99	{
100	char_eq f(e);	241,594✔
101	return std::find_if(quote_.begin(), quote_.end(), f) !=	483,188✔
102	quote_.end();	241,594✔
103	}
104
105	template <typename Iterator, typename Token>
106	void do_escape(Iterator& next, Iterator end, Token& tok)	17✔
107	{
108	if (++next == end)	17✔
109	{
110	HPX_THROW_EXCEPTION(hpx::error::invalid_status,	×
111	"escaped_list_separator::do_escape",
112	"cannot end with escape");
113	}
114
115	if (Traits::eq(*next, 'n'))	17✔
116	{
117	tok += '\n';	×
118	return;	×
119	}
120	else if (is_quote(next) \|\| is_c(next) \|\| is_escape(*next))	17✔
121	{
122	tok += *next;	17✔
123	return;	17✔
124	}
125	else
126	{
127	HPX_THROW_EXCEPTION(hpx::error::invalid_status,	×
128	"escaped_list_separator::do_escape",
129	"unknown escape sequence");
130	}
131	}	17✔
132
133	public:
134	explicit escaped_list_separator(	4✔
135	Char e = '\\', Char c = ',', Char q = '\"')
136	: escape_(1, e)	4✔
137	, c_(1, c)	4✔
138	, quote_(1, q)	4✔
139	{
140	}	4✔
141
142	escaped_list_separator(	1,983✔
143	string_type e, string_type c, string_type q) noexcept
144	: escape_(HPX_MOVE(e))	1,983✔
145	, c_(HPX_MOVE(c))	1,983✔
146	, quote_(HPX_MOVE(q))	1,983✔
147	{
148	}	1,983✔
149
150	void reset() noexcept	3,971✔
151	{
152	last_ = false;	3,971✔
153	}	3,971✔
154
155	template <typename InputIterator, typename Token>
156	bool operator()(InputIterator& next, InputIterator end, Token& tok)	10,563✔
157	{
158	bool in_quote = false;	10,563✔
159	tok = Token();	10,563✔
160
161	if (next == end)	10,563✔
162	{
163	if (last_)	2,017✔
164	{
165	last_ = false;	30✔
166	return true;	30✔
167	}
168	else
169	{
170	return false;	1,987✔
171	}
172	}
173
174	last_ = false;	8,546✔
175	for (/**/; next != end; ++next)	250,150✔
176	{
177	if (is_escape(*next))	248,193✔
178	{
179	do_escape(next, end, tok);	17✔
180	}	17✔
181	else if (is_c(*next))	248,176✔
182	{
183	if (!in_quote)	6,599✔
184	{
185	// If we are not in quote, then we are done
186	++next;	6,589✔
187
188	// The last character was a c, that means there is 1
189	// more blank field
190	last_ = true;	6,589✔
191	return true;	6,589✔
192	}
193	else
194	{
195	tok += *next;	10✔
196	}
197	}	10✔
198	else if (is_quote(*next))	241,577✔
199	{
200	in_quote = !in_quote;	20✔
201	}	20✔
202	else
203	{
204	tok += *next;	241,557✔
205	}
206	}	241,604✔
207	return true;	1,957✔
208	}	10,563✔
209	};
210
211	//=========================================================================
212	// The classes here are used by offset_separator and char_separator to
213	// implement faster assigning of tokens using assign instead of +=
214
215	namespace detail {
216
217	//=====================================================================
218	// Tokenizer was broken for wide character separators, at least on
219	// Windows, since CRT functions isspace etc only expect values in [0,
220	// 0xFF]. Debug build asserts if higher values are passed in. The traits
221	// extension class should take care of this. Assuming that the
222	// conditional will always get optimized out in the function
223	// implementations, argument types are not a problem since both forms of
224	// character classifiers expect an int.
225	template <typename Traits, int N>
226	struct traits_extension_details : public Traits
227	{
228	using char_type = typename Traits::char_type;
229
230	static bool isspace(char_type c) noexcept
231	{
232	return std::iswspace(c) != 0;
233	}
234
235	static bool ispunct(char_type c) noexcept
236	{
237	return std::iswpunct(c) != 0;
238	}
239	};
240
241	template <typename Traits>
242	struct traits_extension_details<Traits, 1> : public Traits
243	{
244	using char_type = typename Traits::char_type;
245
246	static bool isspace(char_type c) noexcept	98✔
247	{
248	return std::isspace(c) != 0;	98✔
249	}
250
251	static bool ispunct(char_type c) noexcept	92✔
252	{
253	return std::ispunct(c) != 0;	92✔
254	}
255	};
256
257	// In case there is no cwctype header, we implement the checks manually.
258	// We make use of the fact that the tested categories should fit in
259	// ASCII.
260	template <typename Traits>
261	struct traits_extension : public Traits
262	{
263	using char_type = typename Traits::char_type;
264
265	static bool isspace(char_type c) noexcept	98✔
266	{
267	return traits_extension_details<Traits,	98✔
268	sizeof(char_type)>::isspace(c);	98✔
269	}
270
271	static bool ispunct(char_type c) noexcept	92✔
272	{
273	return traits_extension_details<Traits,	92✔
274	sizeof(char_type)>::ispunct(c);	92✔
275	}
276	};
277
278	// The assign_or_plus_equal struct contains functions that implement
279	// assign, +=, and clearing based on the iterator type. The generic case
280	// does nothing for plus_equal and clearing, while passing through the
281	// call for assign.
282	//
283	// When an input iterator is being used, the situation is reversed. The
284	// assign method does nothing, plus_equal invokes operator +=, and the
285	// clearing method sets the supplied token to the default token
286	// constructor's result.
287	template <typename IteratorTag>
288	struct assign_or_plus_equal
289	{
290	template <typename Iterator, typename Token>
291	static constexpr void assign(Iterator b, Iterator e, Token& t)	67,031✔
292	{
293	t.assign(b, e);	67,031✔
294	}	67,031✔
295
296	template <typename Token, typename Value>
297	static constexpr void plus_equal(Token&, Value&&) noexcept	792,920✔
298	{
299	}	792,920✔
300
301	// If we are doing an assign, there is no need for the the clear.
302	template <typename Token>
303	static constexpr void clear(Token&) noexcept	101,162✔
304	{
305	}	101,162✔
306	};
307
308	template <>
309	struct assign_or_plus_equal<std::input_iterator_tag>
310	{
311	template <class Iterator, class Token>
312	static constexpr void assign(Iterator, Iterator, Token&) noexcept
313	{
314	}
315
316	template <class Token, class Value>
317	static constexpr void plus_equal(Token& t, Value&& v)
318	{
319	t += HPX_FORWARD(Value, v);
320	}
321
322	template <class Token>
323	static constexpr void clear(Token& t)
324	{
325	t = Token();
326	}
327	};
328
329	template <typename Iterator>
330	struct class_iterator_category
331	{
332	using type = typename Iterator::iterator_category;
333	};
334
335	// This portably gets the iterator_tag without partial template
336	// specialization
337	template <typename Iterator>
338	struct get_iterator_category
339	{
340	using iterator_category =
341	std::conditional_t<std::is_pointer_v<Iterator>,
342	std::random_access_iterator_tag,
343	typename class_iterator_category<Iterator>::type>;
344	};
345	} // namespace detail
346
347	//===========================================================================
348	// The offset_separator class, which is a model of TokenizerFunction. Offset
349	// breaks a string into tokens based on a range of offsets
350	class offset_separator	63✔
351	{
352	private:
353	std::vector<int> offsets_;
354	unsigned int current_offset_ = 0;	8✔
355	bool wrap_offsets_ = true;	3✔
356	bool return_partial_last_ = true;	3✔
357
358	public:
359	template <typename Iter>
360	offset_separator(Iter begin, Iter end, bool wrap_offsets = true,	4✔
361	bool return_partial_last = true)
362	: offsets_(begin, end)	4✔
363	, wrap_offsets_(wrap_offsets)	4✔
364	, return_partial_last_(return_partial_last)	4✔
365	{
366	}	4✔
367
368	offset_separator(std::initializer_list<int> init,	1✔
369	bool wrap_offsets = true, bool return_partial_last = true)
370	: offsets_(HPX_MOVE(init))	1✔
371	, wrap_offsets_(wrap_offsets)	1✔
372	, return_partial_last_(return_partial_last)	1✔
373	{
374	}	1✔
375
376	offset_separator()	3✔
377	: offsets_(1, 1)	3✔
378	{
379	}	3✔
380
381	void reset()	10✔
382	{
383	current_offset_ = 0;	10✔
384	}	10✔
385
386	template <typename InputIterator, typename Token>
387	bool operator()(InputIterator& next, InputIterator end, Token& tok)	20✔
388	{
389	using assigner = detail::assign_or_plus_equal<typename detail::
390	get_iterator_category<InputIterator>::iterator_category>;
391
392	HPX_ASSERT(!offsets_.empty());	20✔
393
394	assigner::clear(tok);	20✔
395	InputIterator start(next);	20✔
396
397	if (next == end)	20✔
398	{
399	return false;	4✔
400	}
401
402	if (static_cast<std::size_t>(current_offset_) == offsets_.size())	16✔
403	{
404	if (wrap_offsets_)	×
405	{
406	current_offset_ = 0;	×
407	}	×
408	else
409	{
410	return false;	×
411	}
412	}	×
413
414	int c = offsets_[static_cast<std::size_t>(current_offset_)];	16✔
415	int i = 0;	16✔
416	for (; i < c; ++i)	55✔
417	{
418	if (next == end)	39✔
419	{
420	break;	×
421	}
422	assigner::plus_equal(tok, *next++);	39✔
423	}	39✔
424	assigner::assign(start, next, tok);	16✔
425
426	if (!return_partial_last_)	16✔
427	{
428	if (i < (c - 1))	×
429	{
430	return false;	×
431	}
432	}	×
433
434	++current_offset_;	16✔
435	return true;	16✔
436	}	20✔
437	};
438
439	//=========================================================================
440	// The char_separator class breaks a sequence of characters into tokens
441	// based on the character delimiters (very much like bad old strtok). A
442	// delimiter character can either be kept or dropped. A kept delimiter shows
443	// up as an output token, whereas a dropped delimiter does not.
444
445	// This class replaces the char_delimiters_separator class. The constructor
446	// for the char_delimiters_separator class was too confusing and needed to
447	// be deprecated. However, because of the default arguments to the
448	// constructor, adding the new constructor would cause ambiguity, so instead
449	// I deprecated the whole class. The implementation of the class was also
450	// simplified considerably.
451	enum class empty_token_policy
452	{
453	drop,
454	keep
455	};
456
457	template <typename Char,
458	typename Traits = typename std::basic_string<Char>::traits_type,
459	typename Allocator = typename std::basic_string<Char>::allocator_type>
460	class char_separator	425,813✔
461	{
462	using traits_type = detail::traits_extension<Traits>;
463	using string_type = std::basic_string<Char, Traits, Allocator>;
464
465	public:
466	explicit char_separator(Char const* dropped_delims,	44,533✔
467	Char const* kept_delims = nullptr,
468	empty_token_policy empty_tokens = empty_token_policy::drop)
469	: m_dropped_delims(dropped_delims)	44,533✔
470	, m_use_ispunct(false)	44,533✔
471	, m_use_isspace(false)	44,533✔
472	, m_empty_tokens(empty_tokens)	44,533✔
473	{
474	if (kept_delims)	44,533✔
475	m_kept_delims = kept_delims;	97✔
476	}	44,533✔
477
478	// use ispunct() for kept delimiters and isspace for dropped.
479	char_separator() = default;	2✔
480
481	constexpr void reset() noexcept {}	138,845✔
482
483	template <typename InputIterator, typename Token>
484	bool operator()(InputIterator& next, InputIterator end, Token& tok)	101,142✔
485	{
486	using assigner = detail::assign_or_plus_equal<typename detail::
487	get_iterator_category<InputIterator>::iterator_category>;
488
489	assigner::clear(tok);	101,142✔
490
491	// skip past all dropped_delims
492	if (m_empty_tokens == empty_token_policy::drop)	101,142✔
493	{
494	for (/*/; next != end && is_dropped(next); ++next)	126,494✔
495	{
496	}	25,570✔
497	}	100,924✔
498
499	InputIterator start(next);	101,142✔
500
501	if (m_empty_tokens == empty_token_policy::drop)	101,142✔
502	{
503	if (next == end)	100,924✔
504	return false;	34,030✔
505
506	// if we are on a kept_delims move past it and stop
507	if (is_kept(*next))	66,894✔
508	{
509	assigner::plus_equal(tok, *next);	2✔
510	++next;	2✔
511	}	2✔
512	else
513	{
514	// append all the non delim characters
515	for (/**/;	1,697,542✔
516	next != end && !is_dropped(next) && !is_kept(next);	848,771✔
517	++next)	781,879✔
518	{
519	assigner::plus_equal(tok, *next);	781,879✔
520	}	781,879✔
521	}
522	}	66,894✔
523	else
524	{
525	// m_empty_tokens == empty_token_policy::keep
526
527	// Handle empty token at the end
528	if (next == end)	218✔
529	{
530	if (!m_output_done)	98✔
531	{
532	m_output_done = true;	1✔
533	assigner::assign(start, next, tok);	1✔
534	return true;	1✔
535	}
536	else
537	{
538	return false;	97✔
539	}
540	}
541
542	if (is_kept(*next))	120✔
543	{
544	if (!m_output_done)	5✔
545	{
546	m_output_done = true;	1✔
547	}	1✔
548	else
549	{
550	assigner::plus_equal(tok, *next);	4✔
551	++next;	4✔
552	m_output_done = false;	4✔
553	}
554	}	5✔
555	else if (!m_output_done && is_dropped(*next))	115✔
556	{
557	m_output_done = true;	2✔
558	}	2✔
559	else
560	{
561	if (is_dropped(*next))	113✔
562	{
563	start = ++next;	16✔
564	}	16✔
565
566	for (/**/;	22,218✔
567	next != end && !is_dropped(next) && !is_kept(next);	11,109✔
568	++next)	10,996✔
569	{
570	assigner::plus_equal(tok, *next);	10,996✔
571	}	10,996✔
572
573	m_output_done = true;	113✔
574	}
575	}
576
577	assigner::assign(start, next, tok);	67,014✔
578	return true;	67,014✔
579	}	101,142✔
580
581	private:
582	string_type m_kept_delims;
583	string_type m_dropped_delims;
584	bool m_use_ispunct = true;	2✔
585	bool m_use_isspace = true;	2✔
586	empty_token_policy m_empty_tokens = empty_token_policy::drop;	2✔
587	bool m_output_done = false;	44,535✔
588
589	bool is_kept(Char E) const	859,894✔
590	{
591	if (m_kept_delims.length())	859,894✔
592	{
593	return m_kept_delims.find(E) != string_type::npos;	40✔
594	}
595	else if (m_use_ispunct)	859,854✔
596	{
597	return traits_type::ispunct(E) != 0;	34✔
598	}
599	return false;	859,820✔
600	}	859,894✔
601
602	bool is_dropped(Char E) const	908,883✔
603	{
604	if (m_dropped_delims.length())	908,883✔
605	{
606	return m_dropped_delims.find(E) != string_type::npos;	908,837✔
607	}
608	else if (m_use_isspace)	46✔
609	{
610	return traits_type::isspace(E) != 0;	46✔
611	}
612	return false;	×
613	}	908,883✔
614	};
615
616	//===========================================================================
617	// The char_delimiters_separator class, which is a model of
618	// TokenizerFunction. char_delimiters_separator breaks a string into tokens
619	// based on character delimiters. There are 2 types of delimiters.
620	// Returnable delimiters can be returned as tokens. These are often
621	// punctuation. Nonreturnable delimiters cannot be returned as tokens. These
622	// are often whitespace
623
624	template <typename Char,
625	typename Traits = typename std::basic_string<Char>::traits_type,
626	typename Allocator = typename std::basic_string<Char>::allocator_type>
627	class char_delimiters_separator	56✔
628	{
629	private:
630	using traits_type = detail::traits_extension<Traits>;
631	using string_type = std::basic_string<Char, Traits, Allocator>;
632
633	string_type returnable_;
634	string_type nonreturnable_;
635	bool return_delims_;
636	bool no_ispunct_;
637	bool no_isspace_;
638
639	bool is_ret(Char E) const noexcept	86✔
640	{
641	if (returnable_.length())	86✔
642	{
643	return returnable_.find(E) != string_type::npos;	28✔
644	}
645	else
646	{
647	if (no_ispunct_)	58✔
648	{
649	return false;	×
650	}
651	else
652	{
653	int r = traits_type::ispunct(E);	58✔
654	return r != 0;	58✔
655	}
656	}
657	}	86✔
658
659	bool is_nonret(Char E) const noexcept	75✔
660	{
661	if (nonreturnable_.length())	75✔
662	{
663	return nonreturnable_.find(E) != string_type::npos;	×
664	}
665	else
666	{
667	if (no_isspace_)	75✔
668	{
669	return false;	23✔
670	}
671	else
672	{
673	int r = traits_type::isspace(E);	52✔
674	return r != 0;	52✔
675	}
676	}
677	}	75✔
678
679	public:
680	explicit char_delimiters_separator(bool return_delims = false,	4✔
681	Char const* returnable = nullptr,
682	Char const* nonreturnable = nullptr)
683	: returnable_(returnable ? returnable : string_type().c_str())	4✔
684	, nonreturnable_(	8✔
685	nonreturnable ? nonreturnable : string_type().c_str())	4✔
686	, return_delims_(return_delims)	4✔
687	, no_ispunct_(returnable != nullptr)	4✔
688	, no_isspace_(nonreturnable != nullptr)	4✔
689	{
690	}	4✔
691
692	constexpr void reset() noexcept {}	5✔
693
694	public:
695	template <typename InputIterator, typename Token>
696	bool operator()(InputIterator& next, InputIterator end, Token& tok)	16✔
697	{
698	tok = Token();	16✔
699
700	// skip past all nonreturnable delims
701	// skip past the returnable only if we are not returning delims
702	for (/**/; next != end &&	57✔
703	(is_nonret(next) \|\| (is_ret(next) && !return_delims_));	27✔
704	++next)	14✔
705	{
706	}	14✔
707
708	if (next == end)	16✔
709	{
710	return false;	3✔
711	}
712
713	// if we are to return delims and we are one a returnable one move
714	// past it and stop
715	if (is_ret(*next) && return_delims_)	13✔
716	{
717	tok += *next;	2✔
718	++next;	2✔
719	}	2✔
720	else
721	{
722	// append all the non delim characters
723	for (/*/; next != end && !is_nonret(next) && !is_ret(*next);	49✔
724	++next)	38✔
725	{
726	tok += *next;	38✔
727	}	38✔
728	}
729
730	return true;	13✔
731	}	16✔
732	};
733	} // namespace hpx::string_util

STEllAR-GROUP / hpx / #862

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous