Initial commit

This commit is contained in:
Bassem Girgis
2018-12-20 17:34:07 -06:00
parent 7a2d899662
commit 81b4b9e273
34743 changed files with 5940233 additions and 0 deletions

View File

@@ -0,0 +1,114 @@
/*=============================================================================
Copyright (c) 2001-2011 Joel de Guzman
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
==============================================================================*/
#ifndef BOOST_PP_IS_ITERATING
#if !defined(BOOST_SPIRIT_AS_VARIANT_NOVEMBER_16_2007_0420PM)
#define BOOST_SPIRIT_AS_VARIANT_NOVEMBER_16_2007_0420PM
#include <boost/preprocessor/iterate.hpp>
#include <boost/preprocessor/repetition/enum_params.hpp>
#include <boost/preprocessor/repetition/enum_binary_params.hpp>
#include <boost/preprocessor/repetition/enum_params_with_a_default.hpp>
#include <boost/preprocessor/repetition/repeat_from_to.hpp>
#include <boost/variant/variant_fwd.hpp>
#include <boost/fusion/include/size.hpp>
#include <boost/fusion/include/begin.hpp>
#include <boost/fusion/include/next.hpp>
#include <boost/fusion/include/value_of.hpp>
#include <boost/mpl/fold.hpp>
#include <boost/mpl/vector.hpp>
#include <boost/mpl/push_back.hpp>
#include <boost/mpl/equal_to.hpp>
#include <boost/mpl/contains.hpp>
#include <boost/mpl/limits/list.hpp>
#include <boost/type_traits/is_same.hpp>
namespace boost { namespace spirit { namespace detail
{
template <int size>
struct as_variant_impl;
#if !defined(BOOST_VARIANT_DO_NOT_USE_VARIADIC_TEMPLATES)
#else
template <>
struct as_variant_impl<0>
{
template <typename Iterator>
struct apply
{
typedef variant<> type;
};
};
#endif
#define BOOST_FUSION_NEXT_ITERATOR(z, n, data) \
typedef typename fusion::result_of::next<BOOST_PP_CAT(I, n)>::type \
BOOST_PP_CAT(I, BOOST_PP_INC(n));
#define BOOST_FUSION_NEXT_CALL_ITERATOR(z, n, data) \
typename gen::BOOST_PP_CAT(I, BOOST_PP_INC(n)) \
BOOST_PP_CAT(i, BOOST_PP_INC(n)) = fusion::next(BOOST_PP_CAT(i, n));
#define BOOST_FUSION_VALUE_OF_ITERATOR(z, n, data) \
typedef typename fusion::result_of::value_of<BOOST_PP_CAT(I, n)>::type \
BOOST_PP_CAT(T, n);
#define BOOST_PP_FILENAME_1 <boost/spirit/home/support/detail/as_variant.hpp>
#if !defined(BOOST_VARIANT_DO_NOT_USE_VARIADIC_TEMPLATES)
#define BOOST_PP_ITERATION_LIMITS (1, BOOST_MPL_LIMIT_LIST_SIZE)
#else
#define BOOST_PP_ITERATION_LIMITS (1, BOOST_VARIANT_LIMIT_TYPES)
#endif
#include BOOST_PP_ITERATE()
#undef BOOST_FUSION_NEXT_ITERATOR
#undef BOOST_FUSION_NEXT_CALL_ITERATOR
#undef BOOST_FUSION_VALUE_OF_ITERATOR
template <typename Sequence>
struct as_variant
{
// build a variant generator being able to generate a variant holding
// all of the types as given in the typelist
typedef typename
detail::as_variant_impl<fusion::result_of::size<Sequence>::value>
gen;
// use this generator to create the actual variant
typedef typename gen::template apply<
typename fusion::result_of::begin<Sequence>::type
>::type
type;
};
}}}
#endif
#else // defined(BOOST_PP_IS_ITERATING)
///////////////////////////////////////////////////////////////////////////////
//
// Preprocessor vertical repetition code
//
///////////////////////////////////////////////////////////////////////////////
#define N BOOST_PP_ITERATION()
template <>
struct as_variant_impl<N>
{
template <typename I0>
struct apply
{
BOOST_PP_REPEAT(N, BOOST_FUSION_NEXT_ITERATOR, _)
BOOST_PP_REPEAT(N, BOOST_FUSION_VALUE_OF_ITERATOR, _)
typedef variant<BOOST_PP_ENUM_PARAMS(N, T)> type;
};
};
#undef N
#endif // defined(BOOST_PP_IS_ITERATING)

View File

@@ -0,0 +1,29 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
// http://spirit.sourceforge.net/
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(SPIRIT_ENDIAN_MAR_21_2009_0349PM)
#define SPIRIT_ENDIAN_MAR_21_2009_0349PM
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/version.hpp>
// We need to treat the endian number types as PODs
#if !defined(BOOST_ENDIAN_FORCE_PODNESS)
#define BOOST_ENDIAN_FORCE_PODNESS 1
#endif
// If Boost has the endian library, use it, otherwise use an adapted version
// included with Spirit
// #if BOOST_VERSION >= 105100
// #include <boost/endian/integers.hpp>
// #else
#include <boost/spirit/home/support/detail/endian/endian.hpp>
// #endif
#endif

View File

@@ -0,0 +1,115 @@
// boost/integer/cover_operators.hpp ----------------------------------------//
// Copyright Darin Adler 2000
// Copyright Beman Dawes 2008
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//----------------------------------------------------------------------------//
// If the class being covered has a non-explicit conversion to an integer type
// then a smaller number of cover operations are needed. Define the macro
// BOOST_MINIMAL_INTEGER_COVER_OPERATORS to indicate this.
// Define BOOST_NO_IO_COVER_OPERATORS if I/O cover operations are not desired.
//----------------------------------------------------------------------------//
#ifndef BOOST_SPIRIT_INTEGER_COVER_OPERATORS_HPP
#define BOOST_SPIRIT_INTEGER_COVER_OPERATORS_HPP
#if defined(_MSC_VER)
#pragma once
#endif
# ifndef BOOST_MINIMAL_INTEGER_COVER_OPERATORS
# include <boost/operators.hpp>
# endif
#include <iosfwd>
namespace boost { namespace spirit
{
namespace endian
{
// A class that adds integer operators to an integer cover class
template <typename T, typename IntegerType>
class cover_operators
# ifndef BOOST_MINIMAL_INTEGER_COVER_OPERATORS
: boost::operators<T>
# endif
{
// The other operations take advantage of the type conversion that's
// built into unary +.
// Unary operations.
friend IntegerType operator+(const T& x) { return x; }
# ifndef BOOST_MINIMAL_INTEGER_COVER_OPERATORS
friend IntegerType operator-(const T& x) { return -+x; }
friend IntegerType operator~(const T& x) { return ~+x; }
friend IntegerType operator!(const T& x) { return !+x; }
// The basic ordering operations.
friend bool operator==(const T& x, IntegerType y) { return +x == y; }
friend bool operator<(const T& x, IntegerType y) { return +x < y; }
# endif
// The basic arithmetic operations.
friend T& operator+=(T& x, IntegerType y) { return x = +x + y; }
friend T& operator-=(T& x, IntegerType y) { return x = +x - y; }
friend T& operator*=(T& x, IntegerType y) { return x = +x * y; }
friend T& operator/=(T& x, IntegerType y) { return x = +x / y; }
friend T& operator%=(T& x, IntegerType y) { return x = +x % y; }
friend T& operator&=(T& x, IntegerType y) { return x = +x & y; }
friend T& operator|=(T& x, IntegerType y) { return x = +x | y; }
friend T& operator^=(T& x, IntegerType y) { return x = +x ^ y; }
friend T& operator<<=(T& x, IntegerType y) { return x = +x << y; }
friend T& operator>>=(T& x, IntegerType y) { return x = +x >> y; }
// A few binary arithmetic operations not covered by operators base class.
friend IntegerType operator<<(const T& x, IntegerType y) { return +x << y; }
friend IntegerType operator>>(const T& x, IntegerType y) { return +x >> y; }
// Auto-increment and auto-decrement can be defined in terms of the
// arithmetic operations.
friend T& operator++(T& x) { return x += 1; }
friend T& operator--(T& x) { return x -= 1; }
# ifdef BOOST_MINIMAL_INTEGER_COVER_OPERATORS
friend T operator++(T& x, int)
{
T tmp(x);
x += 1;
return tmp;
}
friend T operator--(T& x, int)
{
T tmp(x);
x -= 1;
return tmp;
}
# endif
# ifndef BOOST_NO_IO_COVER_OPERATORS
// TODO: stream I/O needs to be templatized on the stream type, so will
// work with wide streams, etc.
// Stream input and output.
friend std::ostream& operator<<(std::ostream& s, const T& x)
{ return s << +x; }
friend std::istream& operator>>(std::istream& s, T& x)
{
IntegerType i;
if (s >> i)
x = i;
return s;
}
# endif
};
} // namespace endian
}} // namespace boost::spirit
#endif // BOOST_SPIRIT_INTEGER_COVER_OPERATORS_HPP

View File

@@ -0,0 +1,569 @@
// Boost endian.hpp header file -------------------------------------------------------//
// (C) Copyright Darin Adler 2000
// (C) Copyright Beman Dawes 2006, 2009
// Distributed under the Boost Software License, Version 1.0.
// See http://www.boost.org/LICENSE_1_0.txt
// See library home page at http://www.boost.org/libs/endian
//--------------------------------------------------------------------------------------//
// Original design developed by Darin Adler based on classes developed by Mark
// Borgerding. Four original class templates were combined into a single endian
// class template by Beman Dawes, who also added the unrolled_byte_loops sign
// partial specialization to correctly extend the sign when cover integer size
// differs from endian representation size.
// TODO: When a compiler supporting constexpr becomes available, try possible uses.
#ifndef BOOST_SPIRIT_ENDIAN_HPP
#define BOOST_SPIRIT_ENDIAN_HPP
#if defined(_MSC_VER)
#pragma once
#endif
#ifdef BOOST_ENDIAN_LOG
# include <iostream>
#endif
#if defined(__BORLANDC__) || defined( __CODEGEARC__)
# pragma pack(push, 1)
#endif
#include <boost/config.hpp>
#include <boost/detail/endian.hpp>
#define BOOST_MINIMAL_INTEGER_COVER_OPERATORS
#define BOOST_NO_IO_COVER_OPERATORS
#include <boost/spirit/home/support/detail/endian/cover_operators.hpp>
#undef BOOST_NO_IO_COVER_OPERATORS
#undef BOOST_MINIMAL_INTEGER_COVER_OPERATORS
#include <boost/type_traits/is_signed.hpp>
#include <boost/type_traits/make_unsigned.hpp>
#include <boost/cstdint.hpp>
#include <boost/static_assert.hpp>
#include <boost/spirit/home/support/detail/scoped_enum_emulation.hpp>
#include <iosfwd>
#include <climits>
# if CHAR_BIT != 8
# error Platforms with CHAR_BIT != 8 are not supported
# endif
# define BOOST_ENDIAN_DEFAULT_CONSTRUCT {} // C++03
# if defined(BOOST_ENDIAN_FORCE_PODNESS)
# define BOOST_ENDIAN_NO_CTORS
# endif
namespace boost { namespace spirit
{
namespace detail
{
// Unrolled loops for loading and storing streams of bytes.
template <typename T, std::size_t n_bytes,
bool sign=boost::is_signed<T>::value >
struct unrolled_byte_loops
{
typedef unrolled_byte_loops<T, n_bytes - 1, sign> next;
static typename boost::make_unsigned<T>::type load_big(const unsigned char* bytes)
{ return *(bytes - 1) | (next::load_big(bytes - 1) << 8); }
static typename boost::make_unsigned<T>::type load_little(const unsigned char* bytes)
{ return *bytes | (next::load_little(bytes + 1) << 8); }
static void store_big(char* bytes, T value)
{
*(bytes - 1) = static_cast<char>(value);
next::store_big(bytes - 1, value >> 8);
}
static void store_little(char* bytes, T value)
{
*bytes = static_cast<char>(value);
next::store_little(bytes + 1, value >> 8);
}
};
template <typename T>
struct unrolled_byte_loops<T, 1, false>
{
static T load_big(const unsigned char* bytes)
{ return *(bytes - 1); }
static T load_little(const unsigned char* bytes)
{ return *bytes; }
static void store_big(char* bytes, T value)
{ *(bytes - 1) = static_cast<char>(value); }
static void store_little(char* bytes, T value)
{ *bytes = static_cast<char>(value); }
};
template <typename T>
struct unrolled_byte_loops<T, 1, true>
{
static typename boost::make_unsigned<T>::type load_big(const unsigned char* bytes)
{ return *(bytes - 1); }
static typename boost::make_unsigned<T>::type load_little(const unsigned char* bytes)
{ return *bytes; }
static void store_big(char* bytes, T value)
{ *(bytes - 1) = static_cast<char>(value); }
static void store_little(char* bytes, T value)
{ *bytes = static_cast<char>(value); }
};
template <typename T, std::size_t n_bytes>
inline
T load_big_endian(const void* bytes)
{
return static_cast<T>(unrolled_byte_loops<T, n_bytes>::load_big
(static_cast<const unsigned char*>(bytes) + n_bytes));
}
template <>
inline
float load_big_endian<float, 4>(const void* bytes)
{
const unsigned char *b = reinterpret_cast<const unsigned char *>(
bytes);
b += 3;
float value;
unsigned char *v = reinterpret_cast<unsigned char *>(&value);
for(std::size_t i = 0; i < 4; ++i)
{
*v++ = *b--;
}
return value;
}
template <>
inline
double load_big_endian<double, 8>(const void* bytes)
{
const unsigned char *b = reinterpret_cast<const unsigned char *>(
bytes);
b += 7;
double value;
unsigned char *v = reinterpret_cast<unsigned char *>(&value);
for(std::size_t i = 0; i < 8; ++i)
{
*v++ = *b--;
}
return value;
}
template <typename T, std::size_t n_bytes>
inline
T load_little_endian(const void* bytes)
{
return static_cast<T>(unrolled_byte_loops<T, n_bytes>::load_little
(static_cast<const unsigned char*>(bytes)));
}
template <>
inline
float load_little_endian<float, 4>(const void* bytes)
{
const unsigned char *b = reinterpret_cast<const unsigned char *>(
bytes);
float value;
unsigned char *v = reinterpret_cast<unsigned char *>(&value);
for(std::size_t i = 0; i < 4; ++i)
{
*v++ = *b++;
}
return value;
}
template <>
inline
double load_little_endian<double, 8>(const void* bytes)
{
const unsigned char *b = reinterpret_cast<const unsigned char *>(
bytes);
double value;
unsigned char *v = reinterpret_cast<unsigned char *>(&value);
for(std::size_t i = 0; i < 8; ++i)
{
*v++ = *b++;
}
return value;
}
template <typename T, std::size_t n_bytes>
inline
void store_big_endian(void* bytes, T value)
{
unrolled_byte_loops<T, n_bytes>::store_big
(static_cast<char*>(bytes) + n_bytes, value);
}
template <>
inline
void store_big_endian<float, 4>(void* bytes, float value)
{
unsigned char *b = reinterpret_cast<unsigned char *>(bytes);
b += 3;
const unsigned char *v = reinterpret_cast<const unsigned char *>(
&value);
for(std::size_t i = 0; i < 4; ++i)
{
*b-- = *v++;
}
}
template <>
inline
void store_big_endian<double, 8>(void* bytes, double value)
{
unsigned char *b = reinterpret_cast<unsigned char *>(bytes);
b += 7;
const unsigned char *v = reinterpret_cast<const unsigned char *>(
&value);
for(std::size_t i = 0; i < 8; ++i)
{
*b-- = *v++;
}
}
template <typename T, std::size_t n_bytes>
inline
void store_little_endian(void* bytes, T value)
{
unrolled_byte_loops<T, n_bytes>::store_little
(static_cast<char*>(bytes), value);
}
template <>
inline
void store_little_endian<float, 4>(void* bytes, float value)
{
unsigned char *b = reinterpret_cast<unsigned char *>(bytes);
const unsigned char *v = reinterpret_cast<const unsigned char *>(
&value);
for(std::size_t i = 0; i < 4; ++i)
{
*b++ = *v++;
}
}
template <>
inline
void store_little_endian<double, 8>(void* bytes, double value)
{
unsigned char *b = reinterpret_cast<unsigned char *>(bytes);
const unsigned char *v = reinterpret_cast<const unsigned char *>(
&value);
for(std::size_t i = 0; i < 8; ++i)
{
*b++ = *v++;
}
}
} // namespace detail
namespace endian
{
# ifdef BOOST_ENDIAN_LOG
bool endian_log(true);
# endif
// endian class template and specializations ---------------------------------------//
BOOST_SCOPED_ENUM_START(endianness) { big, little, native }; BOOST_SCOPED_ENUM_END
BOOST_SCOPED_ENUM_START(alignment) { unaligned, aligned }; BOOST_SCOPED_ENUM_END
template <BOOST_SCOPED_ENUM(endianness) E, typename T, std::size_t n_bits,
BOOST_SCOPED_ENUM(alignment) A = alignment::unaligned>
class endian;
// Specializations that represent unaligned bytes.
// Taking an integer type as a parameter provides a nice way to pass both
// the size and signedness of the desired integer and get the appropriate
// corresponding integer type for the interface.
// unaligned big endian specialization
template <typename T, std::size_t n_bits>
class endian< endianness::big, T, n_bits, alignment::unaligned >
: cover_operators< endian< endianness::big, T, n_bits >, T >
{
BOOST_STATIC_ASSERT( (n_bits/8)*8 == n_bits );
public:
typedef T value_type;
# ifndef BOOST_ENDIAN_NO_CTORS
endian() BOOST_ENDIAN_DEFAULT_CONSTRUCT
explicit endian(T val)
{
# ifdef BOOST_ENDIAN_LOG
if ( endian_log )
std::clog << "big, unaligned, " << n_bits << "-bits, construct(" << val << ")\n";
# endif
detail::store_big_endian<T, n_bits/8>(m_value, val);
}
# endif
endian & operator=(T val) { detail::store_big_endian<T, n_bits/8>(m_value, val); return *this; }
operator T() const
{
# ifdef BOOST_ENDIAN_LOG
if ( endian_log )
std::clog << "big, unaligned, " << n_bits << "-bits, convert(" << detail::load_big_endian<T, n_bits/8>(m_value) << ")\n";
# endif
return detail::load_big_endian<T, n_bits/8>(m_value);
}
private:
char m_value[n_bits/8];
};
// unaligned little endian specialization
template <typename T, std::size_t n_bits>
class endian< endianness::little, T, n_bits, alignment::unaligned >
: cover_operators< endian< endianness::little, T, n_bits >, T >
{
BOOST_STATIC_ASSERT( (n_bits/8)*8 == n_bits );
public:
typedef T value_type;
# ifndef BOOST_ENDIAN_NO_CTORS
endian() BOOST_ENDIAN_DEFAULT_CONSTRUCT
explicit endian(T val)
{
# ifdef BOOST_ENDIAN_LOG
if ( endian_log )
std::clog << "little, unaligned, " << n_bits << "-bits, construct(" << val << ")\n";
# endif
detail::store_little_endian<T, n_bits/8>(m_value, val);
}
# endif
endian & operator=(T val) { detail::store_little_endian<T, n_bits/8>(m_value, val); return *this; }
operator T() const
{
# ifdef BOOST_ENDIAN_LOG
if ( endian_log )
std::clog << "little, unaligned, " << n_bits << "-bits, convert(" << detail::load_little_endian<T, n_bits/8>(m_value) << ")\n";
# endif
return detail::load_little_endian<T, n_bits/8>(m_value);
}
private:
char m_value[n_bits/8];
};
// unaligned native endian specialization
template <typename T, std::size_t n_bits>
class endian< endianness::native, T, n_bits, alignment::unaligned >
: cover_operators< endian< endianness::native, T, n_bits >, T >
{
BOOST_STATIC_ASSERT( (n_bits/8)*8 == n_bits );
public:
typedef T value_type;
# ifndef BOOST_ENDIAN_NO_CTORS
endian() BOOST_ENDIAN_DEFAULT_CONSTRUCT
# ifdef BOOST_BIG_ENDIAN
explicit endian(T val) { detail::store_big_endian<T, n_bits/8>(m_value, val); }
# else
explicit endian(T val) { detail::store_little_endian<T, n_bits/8>(m_value, val); }
# endif
# endif
# ifdef BOOST_BIG_ENDIAN
endian & operator=(T val) { detail::store_big_endian<T, n_bits/8>(m_value, val); return *this; }
operator T() const { return detail::load_big_endian<T, n_bits/8>(m_value); }
# else
endian & operator=(T val) { detail::store_little_endian<T, n_bits/8>(m_value, val); return *this; }
operator T() const { return detail::load_little_endian<T, n_bits/8>(m_value); }
# endif
private:
char m_value[n_bits/8];
};
// Specializations that mimic built-in integer types.
// These typically have the same alignment as the underlying types.
// aligned big endian specialization
template <typename T, std::size_t n_bits>
class endian< endianness::big, T, n_bits, alignment::aligned >
: cover_operators< endian< endianness::big, T, n_bits, alignment::aligned >, T >
{
BOOST_STATIC_ASSERT( (n_bits/8)*8 == n_bits );
BOOST_STATIC_ASSERT( sizeof(T) == n_bits/8 );
public:
typedef T value_type;
# ifndef BOOST_ENDIAN_NO_CTORS
endian() BOOST_ENDIAN_DEFAULT_CONSTRUCT
# ifdef BOOST_BIG_ENDIAN
endian(T val) : m_value(val) { }
# else
explicit endian(T val) { detail::store_big_endian<T, sizeof(T)>(&m_value, val); }
# endif
# endif
# ifdef BOOST_BIG_ENDIAN
endian & operator=(T val) { m_value = val; return *this; }
operator T() const { return m_value; }
# else
endian & operator=(T val) { detail::store_big_endian<T, sizeof(T)>(&m_value, val); return *this; }
operator T() const { return detail::load_big_endian<T, sizeof(T)>(&m_value); }
# endif
private:
T m_value;
};
// aligned little endian specialization
template <typename T, std::size_t n_bits>
class endian< endianness::little, T, n_bits, alignment::aligned >
: cover_operators< endian< endianness::little, T, n_bits, alignment::aligned >, T >
{
BOOST_STATIC_ASSERT( (n_bits/8)*8 == n_bits );
BOOST_STATIC_ASSERT( sizeof(T) == n_bits/8 );
public:
typedef T value_type;
# ifndef BOOST_ENDIAN_NO_CTORS
endian() BOOST_ENDIAN_DEFAULT_CONSTRUCT
# ifdef BOOST_LITTLE_ENDIAN
endian(T val) : m_value(val) { }
# else
explicit endian(T val) { detail::store_little_endian<T, sizeof(T)>(&m_value, val); }
# endif
# endif
# ifdef BOOST_LITTLE_ENDIAN
endian & operator=(T val) { m_value = val; return *this; }
operator T() const { return m_value; }
#else
endian & operator=(T val) { detail::store_little_endian<T, sizeof(T)>(&m_value, val); return *this; }
operator T() const { return detail::load_little_endian<T, sizeof(T)>(&m_value); }
#endif
private:
T m_value;
};
// naming convention typedefs ------------------------------------------------------//
// unaligned big endian signed integer types
typedef endian< endianness::big, int_least8_t, 8 > big8_t;
typedef endian< endianness::big, int_least16_t, 16 > big16_t;
typedef endian< endianness::big, int_least32_t, 24 > big24_t;
typedef endian< endianness::big, int_least32_t, 32 > big32_t;
typedef endian< endianness::big, int_least64_t, 40 > big40_t;
typedef endian< endianness::big, int_least64_t, 48 > big48_t;
typedef endian< endianness::big, int_least64_t, 56 > big56_t;
typedef endian< endianness::big, int_least64_t, 64 > big64_t;
// unaligned big endian unsigned integer types
typedef endian< endianness::big, uint_least8_t, 8 > ubig8_t;
typedef endian< endianness::big, uint_least16_t, 16 > ubig16_t;
typedef endian< endianness::big, uint_least32_t, 24 > ubig24_t;
typedef endian< endianness::big, uint_least32_t, 32 > ubig32_t;
typedef endian< endianness::big, uint_least64_t, 40 > ubig40_t;
typedef endian< endianness::big, uint_least64_t, 48 > ubig48_t;
typedef endian< endianness::big, uint_least64_t, 56 > ubig56_t;
typedef endian< endianness::big, uint_least64_t, 64 > ubig64_t;
// unaligned little endian signed integer types
typedef endian< endianness::little, int_least8_t, 8 > little8_t;
typedef endian< endianness::little, int_least16_t, 16 > little16_t;
typedef endian< endianness::little, int_least32_t, 24 > little24_t;
typedef endian< endianness::little, int_least32_t, 32 > little32_t;
typedef endian< endianness::little, int_least64_t, 40 > little40_t;
typedef endian< endianness::little, int_least64_t, 48 > little48_t;
typedef endian< endianness::little, int_least64_t, 56 > little56_t;
typedef endian< endianness::little, int_least64_t, 64 > little64_t;
// unaligned little endian unsigned integer types
typedef endian< endianness::little, uint_least8_t, 8 > ulittle8_t;
typedef endian< endianness::little, uint_least16_t, 16 > ulittle16_t;
typedef endian< endianness::little, uint_least32_t, 24 > ulittle24_t;
typedef endian< endianness::little, uint_least32_t, 32 > ulittle32_t;
typedef endian< endianness::little, uint_least64_t, 40 > ulittle40_t;
typedef endian< endianness::little, uint_least64_t, 48 > ulittle48_t;
typedef endian< endianness::little, uint_least64_t, 56 > ulittle56_t;
typedef endian< endianness::little, uint_least64_t, 64 > ulittle64_t;
// unaligned native endian signed integer types
typedef endian< endianness::native, int_least8_t, 8 > native8_t;
typedef endian< endianness::native, int_least16_t, 16 > native16_t;
typedef endian< endianness::native, int_least32_t, 24 > native24_t;
typedef endian< endianness::native, int_least32_t, 32 > native32_t;
typedef endian< endianness::native, int_least64_t, 40 > native40_t;
typedef endian< endianness::native, int_least64_t, 48 > native48_t;
typedef endian< endianness::native, int_least64_t, 56 > native56_t;
typedef endian< endianness::native, int_least64_t, 64 > native64_t;
// unaligned native endian unsigned integer types
typedef endian< endianness::native, uint_least8_t, 8 > unative8_t;
typedef endian< endianness::native, uint_least16_t, 16 > unative16_t;
typedef endian< endianness::native, uint_least32_t, 24 > unative24_t;
typedef endian< endianness::native, uint_least32_t, 32 > unative32_t;
typedef endian< endianness::native, uint_least64_t, 40 > unative40_t;
typedef endian< endianness::native, uint_least64_t, 48 > unative48_t;
typedef endian< endianness::native, uint_least64_t, 56 > unative56_t;
typedef endian< endianness::native, uint_least64_t, 64 > unative64_t;
#define BOOST_HAS_INT16_T
#define BOOST_HAS_INT32_T
#define BOOST_HAS_INT64_T
// These types only present if platform has exact size integers:
// aligned big endian signed integer types
// aligned big endian unsigned integer types
// aligned little endian signed integer types
// aligned little endian unsigned integer types
// aligned native endian typedefs are not provided because
// <cstdint> types are superior for this use case
# if defined(BOOST_HAS_INT16_T)
typedef endian< endianness::big, int16_t, 16, alignment::aligned > aligned_big16_t;
typedef endian< endianness::big, uint16_t, 16, alignment::aligned > aligned_ubig16_t;
typedef endian< endianness::little, int16_t, 16, alignment::aligned > aligned_little16_t;
typedef endian< endianness::little, uint16_t, 16, alignment::aligned > aligned_ulittle16_t;
# endif
# if defined(BOOST_HAS_INT32_T)
typedef endian< endianness::big, int32_t, 32, alignment::aligned > aligned_big32_t;
typedef endian< endianness::big, uint32_t, 32, alignment::aligned > aligned_ubig32_t;
typedef endian< endianness::little, int32_t, 32, alignment::aligned > aligned_little32_t;
typedef endian< endianness::little, uint32_t, 32, alignment::aligned > aligned_ulittle32_t;
# endif
# if defined(BOOST_HAS_INT64_T)
typedef endian< endianness::big, int64_t, 64, alignment::aligned > aligned_big64_t;
typedef endian< endianness::big, uint64_t, 64, alignment::aligned > aligned_ubig64_t;
typedef endian< endianness::little, int64_t, 64, alignment::aligned > aligned_little64_t;
typedef endian< endianness::little, uint64_t, 64, alignment::aligned > aligned_ulittle64_t;
# endif
} // namespace endian
}} // namespace boost::spirit
// import the namespace above into boost::endian
namespace boost { namespace endian
{
using namespace boost::spirit::endian;
}}
#if defined(__BORLANDC__) || defined( __CODEGEARC__)
# pragma pack(pop)
#endif
#endif // BOOST_SPIRIT_ENDIAN_HPP

View File

@@ -0,0 +1,75 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
// Copyright (c) 2001-2011 Joel de Guzman
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_GET_ENCODING_JANUARY_13_2009_1255PM)
#define BOOST_SPIRIT_GET_ENCODING_JANUARY_13_2009_1255PM
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/mpl/identity.hpp>
#include <boost/type_traits/is_same.hpp>
namespace boost { namespace spirit { namespace detail
{
template <typename Modifiers, typename Encoding>
struct get_implicit_encoding
{
// Extract the implicit encoding from the Modifiers
// If one is not found, Encoding is used. The explicit
// encoding is the first viable encoding that can be
// extracted from the Modifiers (there can be more than one).
typedef typename
mpl::find_if<
char_encodings,
has_modifier<Modifiers, tag::char_encoding_base<mpl::_1> >
>::type
iter;
typedef typename
mpl::eval_if<
is_same<iter, typename mpl::end<char_encodings>::type>,
mpl::identity<Encoding>,
mpl::deref<iter>
>::type
type;
};
template <typename Modifiers, typename Encoding>
struct get_encoding
{
// Extract the explicit encoding from the Modifiers
// If one is not found, get implicit encoding (see above).
// Explicit encoding is the encoding explicitly declared
// using the encoding[c] directive.
typedef typename
mpl::find_if<
char_encodings,
has_modifier<Modifiers, tag::char_code<tag::encoding, mpl::_1> >
>::type
iter;
typedef typename
mpl::eval_if<
is_same<iter, typename mpl::end<char_encodings>::type>,
get_implicit_encoding<Modifiers, Encoding>,
mpl::deref<iter>
>::type
type;
};
template <typename Modifiers, typename Encoding, bool case_modifier = false>
struct get_encoding_with_case : mpl::identity<Encoding> {};
template <typename Modifiers, typename Encoding>
struct get_encoding_with_case<Modifiers, Encoding, true>
: get_encoding<Modifiers, Encoding> {};
}}}
#endif

View File

@@ -0,0 +1,457 @@
/*=============================================================================
Copyright (c) 2007-2011 Hartmut Kaiser
Copyright (c) Christopher Diggins 2005
Copyright (c) Pablo Aguilar 2005
Copyright (c) Kevlin Henney 2001
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
The class boost::spirit::hold_any is built based on the any class
published here: http://www.codeproject.com/cpp/dynamic_typing.asp. It adds
support for std streaming operator<<() and operator>>().
==============================================================================*/
#if !defined(BOOST_SPIRIT_HOLD_ANY_MAY_02_2007_0857AM)
#define BOOST_SPIRIT_HOLD_ANY_MAY_02_2007_0857AM
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/config.hpp>
#include <boost/type_traits/remove_reference.hpp>
#include <boost/type_traits/is_reference.hpp>
#include <boost/throw_exception.hpp>
#include <boost/static_assert.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/assert.hpp>
#include <boost/detail/sp_typeinfo.hpp>
#include <stdexcept>
#include <typeinfo>
#include <algorithm>
#include <iosfwd>
///////////////////////////////////////////////////////////////////////////////
#if BOOST_WORKAROUND(BOOST_MSVC, >= 1400)
# pragma warning(push)
# pragma warning(disable: 4100) // 'x': unreferenced formal parameter
# pragma warning(disable: 4127) // conditional expression is constant
#endif
///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit
{
struct bad_any_cast
: std::bad_cast
{
bad_any_cast(boost::detail::sp_typeinfo const& src, boost::detail::sp_typeinfo const& dest)
: from(src.name()), to(dest.name())
{}
virtual const char* what() const throw() { return "bad any cast"; }
const char* from;
const char* to;
};
namespace detail
{
// function pointer table
template <typename Char>
struct fxn_ptr_table
{
boost::detail::sp_typeinfo const& (*get_type)();
void (*static_delete)(void**);
void (*destruct)(void**);
void (*clone)(void* const*, void**);
void (*move)(void* const*, void**);
std::basic_istream<Char>& (*stream_in)(std::basic_istream<Char>&, void**);
std::basic_ostream<Char>& (*stream_out)(std::basic_ostream<Char>&, void* const*);
};
// static functions for small value-types
template <typename Small>
struct fxns;
template <>
struct fxns<mpl::true_>
{
template<typename T, typename Char>
struct type
{
static boost::detail::sp_typeinfo const& get_type()
{
return BOOST_SP_TYPEID(T);
}
static void static_delete(void** x)
{
reinterpret_cast<T*>(x)->~T();
}
static void destruct(void** x)
{
reinterpret_cast<T*>(x)->~T();
}
static void clone(void* const* src, void** dest)
{
new (dest) T(*reinterpret_cast<T const*>(src));
}
static void move(void* const* src, void** dest)
{
*reinterpret_cast<T*>(dest) =
*reinterpret_cast<T const*>(src);
}
static std::basic_istream<Char>&
stream_in (std::basic_istream<Char>& i, void** obj)
{
i >> *reinterpret_cast<T*>(obj);
return i;
}
static std::basic_ostream<Char>&
stream_out(std::basic_ostream<Char>& o, void* const* obj)
{
o << *reinterpret_cast<T const*>(obj);
return o;
}
};
};
// static functions for big value-types (bigger than a void*)
template <>
struct fxns<mpl::false_>
{
template<typename T, typename Char>
struct type
{
static boost::detail::sp_typeinfo const& get_type()
{
return BOOST_SP_TYPEID(T);
}
static void static_delete(void** x)
{
// destruct and free memory
delete (*reinterpret_cast<T**>(x));
}
static void destruct(void** x)
{
// destruct only, we'll reuse memory
(*reinterpret_cast<T**>(x))->~T();
}
static void clone(void* const* src, void** dest)
{
*dest = new T(**reinterpret_cast<T* const*>(src));
}
static void move(void* const* src, void** dest)
{
**reinterpret_cast<T**>(dest) =
**reinterpret_cast<T* const*>(src);
}
static std::basic_istream<Char>&
stream_in(std::basic_istream<Char>& i, void** obj)
{
i >> **reinterpret_cast<T**>(obj);
return i;
}
static std::basic_ostream<Char>&
stream_out(std::basic_ostream<Char>& o, void* const* obj)
{
o << **reinterpret_cast<T* const*>(obj);
return o;
}
};
};
template <typename T>
struct get_table
{
typedef mpl::bool_<(sizeof(T) <= sizeof(void*))> is_small;
template <typename Char>
static fxn_ptr_table<Char>* get()
{
static fxn_ptr_table<Char> static_table =
{
fxns<is_small>::template type<T, Char>::get_type,
fxns<is_small>::template type<T, Char>::static_delete,
fxns<is_small>::template type<T, Char>::destruct,
fxns<is_small>::template type<T, Char>::clone,
fxns<is_small>::template type<T, Char>::move,
fxns<is_small>::template type<T, Char>::stream_in,
fxns<is_small>::template type<T, Char>::stream_out
};
return &static_table;
}
};
///////////////////////////////////////////////////////////////////////
struct empty {};
template <typename Char>
inline std::basic_istream<Char>&
operator>> (std::basic_istream<Char>& i, empty&)
{
// If this assertion fires you tried to insert from a std istream
// into an empty hold_any instance. This simply can't work, because
// there is no way to figure out what type to extract from the
// stream.
// The only way to make this work is to assign an arbitrary
// value of the required type to the hold_any instance you want to
// stream to. This assignment has to be executed before the actual
// call to the operator>>().
BOOST_ASSERT(false &&
"Tried to insert from a std istream into an empty "
"hold_any instance");
return i;
}
template <typename Char>
inline std::basic_ostream<Char>&
operator<< (std::basic_ostream<Char>& o, empty const&)
{
return o;
}
}
///////////////////////////////////////////////////////////////////////////
template <typename Char>
class basic_hold_any
{
public:
// constructors
template <typename T>
explicit basic_hold_any(T const& x)
: table(spirit::detail::get_table<T>::template get<Char>()), object(0)
{
if (spirit::detail::get_table<T>::is_small::value)
new (&object) T(x);
else
object = new T(x);
}
basic_hold_any()
: table(spirit::detail::get_table<spirit::detail::empty>::template get<Char>()),
object(0)
{
}
basic_hold_any(basic_hold_any const& x)
: table(spirit::detail::get_table<spirit::detail::empty>::template get<Char>()),
object(0)
{
assign(x);
}
~basic_hold_any()
{
table->static_delete(&object);
}
// assignment
basic_hold_any& assign(basic_hold_any const& x)
{
if (&x != this) {
// are we copying between the same type?
if (table == x.table) {
// if so, we can avoid reallocation
table->move(&x.object, &object);
}
else {
reset();
x.table->clone(&x.object, &object);
table = x.table;
}
}
return *this;
}
template <typename T>
basic_hold_any& assign(T const& x)
{
// are we copying between the same type?
spirit::detail::fxn_ptr_table<Char>* x_table =
spirit::detail::get_table<T>::template get<Char>();
if (table == x_table) {
// if so, we can avoid deallocating and re-use memory
table->destruct(&object); // first destruct the old content
if (spirit::detail::get_table<T>::is_small::value) {
// create copy on-top of object pointer itself
new (&object) T(x);
}
else {
// create copy on-top of old version
new (object) T(x);
}
}
else {
if (spirit::detail::get_table<T>::is_small::value) {
// create copy on-top of object pointer itself
table->destruct(&object); // first destruct the old content
new (&object) T(x);
}
else {
reset(); // first delete the old content
object = new T(x);
}
table = x_table; // update table pointer
}
return *this;
}
// assignment operator
#ifdef BOOST_HAS_RVALUE_REFS
template <typename T>
basic_hold_any& operator=(T&& x)
{
return assign(std::forward<T>(x));
}
#else
template <typename T>
basic_hold_any& operator=(T& x)
{
return assign(x);
}
template <typename T>
basic_hold_any& operator=(T const& x)
{
return assign(x);
}
#endif
// utility functions
basic_hold_any& swap(basic_hold_any& x)
{
std::swap(table, x.table);
std::swap(object, x.object);
return *this;
}
boost::detail::sp_typeinfo const& type() const
{
return table->get_type();
}
template <typename T>
T const& cast() const
{
if (type() != BOOST_SP_TYPEID(T))
throw bad_any_cast(type(), BOOST_SP_TYPEID(T));
return spirit::detail::get_table<T>::is_small::value ?
*reinterpret_cast<T const*>(&object) :
*reinterpret_cast<T const*>(object);
}
// implicit casting is disabled by default for compatibility with boost::any
#ifdef BOOST_SPIRIT_ANY_IMPLICIT_CASTING
// automatic casting operator
template <typename T>
operator T const& () const { return cast<T>(); }
#endif // implicit casting
bool empty() const
{
return table == spirit::detail::get_table<spirit::detail::empty>::template get<Char>();
}
void reset()
{
if (!empty())
{
table->static_delete(&object);
table = spirit::detail::get_table<spirit::detail::empty>::template get<Char>();
object = 0;
}
}
// these functions have been added in the assumption that the embedded
// type has a corresponding operator defined, which is completely safe
// because spirit::hold_any is used only in contexts where these operators
// do exist
template <typename Char_>
friend inline std::basic_istream<Char_>&
operator>> (std::basic_istream<Char_>& i, basic_hold_any<Char_>& obj)
{
return obj.table->stream_in(i, &obj.object);
}
template <typename Char_>
friend inline std::basic_ostream<Char_>&
operator<< (std::basic_ostream<Char_>& o, basic_hold_any<Char_> const& obj)
{
return obj.table->stream_out(o, &obj.object);
}
#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
private: // types
template <typename T, typename Char_>
friend T* any_cast(basic_hold_any<Char_> *);
#else
public: // types (public so any_cast can be non-friend)
#endif
// fields
spirit::detail::fxn_ptr_table<Char>* table;
void* object;
};
// boost::any-like casting
template <typename T, typename Char>
inline T* any_cast (basic_hold_any<Char>* operand)
{
if (operand && operand->type() == BOOST_SP_TYPEID(T)) {
return spirit::detail::get_table<T>::is_small::value ?
reinterpret_cast<T*>(&operand->object) :
reinterpret_cast<T*>(operand->object);
}
return 0;
}
template <typename T, typename Char>
inline T const* any_cast(basic_hold_any<Char> const* operand)
{
return any_cast<T>(const_cast<basic_hold_any<Char>*>(operand));
}
template <typename T, typename Char>
T any_cast(basic_hold_any<Char>& operand)
{
typedef BOOST_DEDUCED_TYPENAME remove_reference<T>::type nonref;
nonref* result = any_cast<nonref>(&operand);
if(!result)
boost::throw_exception(bad_any_cast(operand.type(), BOOST_SP_TYPEID(T)));
return *result;
}
template <typename T, typename Char>
T const& any_cast(basic_hold_any<Char> const& operand)
{
typedef BOOST_DEDUCED_TYPENAME remove_reference<T>::type nonref;
return any_cast<nonref const&>(const_cast<basic_hold_any<Char> &>(operand));
}
///////////////////////////////////////////////////////////////////////////////
// backwards compatibility
typedef basic_hold_any<char> hold_any;
typedef basic_hold_any<wchar_t> whold_any;
namespace traits
{
template <typename T>
struct is_hold_any : mpl::false_ {};
template <typename Char>
struct is_hold_any<basic_hold_any<Char> > : mpl::true_ {};
}
}} // namespace boost::spirit
///////////////////////////////////////////////////////////////////////////////
#if BOOST_WORKAROUND(BOOST_MSVC, >= 1400)
# pragma warning(pop)
#endif
#endif

View File

@@ -0,0 +1,16 @@
/*=============================================================================
Copyright (c) 2001-2011 Hartmut Kaiser
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
==============================================================================*/
#if !defined(BOOST_SPIRIT_SUPPORT_DETAIL_IS_SPIRIT_TAG_MAR_28_2011_0341PM)
#define BOOST_SPIRIT_SUPPORT_DETAIL_IS_SPIRIT_TAG_MAR_28_2011_0341PM
#if defined(_MSC_VER)
#pragma once
#endif
#define BOOST_SPIRIT_IS_TAG() typedef void is_spirit_tag;
#endif

View File

@@ -0,0 +1,54 @@
// char_traits.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_CHAR_TRAITS_H
#define BOOST_LEXER_CHAR_TRAITS_H
// Make sure wchar_t is defined
#include <string>
namespace boost
{
namespace lexer
{
template<typename CharT>
struct char_traits
{
typedef CharT char_type;
typedef CharT index_type;
static index_type call (CharT ch)
{
return ch;
}
};
template<>
struct char_traits<char>
{
typedef char char_type;
typedef unsigned char index_type;
static index_type call (char ch)
{
return static_cast<index_type>(ch);
}
};
template<>
struct char_traits<wchar_t>
{
typedef wchar_t char_type;
typedef wchar_t index_type;
static index_type call (wchar_t ch)
{
return ch;
}
};
}
}
#endif

View File

@@ -0,0 +1,39 @@
// consts.h
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_CONSTS_H
#define BOOST_LEXER_CONSTS_H
#include <boost/config.hpp>
#include <boost/integer_traits.hpp>
#include "size_t.hpp"
namespace boost
{
namespace lexer
{
enum regex_flags {none = 0, icase = 1, dot_not_newline = 2};
// 0 = end state, 1 = id, 2 = unique_id, 3 = lex state, 4 = bol, 5 = eol,
// 6 = dead_state_index
enum {end_state_index, id_index, unique_id_index, state_index, bol_index,
eol_index, dead_state_index, dfa_offset};
const std::size_t max_macro_len = 30;
const std::size_t num_chars = 256;
// If sizeof(wchar_t) == sizeof(size_t) then don't overflow to 0
// by adding one to comparison.
const std::size_t num_wchar_ts =
(boost::integer_traits<wchar_t>::const_max < 0x110000) ?
boost::integer_traits<wchar_t>::const_max +
static_cast<std::size_t> (1) : 0x110000;
const std::size_t null_token = static_cast<std::size_t> (~0);
const std::size_t bol_token = static_cast<std::size_t> (~1);
const std::size_t eol_token = static_cast<std::size_t> (~2);
const std::size_t end_state = 1;
const std::size_t npos = static_cast<std::size_t> (~0);
}
}
#endif

View File

@@ -0,0 +1,71 @@
// ptr_list.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_PTR_LIST_HPP
#define BOOST_LEXER_PTR_LIST_HPP
#include <list>
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename Type>
class ptr_list
{
public:
typedef std::list<Type *> list;
ptr_list ()
{
}
~ptr_list ()
{
clear ();
}
list *operator -> ()
{
return &_list;
}
const list *operator -> () const
{
return &_list;
}
list &operator * ()
{
return _list;
}
const list &operator * () const
{
return _list;
}
void clear ()
{
while (!_list.empty ())
{
delete _list.front ();
_list.pop_front ();
}
}
private:
list _list;
ptr_list (const ptr_list &); // No copy construction.
ptr_list &operator = (const ptr_list &); // No assignment.
};
}
}
}
#endif

View File

@@ -0,0 +1,108 @@
// ptr_vector.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_PTR_VECTOR_HPP
#define BOOST_LEXER_PTR_VECTOR_HPP
#include "../size_t.hpp"
#include <vector>
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename Type>
class ptr_vector
{
public:
typedef std::vector<Type *> vector;
ptr_vector ()
{
}
~ptr_vector ()
{
clear ();
}
vector *operator -> ()
{
return &_vector;
}
const vector *operator -> () const
{
return &_vector;
}
vector &operator * ()
{
return _vector;
}
const vector &operator * () const
{
return _vector;
}
Type * &operator [] (const std::size_t index_)
{
return _vector[index_];
}
Type * const &operator [] (const std::size_t index_) const
{
return _vector[index_];
}
bool operator == (const ptr_vector &rhs_) const
{
bool equal_ = _vector.size () == rhs_._vector.size ();
if (equal_)
{
typename vector::const_iterator lhs_iter_ = _vector.begin ();
typename vector::const_iterator end_ = _vector.end ();
typename vector::const_iterator rhs_iter_ = rhs_._vector.begin ();
for (; equal_ && lhs_iter_ != end_; ++lhs_iter_, ++rhs_iter_)
{
equal_ = **lhs_iter_ == **rhs_iter_;
}
}
return equal_;
}
void clear ()
{
if (!_vector.empty ())
{
Type **iter_ = &_vector.front ();
Type **end_ = iter_ + _vector.size ();
for (; iter_ != end_; ++iter_)
{
delete *iter_;
}
}
_vector.clear ();
}
private:
vector _vector;
ptr_vector (const ptr_vector &); // No copy construction.
ptr_vector &operator = (const ptr_vector &); // No assignment.
};
}
}
}
#endif

View File

@@ -0,0 +1,77 @@
// char_state_machine.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_CHAR_STATE_MACHINE_HPP
#define BOOST_LEXER_CHAR_STATE_MACHINE_HPP
#include "../consts.hpp"
#include <map>
#include "../size_t.hpp"
#include "../string_token.hpp"
#include <vector>
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT>
struct basic_char_state_machine
{
struct state
{
typedef basic_string_token<CharT> string_token;
typedef std::map<std::size_t, string_token> size_t_string_token_map;
typedef std::pair<std::size_t, string_token> size_t_string_token_pair;
bool _end_state;
std::size_t _id;
std::size_t _unique_id;
std::size_t _state;
std::size_t _bol_index;
std::size_t _eol_index;
size_t_string_token_map _transitions;
state () :
_end_state (false),
_id (0),
_unique_id (npos),
_state (0),
_bol_index (npos),
_eol_index (npos)
{
}
};
typedef std::vector<state> state_vector;
typedef std::vector<state_vector> state_vector_vector;
state_vector_vector _sm_vector;
bool empty () const
{
return _sm_vector.empty ();
}
void clear ()
{
_sm_vector.clear ();
}
void swap (basic_char_state_machine &csm_)
{
_sm_vector.swap (csm_._sm_vector);
}
};
typedef basic_char_state_machine<char> char_state_machine;
typedef basic_char_state_machine<wchar_t> wchar_state_machine;
}
}
}
#endif

View File

@@ -0,0 +1,281 @@
// debug.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_DEBUG_HPP
#define BOOST_LEXER_DEBUG_HPP
#include <map>
#include <ostream>
#include "rules.hpp"
#include "size_t.hpp"
#include "state_machine.hpp"
#include "string_token.hpp"
#include <vector>
namespace boost
{
namespace lexer
{
template<typename CharT>
class basic_debug
{
public:
typedef std::basic_ostream<CharT> ostream;
typedef std::basic_string<CharT> string;
typedef std::vector<std::size_t> size_t_vector;
static void escape_control_chars (const string &in_, string &out_)
{
const CharT *ptr_ = in_.c_str ();
std::size_t size_ = in_.size ();
out_.clear ();
while (size_)
{
basic_string_token<CharT>::escape_char (*ptr_, out_);
++ptr_;
--size_;
}
}
static void dump (const basic_state_machine<CharT> &state_machine_,
basic_rules<CharT> &rules_, ostream &stream_)
{
typename basic_state_machine<CharT>::iterator iter_ =
state_machine_.begin ();
typename basic_state_machine<CharT>::iterator end_ =
state_machine_.end ();
for (std::size_t dfa_ = 0, dfas_ = state_machine_.size ();
dfa_ < dfas_; ++dfa_)
{
lexer_state (stream_);
stream_ << rules_.state (dfa_) << std::endl << std::endl;
dump_ex (iter_, stream_);
}
}
static void dump (const basic_state_machine<CharT> &state_machine_,
ostream &stream_)
{
typename basic_state_machine<CharT>::iterator iter_ =
state_machine_.begin ();
typename basic_state_machine<CharT>::iterator end_ =
state_machine_.end ();
for (std::size_t dfa_ = 0, dfas_ = state_machine_.size ();
dfa_ < dfas_; ++dfa_)
{
lexer_state (stream_);
stream_ << dfa_ << std::endl << std::endl;
dump_ex (iter_, stream_);
}
}
protected:
typedef std::basic_stringstream<CharT> stringstream;
static void dump_ex (typename basic_state_machine<CharT>::iterator &iter_,
ostream &stream_)
{
const std::size_t states_ = iter_->states;
for (std::size_t i_ = 0; i_ < states_; ++i_)
{
state (stream_);
stream_ << i_ << std::endl;
if (iter_->end_state)
{
end_state (stream_);
stream_ << iter_->id;
unique_id (stream_);
stream_ << iter_->unique_id;
dfa (stream_);
stream_ << iter_->goto_dfa;
stream_ << std::endl;
}
if (iter_->bol_index != npos)
{
bol (stream_);
stream_ << iter_->bol_index << std::endl;
}
if (iter_->eol_index != npos)
{
eol (stream_);
stream_ << iter_->eol_index << std::endl;
}
const std::size_t transitions_ = iter_->transitions;
if (transitions_ == 0)
{
++iter_;
}
for (std::size_t t_ = 0; t_ < transitions_; ++t_)
{
std::size_t goto_state_ = iter_->goto_state;
if (iter_->token.any ())
{
any (stream_);
}
else
{
open_bracket (stream_);
if (iter_->token._negated)
{
negated (stream_);
}
string charset_;
CharT c_ = 0;
escape_control_chars (iter_->token._charset,
charset_);
c_ = *charset_.c_str ();
if (!iter_->token._negated &&
(c_ == '^' || c_ == ']'))
{
stream_ << '\\';
}
stream_ << charset_;
close_bracket (stream_);
}
stream_ << goto_state_ << std::endl;
++iter_;
}
stream_ << std::endl;
}
}
static void lexer_state (std::ostream &stream_)
{
stream_ << "Lexer state: ";
}
static void lexer_state (std::wostream &stream_)
{
stream_ << L"Lexer state: ";
}
static void state (std::ostream &stream_)
{
stream_ << "State: ";
}
static void state (std::wostream &stream_)
{
stream_ << L"State: ";
}
static void bol (std::ostream &stream_)
{
stream_ << " BOL -> ";
}
static void bol (std::wostream &stream_)
{
stream_ << L" BOL -> ";
}
static void eol (std::ostream &stream_)
{
stream_ << " EOL -> ";
}
static void eol (std::wostream &stream_)
{
stream_ << L" EOL -> ";
}
static void end_state (std::ostream &stream_)
{
stream_ << " END STATE, Id = ";
}
static void end_state (std::wostream &stream_)
{
stream_ << L" END STATE, Id = ";
}
static void unique_id (std::ostream &stream_)
{
stream_ << ", Unique Id = ";
}
static void unique_id (std::wostream &stream_)
{
stream_ << L", Unique Id = ";
}
static void any (std::ostream &stream_)
{
stream_ << " . -> ";
}
static void any (std::wostream &stream_)
{
stream_ << L" . -> ";
}
static void open_bracket (std::ostream &stream_)
{
stream_ << " [";
}
static void open_bracket (std::wostream &stream_)
{
stream_ << L" [";
}
static void negated (std::ostream &stream_)
{
stream_ << "^";
}
static void negated (std::wostream &stream_)
{
stream_ << L"^";
}
static void close_bracket (std::ostream &stream_)
{
stream_ << "] -> ";
}
static void close_bracket (std::wostream &stream_)
{
stream_ << L"] -> ";
}
static void dfa (std::ostream &stream_)
{
stream_ << ", dfa = ";
}
static void dfa (std::wostream &stream_)
{
stream_ << L", dfa = ";
}
};
typedef basic_debug<char> debug;
typedef basic_debug<wchar_t> wdebug;
}
}
#endif

View File

@@ -0,0 +1,475 @@
// file_input.hpp
// Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_FILE_INPUT
#define BOOST_LEXER_FILE_INPUT
#include "char_traits.hpp"
// memcpy
#include <cstring>
#include <fstream>
#include "size_t.hpp"
#include "state_machine.hpp"
namespace boost
{
namespace lexer
{
template<typename CharT, typename Traits = char_traits<CharT> >
class basic_file_input
{
public:
class iterator
{
public:
friend class basic_file_input;
struct data
{
std::size_t id;
std::size_t unique_id;
const CharT *start;
const CharT *end;
std::size_t state;
// Construct in end() state.
data () :
id (0),
unique_id (npos),
state (npos)
{
}
bool operator == (const data &rhs_) const
{
return id == rhs_.id && unique_id == rhs_.unique_id &&
start == rhs_.start && end == rhs_.end &&
state == rhs_.state;
}
};
iterator () :
_input (0)
{
}
bool operator == (const iterator &rhs_) const
{
return _data == rhs_._data;
}
bool operator != (const iterator &rhs_) const
{
return !(*this == rhs_);
}
data &operator * ()
{
return _data;
}
data *operator -> ()
{
return &_data;
}
// Let compiler generate operator = ().
// prefix version
iterator &operator ++ ()
{
next_token ();
return *this;
}
// postfix version
iterator operator ++ (int)
{
iterator iter_ = *this;
next_token ();
return iter_;
}
void next_token ()
{
const detail::internals &internals_ =
_input->_state_machine->data ();
_data.start = _data.end;
if (internals_._dfa->size () == 1)
{
_data.id = _input->next (&internals_._lookup->front ()->
front (), internals_._dfa_alphabet.front (),
&internals_._dfa->front ()->front (), _data.start,
_data.end, _data.unique_id);
}
else
{
_data.id = _input->next (internals_, _data.state, _data.start,
_data.end, _data.unique_id);
}
if (_data.id == 0)
{
_data.start = 0;
_data.end = 0;
// Ensure current state matches that returned by end().
_data.state = npos;
}
}
private:
// Not owner (obviously!)
basic_file_input *_input;
data _data;
};
friend class iterator;
// Make it explict that we are NOT taking a copy of state_machine_!
basic_file_input (const basic_state_machine<CharT> *state_machine_,
std::basic_ifstream<CharT> *is_,
const std::streamsize buffer_size_ = 4096,
const std::streamsize buffer_increment_ = 1024) :
_state_machine (state_machine_),
_stream (is_),
_buffer_size (buffer_size_),
_buffer_increment (buffer_increment_),
_buffer (_buffer_size, '!')
{
_start_buffer = &_buffer.front ();
_end_buffer = _start_buffer + _buffer.size ();
_start_token = _end_buffer;
_end_token = _end_buffer;
}
iterator begin ()
{
iterator iter_;
iter_._input = this;
// Over-ride default of 0 (EOF)
iter_._data.id = npos;
iter_._data.start = 0;
iter_._data.end = 0;
iter_._data.state = 0;
++iter_;
return iter_;
}
iterator end ()
{
iterator iter_;
iter_._input = this;
iter_._data.start = 0;
iter_._data.end = 0;
return iter_;
}
void flush ()
{
// This temporary is mandatory, otherwise the
// pointer calculations won't work!
const CharT *temp_ = _end_buffer;
_start_token = _end_token = _end_buffer;
reload_buffer (temp_, true, _end_token);
}
private:
typedef std::basic_istream<CharT> istream;
typedef std::vector<CharT> buffer;
const basic_state_machine<CharT> *_state_machine;
const std::streamsize _buffer_size;
const std::streamsize _buffer_increment;
buffer _buffer;
CharT *_start_buffer;
istream *_stream;
const CharT *_start_token;
const CharT *_end_token;
CharT *_end_buffer;
std::size_t next (const detail::internals &internals_,
std::size_t &start_state_, const CharT * &start_, const CharT * &end_,
std::size_t &unique_id_)
{
_start_token = _end_token;
again:
const std::size_t * lookup_ = &internals_._lookup[start_state_]->
front ();
std::size_t dfa_alphabet_ = internals_._dfa_alphabet[start_state_];
const std::size_t *dfa_ = &internals_._dfa[start_state_]->front ();
const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
const CharT *curr_ = _start_token;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
const CharT *end_token_ = curr_;
for (;;)
{
if (curr_ >= _end_buffer)
{
if (!reload_buffer (curr_, end_state_, end_token_))
{
// EOF
break;
}
}
const std::size_t BOL_state_ = ptr_[bol_index];
const std::size_t EOL_state_ = ptr_[eol_index];
if (BOL_state_ && (_start_token == _start_buffer ||
*(_start_token - 1) == '\n'))
{
ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
}
else if (EOL_state_ && *curr_ == '\n')
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
}
else
{
const std::size_t state_ =
ptr_[lookup_[static_cast<typename Traits::index_type>
(*curr_++)]];
if (state_ == 0)
{
break;
}
ptr_ = &dfa_[state_ * dfa_alphabet_];
}
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
start_state_ = *(ptr_ + state_index);
end_token_ = curr_;
}
}
if (_start_token >= _end_buffer)
{
// No more tokens...
unique_id_ = npos;
return 0;
}
const std::size_t EOL_state_ = ptr_[eol_index];
if (EOL_state_ && curr_ == end_)
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
start_state_ = *(ptr_ + state_index);
end_token_ = curr_;
}
}
if (end_state_)
{
// return longest match
_end_token = end_token_;
if (id_ == 0) goto again;
}
else
{
// No match causes char to be skipped
_end_token = _start_token + 1;
id_ = npos;
uid_ = npos;
}
start_ = _start_token;
end_ = _end_token;
unique_id_ = uid_;
return id_;
}
std::size_t next (const std::size_t * const lookup_,
const std::size_t dfa_alphabet_, const std::size_t * const dfa_,
const CharT * &start_, const CharT * &end_, std::size_t &unique_id_)
{
_start_token = _end_token;
const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
const CharT *curr_ = _start_token;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
const CharT *end_token_ = curr_;
for (;;)
{
if (curr_ >= _end_buffer)
{
if (!reload_buffer (curr_, end_state_, end_token_))
{
// EOF
break;
}
}
const std::size_t BOL_state_ = ptr_[bol_index];
const std::size_t EOL_state_ = ptr_[eol_index];
if (BOL_state_ && (_start_token == _start_buffer ||
*(_start_token - 1) == '\n'))
{
ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
}
else if (EOL_state_ && *curr_ == '\n')
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
}
else
{
const std::size_t state_ =
ptr_[lookup_[static_cast<typename Traits::index_type>
(*curr_++)]];
if (state_ == 0)
{
break;
}
ptr_ = &dfa_[state_ * dfa_alphabet_];
}
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_token_ = curr_;
}
}
if (_start_token >= _end_buffer)
{
// No more tokens...
unique_id_ = npos;
return 0;
}
const std::size_t EOL_state_ = ptr_[eol_index];
if (EOL_state_ && curr_ == end_)
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_token_ = curr_;
}
}
if (end_state_)
{
// return longest match
_end_token = end_token_;
}
else
{
// No match causes char to be skipped
_end_token = _start_token + 1;
id_ = npos;
uid_ = npos;
}
start_ = _start_token;
end_ = _end_token;
unique_id_ = uid_;
return id_;
}
bool reload_buffer (const CharT * &curr_, const bool end_state_,
const CharT * &end_token_)
{
bool success_ = !_stream->eof ();
if (success_)
{
const CharT *old_start_token_ = _start_token;
std::size_t old_size_ = _buffer.size ();
std::size_t count_ = 0;
if (_start_token - 1 == _start_buffer)
{
// Run out of buffer space, so increase.
_buffer.resize (old_size_ + _buffer_increment, '!');
_start_buffer = &_buffer.front ();
_start_token = _start_buffer + 1;
_stream->read (_start_buffer + old_size_,
_buffer_increment);
count_ = _stream->gcount ();
_end_buffer = _start_buffer + old_size_ + count_;
}
else if (_start_token < _end_buffer)
{
const std::size_t len_ = _end_buffer - _start_token;
// Some systems have memcpy in namespace std.
using namespace std;
memcpy (_start_buffer, _start_token - 1, (len_ + 1) *
sizeof (CharT));
_stream->read (_start_buffer + len_ + 1,
static_cast<std::streamsize> (_buffer.size () - len_ - 1));
count_ = _stream->gcount ();
_start_token = _start_buffer + 1;
_end_buffer = _start_buffer + len_ + 1 + count_;
}
else
{
_stream->read (_start_buffer, static_cast<std::streamsize>
(_buffer.size ()));
count_ = _stream->gcount ();
_start_token = _start_buffer;
_end_buffer = _start_buffer + count_;
}
if (end_state_)
{
end_token_ = _start_token +
(end_token_ - old_start_token_);
}
curr_ = _start_token + (curr_ - old_start_token_);
}
return success_;
}
// Disallow copying of buffer
basic_file_input (const basic_file_input &);
const basic_file_input &operator = (const basic_file_input &);
};
typedef basic_file_input<char> file_input;
typedef basic_file_input<wchar_t> wfile_input;
}
}
#endif

View File

@@ -0,0 +1,577 @@
// generate_cpp.hpp
// Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_GENERATE_CPP_HPP
#define BOOST_LEXER_GENERATE_CPP_HPP
#include "char_traits.hpp"
#include "consts.hpp"
#include "internals.hpp"
#include <iostream>
#include <boost/detail/iterator.hpp>
#include "runtime_error.hpp"
#include "size_t.hpp"
#include "state_machine.hpp"
#include <vector>
namespace boost
{
namespace lexer
{
template<typename CharT>
void generate_cpp (const basic_state_machine<CharT> &state_machine_,
std::ostream &os_, const bool use_pointers_ = false,
const bool skip_unknown_ = true, const bool optimise_parameters_ = true,
const char *name_ = "next_token")
{
const detail::internals &sm_ = state_machine_.data ();
if (sm_._lookup->size () == 0)
{
throw runtime_error ("Cannot generate code from an empty "
"state machine");
}
std::string upper_name_ (__DATE__);
const std::size_t lookups_ = sm_._lookup->front ()->size ();
const std::size_t dfas_ = sm_._dfa->size ();
std::string::size_type pos_ = upper_name_.find (' ');
const char *iterator_ = 0;
if (use_pointers_)
{
if (lookups_ == 256)
{
iterator_ = "const char *";
}
else
{
iterator_ = "const wchar_t *";
}
}
else
{
iterator_ = "Iterator &";
}
while (pos_ != std::string::npos)
{
upper_name_.replace (pos_, 1, "_");
pos_ = upper_name_.find (' ', pos_);
}
upper_name_ += '_';
upper_name_ += __TIME__;
pos_ = upper_name_.find (':');
while (pos_ != std::string::npos)
{
upper_name_.erase (pos_, 1);
pos_ = upper_name_.find (':', pos_);
}
upper_name_ = '_' + upper_name_;
upper_name_ = name_ + upper_name_;
std::transform (upper_name_.begin (), upper_name_.end (),
upper_name_.begin (), ::toupper);
os_ << "#ifndef " << upper_name_ + '\n';
os_ << "#define " << upper_name_ + '\n';
os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
os_ << "//\n";
os_ << "// Distributed under the Boost Software License, "
"Version 1.0. (See accompanying\n";
os_ << "// file licence_1_0.txt or copy at "
"http://www.boost.org/LICENSE_1_0.txt)\n\n";
os_ << "// Auto-generated by boost::lexer\n";
os_ << "template<typename Iterator>\n";
os_ << "std::size_t " << name_ << " (";
if (dfas_ > 1 || !optimise_parameters_)
{
os_ << "std::size_t &start_state_, ";
}
if (use_pointers_)
{
os_ << iterator_ << " &";
}
else
{
os_ << iterator_;
}
os_ << "start_token_, ";
if (use_pointers_)
{
os_ << iterator_ << " const ";
}
else
{
os_ << "const " << iterator_;
}
os_ << "end_, \n";
os_ << " std::size_t &unique_id_";
if (sm_._seen_BOL_assertion || !optimise_parameters_)
{
os_ << ", bool &beg_of_line_";
}
os_ << ")\n";
os_ << "{\n";
os_ << " enum {end_state_index, id_index, unique_id_index, state_index, bol_index,\n";
os_ << " eol_index, dead_state_index, dfa_offset};\n";
os_ << " static const std::size_t npos = static_cast"
"<std::size_t>(~0);\n";
if (dfas_ > 1)
{
std::size_t state_ = 0;
for (; state_ < dfas_; ++state_)
{
std::size_t i_ = 0;
std::size_t j_ = 1;
std::size_t count_ = lookups_ / 8;
const std::size_t *lookup_ = &sm_._lookup[state_]->front ();
const std::size_t *dfa_ = &sm_._dfa[state_]->front ();
os_ << " static const std::size_t lookup" << state_ << "_[" <<
lookups_ << "] = {";
for (; i_ < count_; ++i_)
{
const std::size_t index_ = i_ * 8;
os_ << lookup_[index_];
for (; j_ < 8; ++j_)
{
os_ << ", " << lookup_[index_ + j_];
}
if (i_ < count_ - 1)
{
os_ << "," << std::endl << " ";
}
j_ = 1;
}
os_ << "};\n";
count_ = sm_._dfa[state_]->size ();
os_ << " static const std::size_t dfa" << state_ << "_[" <<
count_ << "] = {";
count_ /= 8;
for (i_ = 0; i_ < count_; ++i_)
{
const std::size_t index_ = i_ * 8;
os_ << dfa_[index_];
for (j_ = 1; j_ < 8; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
if (i_ < count_ - 1)
{
os_ << "," << std::endl << " ";
}
}
const std::size_t mod_ = sm_._dfa[state_]->size () % 8;
if (mod_)
{
const std::size_t index_ = count_ * 8;
if (count_)
{
os_ << ",\n ";
}
os_ << dfa_[index_];
for (j_ = 1; j_ < mod_; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
}
os_ << "};\n";
}
std::size_t count_ = sm_._dfa_alphabet.size ();
std::size_t i_ = 1;
os_ << " static const std::size_t *lookup_arr_[" << count_ <<
"] = {";
os_ << "lookup0_";
for (i_ = 1; i_ < count_; ++i_)
{
os_ << ", " << "lookup" << i_ << "_";
}
os_ << "};\n";
os_ << " static const std::size_t dfa_alphabet_arr_[" << count_ <<
"] = {";
os_ << sm_._dfa_alphabet.front ();
for (i_ = 1; i_ < count_; ++i_)
{
os_ << ", " << sm_._dfa_alphabet[i_];
}
os_ << "};\n";
os_ << " static const std::size_t *dfa_arr_[" << count_ <<
"] = {";
os_ << "dfa0_";
for (i_ = 1; i_ < count_; ++i_)
{
os_ << ", " << "dfa" << i_ << "_";
}
os_ << "};\n";
}
else
{
const std::size_t *lookup_ = &sm_._lookup->front ()->front ();
const std::size_t *dfa_ = &sm_._dfa->front ()->front ();
std::size_t i_ = 0;
std::size_t j_ = 1;
std::size_t count_ = lookups_ / 8;
os_ << " static const std::size_t lookup_[";
os_ << sm_._lookup->front ()->size () << "] = {";
for (; i_ < count_; ++i_)
{
const std::size_t index_ = i_ * 8;
os_ << lookup_[index_];
for (; j_ < 8; ++j_)
{
os_ << ", " << lookup_[index_ + j_];
}
if (i_ < count_ - 1)
{
os_ << "," << std::endl << " ";
}
j_ = 1;
}
os_ << "};\n";
os_ << " static const std::size_t dfa_alphabet_ = " <<
sm_._dfa_alphabet.front () << ";\n";
os_ << " static const std::size_t dfa_[" <<
sm_._dfa->front ()->size () << "] = {";
count_ = sm_._dfa->front ()->size () / 8;
for (i_ = 0; i_ < count_; ++i_)
{
const std::size_t index_ = i_ * 8;
os_ << dfa_[index_];
for (j_ = 1; j_ < 8; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
if (i_ < count_ - 1)
{
os_ << "," << std::endl << " ";
}
}
const std::size_t mod_ = sm_._dfa->front ()->size () % 8;
if (mod_)
{
const std::size_t index_ = count_ * 8;
if (count_)
{
os_ << ",\n ";
}
os_ << dfa_[index_];
for (j_ = 1; j_ < mod_; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
}
os_ << "};\n";
}
os_ << "\n if (start_token_ == end_)\n";
os_ << " {\n";
os_ << " unique_id_ = npos;\n";
os_ << " return 0;\n";
os_ << " }\n\n";
if (dfas_ > 1)
{
os_ << "again:\n";
os_ << " const std::size_t * lookup_ = "
"lookup_arr_[start_state_];\n";
os_ << " std::size_t dfa_alphabet_ = "
"dfa_alphabet_arr_[start_state_];\n";
os_ << " const std::size_t *dfa_ = dfa_arr_[start_state_];\n";
}
os_ << " const std::size_t *ptr_ = dfa_ + dfa_alphabet_;\n";
os_ << " Iterator curr_ = start_token_;\n";
os_ << " bool end_state_ = *ptr_ != 0;\n";
os_ << " std::size_t id_ = *(ptr_ + id_index);\n";
os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n";
if (dfas_ > 1)
{
os_ << " std::size_t end_start_state_ = start_state_;\n";
}
if (sm_._seen_BOL_assertion)
{
os_ << " bool bol_ = beg_of_line_;\n";
os_ << " bool end_bol_ = bol_;\n";
}
os_ << " Iterator end_token_ = start_token_;\n";
os_ << '\n';
os_ << " while (curr_ != end_)\n";
os_ << " {\n";
if (sm_._seen_BOL_assertion)
{
os_ << " const std::size_t BOL_state_ = ptr_[bol_index];\n";
}
if (sm_._seen_EOL_assertion)
{
os_ << " const std::size_t EOL_state_ = ptr_[eol_index];\n";
}
if (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion)
{
os_ << '\n';
}
if (sm_._seen_BOL_assertion)
{
os_ << " if (BOL_state_ && bol_)\n";
os_ << " {\n";
os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
os_ << " }\n";
}
if (sm_._seen_EOL_assertion)
{
os_ << " ";
if (sm_._seen_BOL_assertion)
{
os_ << "else ";
}
os_ << "if (EOL_state_ && *curr_ == '\\n')\n";
os_ << " {\n";
os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
os_ << " }\n";
}
std::string tab_ (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion ? " " : "");
if (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion)
{
os_ << " else\n";
os_ << " {\n";
}
if (sm_._seen_BOL_assertion)
{
os_ << " ";
if (lookups_ == 256)
{
os_ << "char";
}
else
{
os_ << "wchar_t";
}
os_ << " prev_char_ = *curr_++;\n\n";
os_ << " bol_ = prev_char_ == '\\n';\n\n";
}
os_ << tab_;
os_ << " const std::size_t state_ =\n";
os_ << tab_;
os_ << " ptr_[lookup_[";
if (lookups_ == 256)
{
os_ << "static_cast<unsigned char>(";
}
if (sm_._seen_BOL_assertion)
{
os_ << "prev_char";
}
else
{
os_ << "*curr_++";
}
if (lookups_ == 256)
{
os_ << ')';
}
os_ << "]];\n\n";
os_ << tab_;
os_ << " if (state_ == 0) break;\n\n";
os_ << tab_;
os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
if (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion)
{
os_ << " }\n";
}
os_ << '\n';
os_ << " if (*ptr_)\n";
os_ << " {\n";
os_ << " end_state_ = true;\n";
os_ << " id_ = *(ptr_ + id_index);\n";
os_ << " uid_ = *(ptr_ + unique_id_index);\n";
if (dfas_ > 1)
{
os_ << " end_start_state_ = *(ptr_ + state_index);\n";
}
if (sm_._seen_BOL_assertion)
{
os_ << " end_bol_ = bol_;\n";
}
os_ << " end_token_ = curr_;\n";
os_ << " }\n";
os_ << " }\n";
os_ << '\n';
if (sm_._seen_EOL_assertion)
{
os_ << " const std::size_t EOL_state_ = ptr_[eol_index];\n";
os_ << '\n';
os_ << " if (EOL_state_ && curr_ == end_)\n";
os_ << " {\n";
os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
os_ << '\n';
os_ << " if (*ptr_)\n";
os_ << " {\n";
os_ << " end_state_ = true;\n";
os_ << " id_ = *(ptr_ + id_index);\n";
os_ << " uid_ = *(ptr_ + unique_id_index);\n";
if (dfas_ > 1)
{
os_ << " end_start_state_ = *(ptr_ + state_index);\n";
}
if (sm_._seen_BOL_assertion)
{
os_ << " end_bol_ = bol_;\n";
}
os_ << " end_token_ = curr_;\n";
os_ << " }\n";
os_ << " }\n";
os_ << '\n';
}
os_ << " if (end_state_)\n";
os_ << " {\n";
os_ << " // return longest match\n";
if (dfas_ > 1)
{
os_ << " start_state_ = end_start_state_;\n";
}
if (sm_._seen_BOL_assertion && dfas_ < 2)
{
os_ << " beg_of_line_ = end_bol_;\n";
}
os_ << " start_token_ = end_token_;\n";
if (dfas_ > 1)
{
os_ << '\n';
os_ << " if (id_ == 0)\n";
os_ << " {\n";
if (sm_._seen_BOL_assertion)
{
os_ << " bol_ = end_bol_;\n";
}
os_ << " goto again;\n";
os_ << " }\n";
if (sm_._seen_BOL_assertion)
{
os_ << " else\n";
os_ << " {\n";
os_ << " beg_of_line_ = end_bol_;\n";
os_ << " }\n";
}
}
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
if (sm_._seen_BOL_assertion)
{
os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
}
if (skip_unknown_)
{
os_ << " // No match causes char to be skipped\n";
os_ << " ++start_token_;\n";
}
os_ << " id_ = npos;\n";
os_ << " uid_ = npos;\n";
os_ << " }\n";
os_ << '\n';
os_ << " unique_id_ = uid_;\n";
os_ << " return id_;\n";
os_ << "}\n";
os_ << "\n#endif\n";
}
}
}
#endif

View File

@@ -0,0 +1,440 @@
// generate_re2c.hpp
// Copyright (c) 2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef LEXERTL_GENERATE_RE2C_HPP
#define LEXERTL_GENERATE_RE2C_HPP
#include "char_traits.hpp"
#include "consts.hpp"
#include "internals.hpp"
#include <iostream>
#include "runtime_error.hpp"
#include "size_t.hpp"
#include "state_machine.hpp"
#include <vector>
namespace boost
{
namespace lexer
{
// check whether state0_0 is referenced from any of the other states
template <typename Char>
bool need_label0_0(boost::lexer::basic_state_machine<Char> const &sm_)
{
typedef typename boost::lexer::basic_state_machine<Char>::iterator
iterator_type;
iterator_type iter_ = sm_.begin();
std::size_t states_ = iter_->states;
for (std::size_t state_ = 0; state_ < states_; ++state_)
{
if (0 == iter_->bol_index || 0 == iter_->eol_index)
{
return true;
}
std::size_t const transitions_ = iter_->transitions;
for (std::size_t t_ = 0; t_ < transitions_; ++t_)
{
if (0 == iter_->goto_state)
{
return true;
}
++iter_;
}
if (transitions_ == 0) ++iter_;
}
return false;
}
template<typename CharT>
void generate_re2c (const basic_state_machine<CharT> &state_machine_,
std::ostream &os_, const bool use_pointers_ = false,
const bool skip_unknown_ = true, const bool optimise_parameters_ = true,
const char *name_ = "next_token")
{
typedef typename boost::lexer::basic_string_token<CharT> string_token;
const detail::internals &sm_ = state_machine_.data ();
if (sm_._lookup->size () == 0)
{
throw runtime_error ("Cannot generate code from an empty "
"state machine");
}
std::string upper_name_ (__DATE__);
const std::size_t lookups_ = sm_._lookup->front ()->size ();
typename boost::lexer::basic_state_machine<CharT>::iterator iter_ =
state_machine_.begin();
typename boost::lexer::basic_state_machine<CharT>::iterator end_ =
state_machine_.end();
const std::size_t dfas_ = sm_._dfa->size ();
std::string::size_type pos_ = upper_name_.find (' ');
const char *iterator_ = 0;
if (use_pointers_)
{
if (lookups_ == 256)
{
iterator_ = "const char *";
}
else
{
iterator_ = "const wchar_t *";
}
}
else
{
iterator_ = "Iterator &";
}
while (pos_ != std::string::npos)
{
upper_name_.replace (pos_, 1, "_");
pos_ = upper_name_.find (' ', pos_);
}
upper_name_ += '_';
upper_name_ += __TIME__;
pos_ = upper_name_.find (':');
while (pos_ != std::string::npos)
{
upper_name_.erase (pos_, 1);
pos_ = upper_name_.find (':', pos_);
}
upper_name_ = '_' + upper_name_;
upper_name_ = name_ + upper_name_;
std::transform (upper_name_.begin (), upper_name_.end (),
upper_name_.begin (), ::toupper);
os_ << "#ifndef " << upper_name_ + '\n';
os_ << "#define " << upper_name_ + '\n';
os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
os_ << "//\n";
os_ << "// Distributed under the Boost Software License, "
"Version 1.0. (See accompanying\n";
os_ << "// file licence_1_0.txt or copy at "
"http://www.boost.org/LICENSE_1_0.txt)\n\n";
os_ << "// Auto-generated by boost::lexer\n";
os_ << "template<typename Iterator>\n";
os_ << "std::size_t " << name_ << " (";
if (dfas_ > 1 || !optimise_parameters_)
{
os_ << "std::size_t &start_state_, ";
}
if (use_pointers_)
{
os_ << iterator_ << " &";
}
else
{
os_ << iterator_;
}
os_ << "start_token_, ";
if (use_pointers_)
{
os_ << iterator_ << " const ";
}
else
{
os_ << "const " << iterator_;
}
os_ << "end_, \n";
os_ << " std::size_t &unique_id_";
if (sm_._seen_BOL_assertion || !optimise_parameters_)
{
os_ << ", bool &beg_of_line_";
}
os_ << ")\n";
os_ << "{\n";
os_ << " static const std::size_t npos = static_cast"
"<std::size_t>(~0);\n";
os_ << "\n if (start_token_ == end_)\n";
os_ << " {\n";
os_ << " unique_id_ = npos;\n";
os_ << " return 0;\n";
os_ << " }\n\n";
if (dfas_ > 1)
{
os_ << "again:\n";
}
os_ << " Iterator curr_ = start_token_;\n";
os_ << " bool end_state_ = false;\n";
os_ << " std::size_t id_ = npos;\n";
os_ << " std::size_t uid_ = npos;\n";
if (dfas_ > 1)
{
os_ << " std::size_t end_start_state_ = start_state_;\n";
}
if (sm_._seen_BOL_assertion)
{
os_ << " bool bol_ = beg_of_line_;\n";
os_ << " bool end_bol_ = bol_;\n";
}
os_ << " Iterator end_token_ = start_token_;\n";
os_ << '\n';
if (dfas_ > 1)
{
os_ << " switch (start_state_)\n";
os_ << " {\n";
for (std::size_t i_ = 0; i_ < dfas_; ++i_)
{
os_ << " case " << i_ << ":\n";
os_ << " goto " << i_ << "_0;\n";
os_ << " // Not needed, but to prevent warnings\n";
os_ << " break;\n";
}
os_ << " default:\n";
os_ << " throw std::runtime_error (\"Invalid start state!\")\n";
os_ << " break;\n";
os_ << " }\n\n";
}
os_ << " ";
if (lookups_ == 256)
{
os_ << "char";
}
else
{
os_ << "wchar_t";
}
os_ << " ch_ = 0;\n\n";
bool need_state0_0_label = need_label0_0(state_machine_);
for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_)
{
const std::size_t states_ = iter_->states;
for (std::size_t state_ = 0; state_ < states_; ++state_)
{
const std::size_t transitions_ = iter_->transitions;
std::size_t t_ = 0;
if (dfas_ > 1 || dfa_ != 0 || state_ != 0 || need_state0_0_label)
{
os_ << "state" << dfa_ << '_' << state_ << ":\n";
}
if (iter_->end_state)
{
os_ << " end_state_ = true;\n";
os_ << " id_ = " << iter_->id << ";\n";
os_ << " uid_ = " << iter_->unique_id << ";\n";
os_ << " end_token_ = curr_;\n";
if (dfas_ > 1)
{
os_ << " end_start_state_ = " << iter_->goto_dfa <<
";\n";
}
if (sm_._seen_BOL_assertion)
{
os_ << " end_bol_ = bol_;\n";
}
if (transitions_) os_ << '\n';
}
if (t_ < transitions_ || iter_->bol_index != boost::lexer::npos ||
iter_->eol_index != boost::lexer::npos)
{
os_ << " if (curr_ == end_) goto end;\n\n";
os_ << " ch_ = *curr_;\n";
if (iter_->bol_index != boost::lexer::npos)
{
os_ << "\n if (bol_) goto state" << dfa_ << '_' <<
iter_->bol_index << ";\n\n";
}
if (iter_->eol_index != boost::lexer::npos)
{
os_ << "\n if (ch_ == '\n') goto state" << dfa_ << '_' <<
iter_->eol_index << ";\n\n";
}
os_ << " ++curr_;\n";
}
for (; t_ < transitions_; ++t_)
{
const char *ptr_ = iter_->token._charset.c_str();
const char *end_ = ptr_ + iter_->token._charset.size();
char start_char_ = 0;
char curr_char_ = 0;
bool range_ = false;
bool first_char_ = true;
os_ << "\n if (";
while (ptr_ != end_)
{
curr_char_ = *ptr_++;
if (*ptr_ == curr_char_ + 1)
{
if (!range_)
{
start_char_ = curr_char_;
}
range_ = true;
}
else
{
if (!first_char_)
{
if (iter_->token._negated)
{
os_ << " && ";
}
else
{
os_ << " || ";
}
}
first_char_ = false;
if (range_)
{
typename string_token::string temp_;
if (iter_->token._negated)
{
os_ << "!";
}
string_token::escape_char (start_char_, temp_);
os_ << "(ch_ >= '" << temp_;
temp_.clear ();
string_token::escape_char (curr_char_, temp_);
os_ << "' && ch_ <= '" << temp_ << "')";
range_ = false;
}
else
{
typename string_token::string temp_;
os_ << "ch_ ";
if (iter_->token._negated)
{
os_ << "!=";
}
else
{
os_ << "==";
}
string_token::escape_char (curr_char_, temp_);
os_ << " '" << temp_ << "'";
}
}
}
os_ << ") goto state" << dfa_ << '_' << iter_->goto_state <<
";\n\n";
++iter_;
}
if (!(dfa_ == dfas_ - 1 && state_ == states_ - 1))
{
os_ << " goto end;\n";
}
if (transitions_ == 0) ++iter_;
}
}
os_ << "end:\n";
os_ << " if (end_state_)\n";
os_ << " {\n";
os_ << " // return longest match\n";
if (dfas_ > 1)
{
os_ << " start_state_ = end_start_state_;\n";
}
if (sm_._seen_BOL_assertion && dfas_ < 2)
{
os_ << " beg_of_line_ = end_bol_;\n";
}
os_ << " start_token_ = end_token_;\n";
if (dfas_ > 1)
{
os_ << '\n';
os_ << " if (id_ == 0)\n";
os_ << " {\n";
if (sm_._seen_BOL_assertion)
{
os_ << " bol_ = end_bol_;\n";
}
os_ << " goto again;\n";
os_ << " }\n";
if (sm_._seen_BOL_assertion)
{
os_ << " else\n";
os_ << " {\n";
os_ << " beg_of_line_ = end_bol_;\n";
os_ << " }\n";
}
}
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
if (sm_._seen_BOL_assertion)
{
os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
}
if (skip_unknown_)
{
os_ << " // No match causes char to be skipped\n";
os_ << " ++start_token_;\n";
}
os_ << " id_ = npos;\n";
os_ << " uid_ = npos;\n";
os_ << " }\n";
os_ << '\n';
os_ << " unique_id_ = uid_;\n";
os_ << " return id_;\n";
os_ << "}\n";
os_ << "\n#endif\n";
}
}
}
#endif

View File

@@ -0,0 +1,864 @@
// generator.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_GENERATOR_HPP
#define BOOST_LEXER_GENERATOR_HPP
#include "char_traits.hpp"
// memcmp()
#include <cstring>
#include "partition/charset.hpp"
#include "partition/equivset.hpp"
#include <memory>
#include <limits>
#include "parser/tree/node.hpp"
#include "parser/parser.hpp"
#include "containers/ptr_list.hpp"
#include "rules.hpp"
#include "state_machine.hpp"
namespace boost
{
namespace lexer
{
template<typename CharT, typename Traits = char_traits<CharT> >
class basic_generator
{
public:
typedef typename detail::internals::size_t_vector size_t_vector;
typedef basic_rules<CharT> rules;
static void build (const rules &rules_,
basic_state_machine<CharT> &state_machine_)
{
std::size_t index_ = 0;
std::size_t size_ = rules_.statemap ().size ();
node_ptr_vector node_ptr_vector_;
detail::internals &internals_ = const_cast<detail::internals &>
(state_machine_.data ());
bool seen_BOL_assertion_ = false;
bool seen_EOL_assertion_ = false;
state_machine_.clear ();
for (; index_ < size_; ++index_)
{
internals_._lookup->push_back (static_cast<size_t_vector *>(0));
internals_._lookup->back () = new size_t_vector;
internals_._dfa_alphabet.push_back (0);
internals_._dfa->push_back (static_cast<size_t_vector *>(0));
internals_._dfa->back () = new size_t_vector;
}
for (index_ = 0, size_ = internals_._lookup->size ();
index_ < size_; ++index_)
{
internals_._lookup[index_]->resize (sizeof (CharT) == 1 ?
num_chars : num_wchar_ts, dead_state_index);
if (!rules_.regexes ()[index_].empty ())
{
// vector mapping token indexes to partitioned token index sets
index_set_vector set_mapping_;
// syntax tree
detail::node *root_ = build_tree (rules_, index_,
node_ptr_vector_, internals_, set_mapping_);
build_dfa (root_, set_mapping_,
internals_._dfa_alphabet[index_],
*internals_._dfa[index_]);
if (internals_._seen_BOL_assertion)
{
seen_BOL_assertion_ = true;
}
if (internals_._seen_EOL_assertion)
{
seen_EOL_assertion_ = true;
}
internals_._seen_BOL_assertion = false;
internals_._seen_EOL_assertion = false;
}
}
internals_._seen_BOL_assertion = seen_BOL_assertion_;
internals_._seen_EOL_assertion = seen_EOL_assertion_;
}
static void minimise (basic_state_machine<CharT> &state_machine_)
{
detail::internals &internals_ = const_cast<detail::internals &>
(state_machine_.data ());
const std::size_t machines_ = internals_._dfa->size ();
for (std::size_t i_ = 0; i_ < machines_; ++i_)
{
const std::size_t dfa_alphabet_ = internals_._dfa_alphabet[i_];
size_t_vector *dfa_ = internals_._dfa[i_];
if (dfa_alphabet_ != 0)
{
std::size_t size_ = 0;
do
{
size_ = dfa_->size ();
minimise_dfa (dfa_alphabet_, *dfa_, size_);
} while (dfa_->size () != size_);
}
}
}
protected:
typedef detail::basic_charset<CharT> charset;
typedef detail::ptr_list<charset> charset_list;
typedef std::auto_ptr<charset> charset_ptr;
typedef detail::equivset equivset;
typedef detail::ptr_list<equivset> equivset_list;
typedef std::auto_ptr<equivset> equivset_ptr;
typedef typename charset::index_set index_set;
typedef std::vector<index_set> index_set_vector;
typedef detail::basic_parser<CharT> parser;
typedef typename parser::node_ptr_vector node_ptr_vector;
typedef std::set<const detail::node *> node_set;
typedef detail::ptr_vector<node_set> node_set_vector;
typedef std::vector<const detail::node *> node_vector;
typedef detail::ptr_vector<node_vector> node_vector_vector;
typedef typename parser::string string;
typedef std::pair<string, string> string_pair;
typedef typename parser::tokeniser::string_token string_token;
typedef std::deque<string_pair> macro_deque;
typedef std::pair<string, const detail::node *> macro_pair;
typedef typename parser::macro_map::iterator macro_iter;
typedef std::pair<macro_iter, bool> macro_iter_pair;
typedef typename parser::tokeniser::token_map token_map;
static detail::node *build_tree (const rules &rules_,
const std::size_t state_, node_ptr_vector &node_ptr_vector_,
detail::internals &internals_, index_set_vector &set_mapping_)
{
size_t_vector *lookup_ = internals_._lookup[state_];
const typename rules::string_deque_deque &regexes_ =
rules_.regexes ();
const typename rules::id_vector_deque &ids_ = rules_.ids ();
const typename rules::id_vector_deque &unique_ids_ =
rules_.unique_ids ();
const typename rules::id_vector_deque &states_ = rules_.states ();
typename rules::string_deque::const_iterator regex_iter_ =
regexes_[state_].begin ();
typename rules::string_deque::const_iterator regex_iter_end_ =
regexes_[state_].end ();
typename rules::id_vector::const_iterator ids_iter_ =
ids_[state_].begin ();
typename rules::id_vector::const_iterator unique_ids_iter_ =
unique_ids_[state_].begin ();
typename rules::id_vector::const_iterator states_iter_ =
states_[state_].begin ();
const typename rules::string &regex_ = *regex_iter_;
// map of regex charset tokens (strings) to index
token_map token_map_;
const typename rules::string_pair_deque &macrodeque_ =
rules_.macrodeque ();
typename parser::macro_map macromap_;
typename detail::node::node_vector tree_vector_;
build_macros (token_map_, macrodeque_, macromap_,
rules_.flags (), rules_.locale (), node_ptr_vector_,
internals_._seen_BOL_assertion, internals_._seen_EOL_assertion);
detail::node *root_ = parser::parse (regex_.c_str (),
regex_.c_str () + regex_.size (), *ids_iter_, *unique_ids_iter_,
*states_iter_, rules_.flags (), rules_.locale (), node_ptr_vector_,
macromap_, token_map_, internals_._seen_BOL_assertion,
internals_._seen_EOL_assertion);
++regex_iter_;
++ids_iter_;
++unique_ids_iter_;
++states_iter_;
tree_vector_.push_back (root_);
// build syntax trees
while (regex_iter_ != regex_iter_end_)
{
// re-declare var, otherwise we perform an assignment..!
const typename rules::string &regex2_ = *regex_iter_;
root_ = parser::parse (regex2_.c_str (),
regex2_.c_str () + regex2_.size (), *ids_iter_,
*unique_ids_iter_, *states_iter_, rules_.flags (),
rules_.locale (), node_ptr_vector_, macromap_, token_map_,
internals_._seen_BOL_assertion,
internals_._seen_EOL_assertion);
tree_vector_.push_back (root_);
++regex_iter_;
++ids_iter_;
++unique_ids_iter_;
++states_iter_;
}
if (internals_._seen_BOL_assertion)
{
// Fixup BOLs
typename detail::node::node_vector::iterator iter_ =
tree_vector_.begin ();
typename detail::node::node_vector::iterator end_ =
tree_vector_.end ();
for (; iter_ != end_; ++iter_)
{
fixup_bol (*iter_, node_ptr_vector_);
}
}
// join trees
{
typename detail::node::node_vector::iterator iter_ =
tree_vector_.begin ();
typename detail::node::node_vector::iterator end_ =
tree_vector_.end ();
if (iter_ != end_)
{
root_ = *iter_;
++iter_;
}
for (; iter_ != end_; ++iter_)
{
node_ptr_vector_->push_back (static_cast<detail::selection_node *>(0));
node_ptr_vector_->back () = new detail::selection_node
(root_, *iter_);
root_ = node_ptr_vector_->back ();
}
}
// partitioned token list
charset_list token_list_;
set_mapping_.resize (token_map_.size ());
partition_tokens (token_map_, token_list_);
typename charset_list::list::const_iterator iter_ =
token_list_->begin ();
typename charset_list::list::const_iterator end_ =
token_list_->end ();
std::size_t index_ = 0;
for (; iter_ != end_; ++iter_, ++index_)
{
const charset *cs_ = *iter_;
typename charset::index_set::const_iterator set_iter_ =
cs_->_index_set.begin ();
typename charset::index_set::const_iterator set_end_ =
cs_->_index_set.end ();
fill_lookup (cs_->_token, lookup_, index_);
for (; set_iter_ != set_end_; ++set_iter_)
{
set_mapping_[*set_iter_].insert (index_);
}
}
internals_._dfa_alphabet[state_] = token_list_->size () + dfa_offset;
return root_;
}
static void build_macros (token_map &token_map_,
const macro_deque &macrodeque_,
typename parser::macro_map &macromap_, const regex_flags flags_,
const std::locale &locale_, node_ptr_vector &node_ptr_vector_,
bool &seen_BOL_assertion_, bool &seen_EOL_assertion_)
{
for (typename macro_deque::const_iterator iter_ =
macrodeque_.begin (), end_ = macrodeque_.end ();
iter_ != end_; ++iter_)
{
const typename rules::string &name_ = iter_->first;
const typename rules::string &regex_ = iter_->second;
detail::node *node_ = parser::parse (regex_.c_str (),
regex_.c_str () + regex_.size (), 0, 0, 0, flags_,
locale_, node_ptr_vector_, macromap_, token_map_,
seen_BOL_assertion_, seen_EOL_assertion_);
macro_iter_pair map_iter_ = macromap_.
insert (macro_pair (name_, static_cast<const detail::node *>
(0)));
map_iter_.first->second = node_;
}
}
static void build_dfa (detail::node *root_,
const index_set_vector &set_mapping_, const std::size_t dfa_alphabet_,
size_t_vector &dfa_)
{
typename detail::node::node_vector *followpos_ =
&root_->firstpos ();
node_set_vector seen_sets_;
node_vector_vector seen_vectors_;
size_t_vector hash_vector_;
// 'jam' state
dfa_.resize (dfa_alphabet_, 0);
closure (followpos_, seen_sets_, seen_vectors_,
hash_vector_, dfa_alphabet_, dfa_);
std::size_t *ptr_ = 0;
for (std::size_t index_ = 0; index_ < seen_vectors_->size (); ++index_)
{
equivset_list equiv_list_;
build_equiv_list (seen_vectors_[index_], set_mapping_, equiv_list_);
for (typename equivset_list::list::const_iterator iter_ =
equiv_list_->begin (), end_ = equiv_list_->end ();
iter_ != end_; ++iter_)
{
equivset *equivset_ = *iter_;
const std::size_t transition_ = closure
(&equivset_->_followpos, seen_sets_, seen_vectors_,
hash_vector_, dfa_alphabet_, dfa_);
if (transition_ != npos)
{
ptr_ = &dfa_.front () + ((index_ + 1) * dfa_alphabet_);
// Prune abstemious transitions from end states.
if (*ptr_ && !equivset_->_greedy) continue;
for (typename detail::equivset::index_vector::const_iterator
equiv_iter_ = equivset_->_index_vector.begin (),
equiv_end_ = equivset_->_index_vector.end ();
equiv_iter_ != equiv_end_; ++equiv_iter_)
{
const std::size_t equiv_index_ = *equiv_iter_;
if (equiv_index_ == bol_token)
{
if (ptr_[eol_index] == 0)
{
ptr_[bol_index] = transition_;
}
}
else if (equiv_index_ == eol_token)
{
if (ptr_[bol_index] == 0)
{
ptr_[eol_index] = transition_;
}
}
else
{
ptr_[equiv_index_ + dfa_offset] = transition_;
}
}
}
}
}
}
static std::size_t closure (typename detail::node::node_vector *followpos_,
node_set_vector &seen_sets_, node_vector_vector &seen_vectors_,
size_t_vector &hash_vector_, const std::size_t size_,
size_t_vector &dfa_)
{
bool end_state_ = false;
std::size_t id_ = 0;
std::size_t unique_id_ = npos;
std::size_t state_ = 0;
std::size_t hash_ = 0;
if (followpos_->empty ()) return npos;
std::size_t index_ = 0;
std::auto_ptr<node_set> set_ptr_ (new node_set);
std::auto_ptr<node_vector> vector_ptr_ (new node_vector);
for (typename detail::node::node_vector::const_iterator iter_ =
followpos_->begin (), end_ = followpos_->end ();
iter_ != end_; ++iter_)
{
closure_ex (*iter_, end_state_, id_, unique_id_, state_,
set_ptr_.get (), vector_ptr_.get (), hash_);
}
bool found_ = false;
typename size_t_vector::const_iterator hash_iter_ =
hash_vector_.begin ();
typename size_t_vector::const_iterator hash_end_ =
hash_vector_.end ();
typename node_set_vector::vector::const_iterator set_iter_ =
seen_sets_->begin ();
for (; hash_iter_ != hash_end_; ++hash_iter_, ++set_iter_)
{
found_ = *hash_iter_ == hash_ && *(*set_iter_) == *set_ptr_;
++index_;
if (found_) break;
}
if (!found_)
{
seen_sets_->push_back (static_cast<node_set *>(0));
seen_sets_->back () = set_ptr_.release ();
seen_vectors_->push_back (static_cast<node_vector *>(0));
seen_vectors_->back () = vector_ptr_.release ();
hash_vector_.push_back (hash_);
// State 0 is the jam state...
index_ = seen_sets_->size ();
const std::size_t old_size_ = dfa_.size ();
dfa_.resize (old_size_ + size_, 0);
if (end_state_)
{
dfa_[old_size_] |= end_state;
dfa_[old_size_ + id_index] = id_;
dfa_[old_size_ + unique_id_index] = unique_id_;
dfa_[old_size_ + state_index] = state_;
}
}
return index_;
}
static void closure_ex (detail::node *node_, bool &end_state_,
std::size_t &id_, std::size_t &unique_id_, std::size_t &state_,
node_set *set_ptr_, node_vector *vector_ptr_, std::size_t &hash_)
{
const bool temp_end_state_ = node_->end_state ();
if (temp_end_state_)
{
if (!end_state_)
{
end_state_ = true;
id_ = node_->id ();
unique_id_ = node_->unique_id ();
state_ = node_->lexer_state ();
}
}
if (set_ptr_->insert (node_).second)
{
vector_ptr_->push_back (node_);
hash_ += reinterpret_cast<std::size_t> (node_);
}
}
static void partition_tokens (const token_map &map_,
charset_list &lhs_)
{
charset_list rhs_;
fill_rhs_list (map_, rhs_);
if (!rhs_->empty ())
{
typename charset_list::list::iterator iter_;
typename charset_list::list::iterator end_;
charset_ptr overlap_ (new charset);
lhs_->push_back (static_cast<charset *>(0));
lhs_->back () = rhs_->front ();
rhs_->pop_front ();
while (!rhs_->empty ())
{
charset_ptr r_ (rhs_->front ());
rhs_->pop_front ();
iter_ = lhs_->begin ();
end_ = lhs_->end ();
while (!r_->empty () && iter_ != end_)
{
typename charset_list::list::iterator l_iter_ = iter_;
(*l_iter_)->intersect (*r_.get (), *overlap_.get ());
if (overlap_->empty ())
{
++iter_;
}
else if ((*l_iter_)->empty ())
{
delete *l_iter_;
*l_iter_ = overlap_.release ();
// VC++ 6 Hack:
charset_ptr temp_overlap_ (new charset);
overlap_ = temp_overlap_;
++iter_;
}
else if (r_->empty ())
{
delete r_.release ();
r_ = overlap_;
// VC++ 6 Hack:
charset_ptr temp_overlap_ (new charset);
overlap_ = temp_overlap_;
break;
}
else
{
iter_ = lhs_->insert (++iter_,
static_cast<charset *>(0));
*iter_ = overlap_.release ();
// VC++ 6 Hack:
charset_ptr temp_overlap_ (new charset);
overlap_ = temp_overlap_;
++iter_;
end_ = lhs_->end ();
}
}
if (!r_->empty ())
{
lhs_->push_back (static_cast<charset *>(0));
lhs_->back () = r_.release ();
}
}
}
}
static void fill_rhs_list (const token_map &map_,
charset_list &list_)
{
typename parser::tokeniser::token_map::const_iterator iter_ =
map_.begin ();
typename parser::tokeniser::token_map::const_iterator end_ =
map_.end ();
for (; iter_ != end_; ++iter_)
{
list_->push_back (static_cast<charset *>(0));
list_->back () = new charset (iter_->first, iter_->second);
}
}
static void fill_lookup (const string_token &token_,
size_t_vector *lookup_, const std::size_t index_)
{
const CharT *curr_ = token_._charset.c_str ();
const CharT *chars_end_ = curr_ + token_._charset.size ();
std::size_t *ptr_ = &lookup_->front ();
const std::size_t max_ = sizeof (CharT) == 1 ?
num_chars : num_wchar_ts;
if (token_._negated)
{
// $$$ FIXME JDG July 2014 $$$
// this code is problematic on platforms where wchar_t is signed
// with min generating negative numbers. This crashes with BAD_ACCESS
// because of the vector index below:
// ptr_[static_cast<typename Traits::index_type>(curr_char_)]
CharT curr_char_ = 0; // (std::numeric_limits<CharT>::min)();
std::size_t i_ = 0;
while (curr_ < chars_end_)
{
while (*curr_ > curr_char_)
{
ptr_[static_cast<typename Traits::index_type>
(curr_char_)] = index_ + dfa_offset;
++curr_char_;
++i_;
}
++curr_char_;
++curr_;
++i_;
}
for (; i_ < max_; ++i_)
{
ptr_[static_cast<typename Traits::index_type>(curr_char_)] =
index_ + dfa_offset;
++curr_char_;
}
}
else
{
while (curr_ < chars_end_)
{
ptr_[static_cast<typename Traits::index_type>(*curr_)] =
index_ + dfa_offset;
++curr_;
}
}
}
static void build_equiv_list (const node_vector *vector_,
const index_set_vector &set_mapping_, equivset_list &lhs_)
{
equivset_list rhs_;
fill_rhs_list (vector_, set_mapping_, rhs_);
if (!rhs_->empty ())
{
typename equivset_list::list::iterator iter_;
typename equivset_list::list::iterator end_;
equivset_ptr overlap_ (new equivset);
lhs_->push_back (static_cast<equivset *>(0));
lhs_->back () = rhs_->front ();
rhs_->pop_front ();
while (!rhs_->empty ())
{
equivset_ptr r_ (rhs_->front ());
rhs_->pop_front ();
iter_ = lhs_->begin ();
end_ = lhs_->end ();
while (!r_->empty () && iter_ != end_)
{
typename equivset_list::list::iterator l_iter_ = iter_;
(*l_iter_)->intersect (*r_.get (), *overlap_.get ());
if (overlap_->empty ())
{
++iter_;
}
else if ((*l_iter_)->empty ())
{
delete *l_iter_;
*l_iter_ = overlap_.release ();
// VC++ 6 Hack:
equivset_ptr temp_overlap_ (new equivset);
overlap_ = temp_overlap_;
++iter_;
}
else if (r_->empty ())
{
delete r_.release ();
r_ = overlap_;
// VC++ 6 Hack:
equivset_ptr temp_overlap_ (new equivset);
overlap_ = temp_overlap_;
break;
}
else
{
iter_ = lhs_->insert (++iter_,
static_cast<equivset *>(0));
*iter_ = overlap_.release ();
// VC++ 6 Hack:
equivset_ptr temp_overlap_ (new equivset);
overlap_ = temp_overlap_;
++iter_;
end_ = lhs_->end ();
}
}
if (!r_->empty ())
{
lhs_->push_back (static_cast<equivset *>(0));
lhs_->back () = r_.release ();
}
}
}
}
static void fill_rhs_list (const node_vector *vector_,
const index_set_vector &set_mapping_, equivset_list &list_)
{
typename node_vector::const_iterator iter_ =
vector_->begin ();
typename node_vector::const_iterator end_ =
vector_->end ();
for (; iter_ != end_; ++iter_)
{
const detail::node *node_ = *iter_;
if (!node_->end_state ())
{
const std::size_t token_ = node_->token ();
if (token_ != null_token)
{
list_->push_back (static_cast<equivset *>(0));
if (token_ == bol_token || token_ == eol_token)
{
std::set<std::size_t> index_set_;
index_set_.insert (token_);
list_->back () = new equivset (index_set_,
node_->greedy (), token_, node_->followpos ());
}
else
{
list_->back () = new equivset (set_mapping_[token_],
node_->greedy (), token_, node_->followpos ());
}
}
}
}
}
static void fixup_bol (detail::node * &root_,
node_ptr_vector &node_ptr_vector_)
{
typename detail::node::node_vector *first_ = &root_->firstpos ();
bool found_ = false;
typename detail::node::node_vector::const_iterator iter_ =
first_->begin ();
typename detail::node::node_vector::const_iterator end_ =
first_->end ();
for (; iter_ != end_; ++iter_)
{
const detail::node *node_ = *iter_;
found_ = !node_->end_state () && node_->token () == bol_token;
if (found_) break;
}
if (!found_)
{
node_ptr_vector_->push_back (static_cast<detail::leaf_node *>(0));
node_ptr_vector_->back () = new detail::leaf_node
(bol_token, true);
detail::node *lhs_ = node_ptr_vector_->back ();
node_ptr_vector_->push_back (static_cast<detail::leaf_node *>(0));
node_ptr_vector_->back () = new detail::leaf_node
(null_token, true);
detail::node *rhs_ = node_ptr_vector_->back ();
node_ptr_vector_->push_back
(static_cast<detail::selection_node *>(0));
node_ptr_vector_->back () =
new detail::selection_node (lhs_, rhs_);
lhs_ = node_ptr_vector_->back ();
node_ptr_vector_->push_back
(static_cast<detail::sequence_node *>(0));
node_ptr_vector_->back () =
new detail::sequence_node (lhs_, root_);
root_ = node_ptr_vector_->back ();
}
}
static void minimise_dfa (const std::size_t dfa_alphabet_,
size_t_vector &dfa_, std::size_t size_)
{
const std::size_t *first_ = &dfa_.front ();
const std::size_t *second_ = 0;
const std::size_t *end_ = first_ + size_;
std::size_t index_ = 1;
std::size_t new_index_ = 1;
std::size_t curr_index_ = 0;
index_set index_set_;
size_t_vector lookup_;
std::size_t *lookup_ptr_ = 0;
lookup_.resize (size_ / dfa_alphabet_, null_token);
lookup_ptr_ = &lookup_.front ();
*lookup_ptr_ = 0;
// Only one 'jam' state, so skip it.
first_ += dfa_alphabet_;
for (; first_ < end_; first_ += dfa_alphabet_, ++index_)
{
for (second_ = first_ + dfa_alphabet_, curr_index_ = index_ + 1;
second_ < end_; second_ += dfa_alphabet_, ++curr_index_)
{
if (index_set_.find (curr_index_) != index_set_.end ())
{
continue;
}
// Some systems have memcmp in namespace std.
using namespace std;
if (memcmp (first_, second_, sizeof (std::size_t) *
dfa_alphabet_) == 0)
{
index_set_.insert (curr_index_);
lookup_ptr_[curr_index_] = new_index_;
}
}
if (lookup_ptr_[index_] == null_token)
{
lookup_ptr_[index_] = new_index_;
++new_index_;
}
}
if (!index_set_.empty ())
{
const std::size_t *front_ = &dfa_.front ();
size_t_vector new_dfa_ (front_, front_ + dfa_alphabet_);
typename index_set::iterator set_end_ =
index_set_.end ();
const std::size_t *ptr_ = front_ + dfa_alphabet_;
std::size_t *new_ptr_ = 0;
new_dfa_.resize (size_ - index_set_.size () * dfa_alphabet_, 0);
new_ptr_ = &new_dfa_.front () + dfa_alphabet_;
size_ /= dfa_alphabet_;
for (index_ = 1; index_ < size_; ++index_)
{
if (index_set_.find (index_) != set_end_)
{
ptr_ += dfa_alphabet_;
continue;
}
new_ptr_[end_state_index] = ptr_[end_state_index];
new_ptr_[id_index] = ptr_[id_index];
new_ptr_[unique_id_index] = ptr_[unique_id_index];
new_ptr_[state_index] = ptr_[state_index];
new_ptr_[bol_index] = lookup_ptr_[ptr_[bol_index]];
new_ptr_[eol_index] = lookup_ptr_[ptr_[eol_index]];
new_ptr_ += dfa_offset;
ptr_ += dfa_offset;
for (std::size_t i_ = dfa_offset; i_ < dfa_alphabet_; ++i_)
{
*new_ptr_++ = lookup_ptr_[*ptr_++];
}
}
dfa_.swap (new_dfa_);
}
}
};
typedef basic_generator<char> generator;
typedef basic_generator<wchar_t> wgenerator;
}
}
#endif

View File

@@ -0,0 +1,529 @@
// input.hpp
// Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_INPUT
#define BOOST_LEXER_INPUT
#include "char_traits.hpp"
#include <boost/detail/iterator.hpp>
#include "size_t.hpp"
#include "state_machine.hpp"
namespace boost
{
namespace lexer
{
template<typename FwdIter, typename Traits =
char_traits<typename boost::detail::iterator_traits<FwdIter>::value_type> >
class basic_input
{
public:
class iterator
{
public:
friend class basic_input;
struct data
{
std::size_t id;
std::size_t unique_id;
FwdIter start;
FwdIter end;
bool bol;
std::size_t state;
// Construct in end() state.
data () :
id (0),
unique_id (npos),
bol (false),
state (npos)
{
}
bool operator == (const data &rhs_) const
{
return id == rhs_.id && unique_id == rhs_.unique_id &&
start == rhs_.start && end == rhs_.end &&
bol == rhs_.bol && state == rhs_.state;
}
};
iterator () :
_input (0)
{
}
bool operator == (const iterator &rhs_) const
{
return _data == rhs_._data;
}
bool operator != (const iterator &rhs_) const
{
return !(*this == rhs_);
}
data &operator * ()
{
return _data;
}
data *operator -> ()
{
return &_data;
}
// Let compiler generate operator = ().
// prefix version
iterator &operator ++ ()
{
next_token ();
return *this;
}
// postfix version
iterator operator ++ (int)
{
iterator iter_ = *this;
next_token ();
return iter_;
}
private:
// Not owner (obviously!)
const basic_input *_input;
data _data;
void next_token ()
{
const detail::internals &internals_ =
_input->_state_machine->data ();
_data.start = _data.end;
if (internals_._dfa->size () == 1)
{
if (internals_._seen_BOL_assertion ||
internals_._seen_EOL_assertion)
{
_data.id = next
(&internals_._lookup->front ()->front (),
internals_._dfa_alphabet.front (),
&internals_._dfa->front ()->front (),
_data.bol, _data.end, _input->_end, _data.unique_id);
}
else
{
_data.id = next (&internals_._lookup->front ()->front (),
internals_._dfa_alphabet.front (), &internals_.
_dfa->front ()->front (), _data.end, _input->_end,
_data.unique_id);
}
}
else
{
if (internals_._seen_BOL_assertion ||
internals_._seen_EOL_assertion)
{
_data.id = next (internals_, _data.state,
_data.bol, _data.end, _input->_end, _data.unique_id);
}
else
{
_data.id = next (internals_, _data.state,
_data.end, _input->_end, _data.unique_id);
}
}
if (_data.end == _input->_end && _data.start == _data.end)
{
// Ensure current state matches that returned by end().
_data.state = npos;
}
}
std::size_t next (const detail::internals &internals_,
std::size_t &start_state_, bool bol_,
FwdIter &start_token_, const FwdIter &end_,
std::size_t &unique_id_)
{
if (start_token_ == end_)
{
unique_id_ = npos;
return 0;
}
again:
const std::size_t * lookup_ = &internals_._lookup[start_state_]->
front ();
std::size_t dfa_alphabet_ = internals_._dfa_alphabet[start_state_];
const std::size_t *dfa_ = &internals_._dfa[start_state_]->front ();
const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
FwdIter curr_ = start_token_;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
std::size_t end_start_state_ = start_state_;
bool end_bol_ = bol_;
FwdIter end_token_ = start_token_;
while (curr_ != end_)
{
const std::size_t BOL_state_ = ptr_[bol_index];
const std::size_t EOL_state_ = ptr_[eol_index];
if (BOL_state_ && bol_)
{
ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
}
else if (EOL_state_ && *curr_ == '\n')
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
}
else
{
typename Traits::char_type prev_char_ = *curr_++;
bol_ = prev_char_ == '\n';
const std::size_t state_ =
ptr_[lookup_[static_cast<typename Traits::index_type>
(prev_char_)]];
if (state_ == 0)
{
break;
}
ptr_ = &dfa_[state_ * dfa_alphabet_];
}
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_start_state_ = *(ptr_ + state_index);
end_bol_ = bol_;
end_token_ = curr_;
}
}
const std::size_t EOL_state_ = ptr_[eol_index];
if (EOL_state_ && curr_ == end_)
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_start_state_ = *(ptr_ + state_index);
end_bol_ = bol_;
end_token_ = curr_;
}
}
if (end_state_)
{
// return longest match
start_state_ = end_start_state_;
start_token_ = end_token_;
if (id_ == 0)
{
bol_ = end_bol_;
goto again;
}
else
{
_data.bol = end_bol_;
}
}
else
{
// No match causes char to be skipped
_data.bol = *start_token_ == '\n';
++start_token_;
id_ = npos;
uid_ = npos;
}
unique_id_ = uid_;
return id_;
}
std::size_t next (const detail::internals &internals_,
std::size_t &start_state_, FwdIter &start_token_,
FwdIter const &end_, std::size_t &unique_id_)
{
if (start_token_ == end_)
{
unique_id_ = npos;
return 0;
}
again:
const std::size_t * lookup_ = &internals_._lookup[start_state_]->
front ();
std::size_t dfa_alphabet_ = internals_._dfa_alphabet[start_state_];
const std::size_t *dfa_ = &internals_._dfa[start_state_]->front ();
const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
FwdIter curr_ = start_token_;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
std::size_t end_start_state_ = start_state_;
FwdIter end_token_ = start_token_;
while (curr_ != end_)
{
const std::size_t state_ = ptr_[lookup_[static_cast
<typename Traits::index_type>(*curr_++)]];
if (state_ == 0)
{
break;
}
ptr_ = &dfa_[state_ * dfa_alphabet_];
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_start_state_ = *(ptr_ + state_index);
end_token_ = curr_;
}
}
if (end_state_)
{
// return longest match
start_state_ = end_start_state_;
start_token_ = end_token_;
if (id_ == 0) goto again;
}
else
{
// No match causes char to be skipped
++start_token_;
id_ = npos;
uid_ = npos;
}
unique_id_ = uid_;
return id_;
}
std::size_t next (const std::size_t * const lookup_,
const std::size_t dfa_alphabet_, const std::size_t * const dfa_,
bool bol_, FwdIter &start_token_, FwdIter const &end_,
std::size_t &unique_id_)
{
if (start_token_ == end_)
{
unique_id_ = npos;
return 0;
}
const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
FwdIter curr_ = start_token_;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
bool end_bol_ = bol_;
FwdIter end_token_ = start_token_;
while (curr_ != end_)
{
const std::size_t BOL_state_ = ptr_[bol_index];
const std::size_t EOL_state_ = ptr_[eol_index];
if (BOL_state_ && bol_)
{
ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
}
else if (EOL_state_ && *curr_ == '\n')
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
}
else
{
typename Traits::char_type prev_char_ = *curr_++;
bol_ = prev_char_ == '\n';
const std::size_t state_ =
ptr_[lookup_[static_cast<typename Traits::index_type>
(prev_char_)]];
if (state_ == 0)
{
break;
}
ptr_ = &dfa_[state_ * dfa_alphabet_];
}
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_bol_ = bol_;
end_token_ = curr_;
}
}
const std::size_t EOL_state_ = ptr_[eol_index];
if (EOL_state_ && curr_ == end_)
{
ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_bol_ = bol_;
end_token_ = curr_;
}
}
if (end_state_)
{
// return longest match
_data.bol = end_bol_;
start_token_ = end_token_;
}
else
{
// No match causes char to be skipped
_data.bol = *start_token_ == '\n';
++start_token_;
id_ = npos;
uid_ = npos;
}
unique_id_ = uid_;
return id_;
}
std::size_t next (const std::size_t * const lookup_,
const std::size_t dfa_alphabet_, const std::size_t * const dfa_,
FwdIter &start_token_, FwdIter const &end_,
std::size_t &unique_id_)
{
if (start_token_ == end_)
{
unique_id_ = npos;
return 0;
}
const std::size_t *ptr_ = dfa_ + dfa_alphabet_;
FwdIter curr_ = start_token_;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
FwdIter end_token_ = start_token_;
while (curr_ != end_)
{
const std::size_t state_ = ptr_[lookup_[static_cast
<typename Traits::index_type>(*curr_++)]];
if (state_ == 0)
{
break;
}
ptr_ = &dfa_[state_ * dfa_alphabet_];
if (*ptr_)
{
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
end_token_ = curr_;
}
}
if (end_state_)
{
// return longest match
start_token_ = end_token_;
}
else
{
// No match causes char to be skipped
++start_token_;
id_ = npos;
uid_ = npos;
}
unique_id_ = uid_;
return id_;
}
};
friend class iterator;
// Make it explict that we are NOT taking a copy of state_machine_!
basic_input (const basic_state_machine<typename Traits::char_type>
*state_machine_, const FwdIter &begin_, const FwdIter &end_) :
_state_machine (state_machine_),
_begin (begin_),
_end (end_)
{
}
iterator begin () const
{
iterator iter_;
iter_._input = this;
// Over-ride default of 0 (EOI)
iter_._data.id = npos;
iter_._data.start = _begin;
iter_._data.end = _begin;
iter_._data.bol = _state_machine->data ()._seen_BOL_assertion;
iter_._data.state = 0;
++iter_;
return iter_;
}
iterator end () const
{
iterator iter_;
iter_._input = this;
iter_._data.start = _end;
iter_._data.end = _end;
return iter_;
}
private:
const basic_state_machine<typename Traits::char_type> *_state_machine;
FwdIter _begin;
FwdIter _end;
};
typedef basic_input<std::string::iterator> iter_input;
typedef basic_input<std::basic_string<wchar_t>::iterator> iter_winput;
typedef basic_input<const char *> ptr_input;
typedef basic_input<const wchar_t *> ptr_winput;
}
}
#endif

View File

@@ -0,0 +1,60 @@
// internals.hpp
// Copyright (c) 2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_INTERNALS_HPP
#define BOOST_LEXER_INTERNALS_HPP
#include "containers/ptr_vector.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
struct internals
{
typedef std::vector<std::size_t> size_t_vector;
typedef ptr_vector<size_t_vector> size_t_vector_vector;
size_t_vector_vector _lookup;
size_t_vector _dfa_alphabet;
size_t_vector_vector _dfa;
bool _seen_BOL_assertion;
bool _seen_EOL_assertion;
internals () :
_seen_BOL_assertion (false),
_seen_EOL_assertion (false)
{
}
void clear ()
{
_lookup.clear ();
_dfa_alphabet.clear ();
_dfa.clear ();
_seen_BOL_assertion = false;
_seen_EOL_assertion = false;
}
void swap (internals &internals_)
{
_lookup->swap (*internals_._lookup);
_dfa_alphabet.swap (internals_._dfa_alphabet);
_dfa->swap (*internals_._dfa);
std::swap (_seen_BOL_assertion, internals_._seen_BOL_assertion);
std::swap (_seen_EOL_assertion, internals_._seen_EOL_assertion);
}
private:
internals (const internals &); // No copy construction.
internals &operator = (const internals &); // No assignment.
};
}
}
}
#endif

View File

@@ -0,0 +1,511 @@
// parser.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_PARSER_HPP
#define BOOST_LEXER_PARSER_HPP
#include <boost/assert.hpp>
#include "tree/end_node.hpp"
#include "tree/iteration_node.hpp"
#include "tree/leaf_node.hpp"
#include "../runtime_error.hpp"
#include "tree/selection_node.hpp"
#include "tree/sequence_node.hpp"
#include "../size_t.hpp"
#include "tokeniser/re_tokeniser.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT>
class basic_parser
{
public:
typedef basic_re_tokeniser<CharT> tokeniser;
typedef typename tokeniser::string string;
typedef std::map<string, const node *> macro_map;
typedef node::node_ptr_vector node_ptr_vector;
typedef typename tokeniser::num_token token;
/*
General principles of regex parsing:
- Every regex is a sequence of sub-regexes.
- Regexes consist of operands and operators
- All operators decompose to sequence, selection ('|') and iteration ('*')
- Regex tokens are stored on the stack.
- When a complete sequence of regex tokens is on the stack it is processed.
Grammar:
<REGEX> -> <OREXP>
<OREXP> -> <SEQUENCE> | <OREXP>'|'<SEQUENCE>
<SEQUENCE> -> <SUB>
<SUB> -> <EXPRESSION> | <SUB><EXPRESSION>
<EXPRESSION> -> <REPEAT>
<REPEAT> -> charset | macro | '('<REGEX>')' | <REPEAT><DUPLICATE>
<DUPLICATE> -> '?' | '*' | '+' | '{n[,[m]]}'
*/
static node *parse (const CharT *start_, const CharT * const end_,
const std::size_t id_, const std::size_t unique_id_,
const std::size_t dfa_state_, const regex_flags flags_,
const std::locale &locale_, node_ptr_vector &node_ptr_vector_,
const macro_map &macromap_, typename tokeniser::token_map &map_,
bool &seen_BOL_assertion_, bool &seen_EOL_assertion_)
{
node *root_ = 0;
state state_ (start_, end_, flags_, locale_);
token lhs_token_;
token rhs_token_;
token_stack token_stack_;
tree_node_stack tree_node_stack_;
char action_ = 0;
token_stack_.push (rhs_token_);
tokeniser::next (state_, map_, rhs_token_);
do
{
lhs_token_ = token_stack_.top ();
action_ = lhs_token_.precedence (rhs_token_._type);
switch (action_)
{
case '<':
case '=':
token_stack_.push (rhs_token_);
tokeniser::next (state_, map_, rhs_token_);
break;
case '>':
reduce (token_stack_, macromap_, node_ptr_vector_,
tree_node_stack_);
break;
default:
std::ostringstream ss_;
ss_ << "A syntax error occurred: '" <<
lhs_token_.precedence_string () <<
"' against '" << rhs_token_.precedence_string () <<
"' at index " << state_.index () << ".";
throw runtime_error (ss_.str ().c_str ());
break;
}
} while (!token_stack_.empty ());
if (tree_node_stack_.empty ())
{
throw runtime_error ("Empty rules are not allowed.");
}
BOOST_ASSERT(tree_node_stack_.size () == 1);
node *lhs_node_ = tree_node_stack_.top ();
tree_node_stack_.pop ();
if (id_ == 0)
{
// Macros have no end state...
root_ = lhs_node_;
}
else
{
node_ptr_vector_->push_back (static_cast<end_node *>(0));
node *rhs_node_ = new end_node (id_, unique_id_, dfa_state_);
node_ptr_vector_->back () = rhs_node_;
node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node
(lhs_node_, rhs_node_);
root_ = node_ptr_vector_->back ();
}
// Done this way as bug in VC++ 6 prevents |= operator working
// properly!
if (state_._seen_BOL_assertion) seen_BOL_assertion_ = true;
if (state_._seen_EOL_assertion) seen_EOL_assertion_ = true;
return root_;
}
private:
typedef typename tokeniser::state state;
typedef std::stack<token> token_stack;
typedef node::node_stack tree_node_stack;
static void reduce (token_stack &token_stack_,
const macro_map &macromap_, node_ptr_vector &node_vector_ptr_,
tree_node_stack &tree_node_stack_)
{
typename tokeniser::num_token lhs_;
typename tokeniser::num_token rhs_;
token_stack handle_;
char action_ = 0;
do
{
rhs_ = token_stack_.top ();
token_stack_.pop ();
handle_.push (rhs_);
if (!token_stack_.empty ())
{
lhs_ = token_stack_.top ();
action_ = lhs_.precedence (rhs_._type);
}
} while (!token_stack_.empty () && action_ == '=');
BOOST_ASSERT(token_stack_.empty () || action_ == '<');
switch (rhs_._type)
{
case token::BEGIN:
// finished processing so exit
break;
case token::REGEX:
// finished parsing, nothing to do
break;
case token::OREXP:
orexp (handle_, token_stack_, node_vector_ptr_, tree_node_stack_);
break;
case token::SEQUENCE:
token_stack_.push (token::OREXP);
break;
case token::SUB:
sub (handle_, token_stack_, node_vector_ptr_, tree_node_stack_);
break;
case token::EXPRESSION:
token_stack_.push (token::SUB);
break;
case token::REPEAT:
repeat (handle_, token_stack_);
break;
case token::CHARSET:
charset (handle_, token_stack_, node_vector_ptr_,
tree_node_stack_);
break;
case token::MACRO:
macro (handle_, token_stack_, macromap_, node_vector_ptr_,
tree_node_stack_);
break;
case token::OPENPAREN:
openparen (handle_, token_stack_);
break;
case token::OPT:
case token::AOPT:
optional (rhs_._type == token::OPT, node_vector_ptr_,
tree_node_stack_);
token_stack_.push (token::DUP);
break;
case token::ZEROORMORE:
case token::AZEROORMORE:
zero_or_more (rhs_._type == token::ZEROORMORE, node_vector_ptr_,
tree_node_stack_);
token_stack_.push (token::DUP);
break;
case token::ONEORMORE:
case token::AONEORMORE:
one_or_more (rhs_._type == token::ONEORMORE, node_vector_ptr_,
tree_node_stack_);
token_stack_.push (token::DUP);
break;
case token::REPEATN:
case token::AREPEATN:
repeatn (rhs_._type == token::REPEATN, handle_.top (),
node_vector_ptr_, tree_node_stack_);
token_stack_.push (token::DUP);
break;
default:
throw runtime_error
("Internal error regex_parser::reduce");
break;
}
}
static void orexp (token_stack &handle_, token_stack &token_stack_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
BOOST_ASSERT(handle_.top ()._type == token::OREXP &&
(handle_.size () == 1 || handle_.size () == 3));
if (handle_.size () == 1)
{
token_stack_.push (token::REGEX);
}
else
{
handle_.pop ();
BOOST_ASSERT(handle_.top ()._type == token::OR);
handle_.pop ();
BOOST_ASSERT(handle_.top ()._type == token::SEQUENCE);
perform_or (node_ptr_vector_, tree_node_stack_);
token_stack_.push (token::OREXP);
}
}
static void sub (token_stack &handle_, token_stack &token_stack_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
BOOST_ASSERT(handle_.top ()._type == token::SUB &&
(handle_.size () == 1 || handle_.size () == 2));
if (handle_.size () == 1)
{
token_stack_.push (token::SEQUENCE);
}
else
{
handle_.pop ();
BOOST_ASSERT(handle_.top ()._type == token::EXPRESSION);
// perform join
sequence (node_ptr_vector_, tree_node_stack_);
token_stack_.push (token::SUB);
}
}
static void repeat (token_stack &handle_, token_stack &token_stack_)
{
BOOST_ASSERT(handle_.top ()._type == token::REPEAT &&
handle_.size () >= 1 && handle_.size () <= 3);
if (handle_.size () == 1)
{
token_stack_.push (token::EXPRESSION);
}
else
{
handle_.pop ();
BOOST_ASSERT(handle_.top ()._type == token::DUP);
token_stack_.push (token::REPEAT);
}
}
static void charset (token_stack &handle_, token_stack &token_stack_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
BOOST_ASSERT(handle_.top ()._type == token::CHARSET &&
handle_.size () == 1);
// store charset
node_ptr_vector_->push_back (static_cast<leaf_node *>(0));
const size_t id_ = handle_.top ()._id;
node_ptr_vector_->back () = new leaf_node (id_, true);
tree_node_stack_.push (node_ptr_vector_->back ());
token_stack_.push (token::REPEAT);
}
static void macro (token_stack &handle_, token_stack &token_stack_,
const macro_map &macromap_, node_ptr_vector &node_ptr_vector_,
tree_node_stack &tree_node_stack_)
{
token &top_ = handle_.top ();
BOOST_ASSERT(top_._type == token::MACRO && handle_.size () == 1);
typename macro_map::const_iterator iter_ =
macromap_.find (top_._macro);
if (iter_ == macromap_.end ())
{
const CharT *name_ = top_._macro;
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Unknown MACRO name '";
while (*name_)
{
os_ << ss_.narrow (*name_++, ' ');
}
os_ << "'.";
throw runtime_error (os_.str ());
}
tree_node_stack_.push (iter_->second->copy (node_ptr_vector_));
token_stack_.push (token::REPEAT);
}
static void openparen (token_stack &handle_, token_stack &token_stack_)
{
BOOST_ASSERT(handle_.top ()._type == token::OPENPAREN &&
handle_.size () == 3);
handle_.pop ();
BOOST_ASSERT(handle_.top ()._type == token::REGEX);
handle_.pop ();
BOOST_ASSERT(handle_.top ()._type == token::CLOSEPAREN);
token_stack_.push (token::REPEAT);
}
static void perform_or (node_ptr_vector &node_ptr_vector_,
tree_node_stack &tree_node_stack_)
{
// perform or
node *rhs_ = tree_node_stack_.top ();
tree_node_stack_.pop ();
node *lhs_ = tree_node_stack_.top ();
node_ptr_vector_->push_back (static_cast<selection_node *>(0));
node_ptr_vector_->back () = new selection_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
static void sequence (node_ptr_vector &node_ptr_vector_,
tree_node_stack &tree_node_stack_)
{
node *rhs_ = tree_node_stack_.top ();
tree_node_stack_.pop ();
node *lhs_ = tree_node_stack_.top ();
node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
static void optional (const bool greedy_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
// perform ?
node *lhs_ = tree_node_stack_.top ();
// You don't know if lhs_ is a leaf_node, so get firstpos.
node::node_vector &firstpos_ = lhs_->firstpos ();
for (node::node_vector::iterator iter_ = firstpos_.begin (),
end_ = firstpos_.end (); iter_ != end_; ++iter_)
{
// These are leaf_nodes!
(*iter_)->greedy (greedy_);
}
node_ptr_vector_->push_back (static_cast<leaf_node *>(0));
node *rhs_ = new leaf_node (null_token, greedy_);
node_ptr_vector_->back () = rhs_;
node_ptr_vector_->push_back (static_cast<selection_node *>(0));
node_ptr_vector_->back () = new selection_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
static void zero_or_more (const bool greedy_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
// perform *
node *ptr_ = tree_node_stack_.top ();
node_ptr_vector_->push_back (static_cast<iteration_node *>(0));
node_ptr_vector_->back () = new iteration_node (ptr_, greedy_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
static void one_or_more (const bool greedy_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
// perform +
node *lhs_ = tree_node_stack_.top ();
node *copy_ = lhs_->copy (node_ptr_vector_);
node_ptr_vector_->push_back (static_cast<iteration_node *>(0));
node *rhs_ = new iteration_node (copy_, greedy_);
node_ptr_vector_->back () = rhs_;
node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
// This is one of the most mind bending routines in this code...
static void repeatn (const bool greedy_, const token &token_,
node_ptr_vector &node_ptr_vector_, tree_node_stack &tree_node_stack_)
{
// perform {n[,[m]]}
// Semantic checks have already been performed.
// {0,} = *
// {0,1} = ?
// {1,} = +
// therefore we do not check for these cases.
if (!(token_._min == 1 && !token_._comma))
{
const std::size_t top_ = token_._min > 0 ?
token_._min : token_._max;
if (token_._min == 0)
{
optional (greedy_, node_ptr_vector_, tree_node_stack_);
}
node *prev_ = tree_node_stack_.top ()->copy (node_ptr_vector_);
node *curr_ = 0;
for (std::size_t i_ = 2; i_ < top_; ++i_)
{
curr_ = prev_->copy (node_ptr_vector_);
tree_node_stack_.push (static_cast<node *>(0));
tree_node_stack_.top () = prev_;
sequence (node_ptr_vector_, tree_node_stack_);
prev_ = curr_;
}
if (token_._comma && token_._min > 0)
{
if (token_._min > 1)
{
curr_ = prev_->copy (node_ptr_vector_);
tree_node_stack_.push (static_cast<node *>(0));
tree_node_stack_.top () = prev_;
sequence (node_ptr_vector_, tree_node_stack_);
prev_ = curr_;
}
if (token_._comma && token_._max)
{
tree_node_stack_.push (static_cast<node *>(0));
tree_node_stack_.top () = prev_;
optional (greedy_, node_ptr_vector_, tree_node_stack_);
prev_ = tree_node_stack_.top ();
tree_node_stack_.pop ();
const std::size_t count_ = token_._max - token_._min;
for (std::size_t i_ = 1; i_ < count_; ++i_)
{
curr_ = prev_->copy (node_ptr_vector_);
tree_node_stack_.push (static_cast<node *>(0));
tree_node_stack_.top () = prev_;
sequence (node_ptr_vector_, tree_node_stack_);
prev_ = curr_;
}
}
else
{
tree_node_stack_.push (static_cast<node *>(0));
tree_node_stack_.top () = prev_;
zero_or_more (greedy_, node_ptr_vector_, tree_node_stack_);
prev_ = tree_node_stack_.top ();
tree_node_stack_.pop ();
}
}
tree_node_stack_.push (static_cast<node *>(0));
tree_node_stack_.top () = prev_;
sequence (node_ptr_vector_, tree_node_stack_);
}
}
};
}
}
}
#endif

View File

@@ -0,0 +1,146 @@
// num_token.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_NUM_TOKEN_HPP
#define BOOST_LEXER_NUM_TOKEN_HPP
#include <boost/config.hpp>
#include "../../consts.hpp" // null_token
#include "../../size_t.hpp"
#include <boost/detail/workaround.hpp>
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT>
struct basic_num_token
{
enum type {BEGIN, REGEX, OREXP, SEQUENCE, SUB, EXPRESSION, REPEAT,
DUP, OR, CHARSET, MACRO, OPENPAREN, CLOSEPAREN, OPT, AOPT,
ZEROORMORE, AZEROORMORE, ONEORMORE, AONEORMORE, REPEATN, AREPEATN,
END};
type _type;
std::size_t _id;
std::size_t _min;
bool _comma;
std::size_t _max;
CharT _macro[max_macro_len + 1];
static const char _precedence_table[END + 1][END + 1];
static const char *_precedence_strings[END + 1];
basic_num_token (const type type_ = BEGIN,
const std::size_t id_ = null_token) :
_type (type_),
_id (id_),
_min (0),
_comma (false),
_max (0)
{
*_macro = 0;
}
basic_num_token &operator = (const basic_num_token &rhs_)
{
_type = rhs_._type;
_id = rhs_._id;
_min = rhs_._min;
_comma = rhs_._comma;
_max = rhs_._max;
if (_type == MACRO)
{
const CharT *read_ = rhs_._macro;
CharT *write_ = _macro;
while (*read_)
{
*write_++ = *read_++;
}
*write_ = 0;
}
return *this;
}
void set (const type type_)
{
_type = type_;
_id = null_token;
}
void set (const type type_, const std::size_t id_)
{
_type = type_;
_id = id_;
}
void min_max (const std::size_t min_, const bool comma_,
const std::size_t max_)
{
_min = min_;
_comma = comma_;
_max = max_;
}
char precedence (const type type_) const
{
return _precedence_table[_type][type_];
}
const char *precedence_string () const
{
return _precedence_strings[_type];
}
};
template<typename CharT>
const char basic_num_token<CharT>::_precedence_table[END + 1][END + 1] = {
// BEG, REG, ORE, SEQ, SUB, EXP, RPT, DUP, | , CHR, MCR, ( , ) , ? , ?? , * , *? , + , +?, {n}?, {n}, END
/*BEGIN*/{' ', '<', '<', '<', '<', '<', '<', ' ', ' ', '<', '<', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/*REGEX*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '=', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/*OREXP*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '=', '>', '>', ' ', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* SEQ */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', ' ', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* SUB */{' ', ' ', ' ', ' ', ' ', '=', '<', ' ', '>', '<', '<', '<', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/*EXPRE*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* RPT */{' ', ' ', ' ', ' ', ' ', ' ', ' ', '=', '>', '>', '>', '>', '>', '<', '<', '<', '<', '<', '<', '<', '<', '>'},
/*DUPLI*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* | */{' ', ' ', ' ', '=', '<', '<', '<', ' ', ' ', '<', '<', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '},
/*CHARA*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>'},
/*MACRO*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>'},
/* ( */{' ', '=', '<', '<', '<', '<', '<', ' ', ' ', '<', '<', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '},
/* ) */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>', '>'},
/* ? */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* ?? */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* * */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* *? */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* + */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* +? */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/*{n,m}*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', '<', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/*{nm}?*/{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>', '>', '>', '>', '>', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '>'},
/* END */{' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '}
};
template<typename CharT>
const char *basic_num_token<CharT>::_precedence_strings[END + 1] =
#if BOOST_WORKAROUND(BOOST_INTEL_CXX_VERSION, BOOST_TESTED_AT(910))
{{"BEGIN"}, {"REGEX"}, {"OREXP"}, {"SEQUENCE"}, {"SUB"}, {"EXPRESSION"},
{"REPEAT"}, {"DUPLICATE"}, {"|"}, {"CHARSET"}, {"MACRO"},
{"("}, {")"}, {"?"}, {"??"}, {"*"}, {"*?"}, {"+"}, {"+?"}, {"{n[,[m]]}"},
{"{n[,[m]]}?"}, {"END"}};
#else
{"BEGIN", "REGEX", "OREXP", "SEQUENCE", "SUB", "EXPRESSION", "REPEAT",
"DUPLICATE", "|", "CHARSET", "MACRO", "(", ")", "?", "??", "*", "*?",
"+", "+?", "{n[,[m]]}", "{n[,[m]]}?", "END"};
#endif
}
}
}
#endif

View File

@@ -0,0 +1,574 @@
// tokeniser.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_RE_TOKENISER_HPP
#define BOOST_LEXER_RE_TOKENISER_HPP
// memcpy()
#include <cstring>
#include <map>
#include "num_token.hpp"
#include "../../runtime_error.hpp"
#include "../../size_t.hpp"
#include <sstream>
#include "../../string_token.hpp"
#include "re_tokeniser_helper.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT>
class basic_re_tokeniser
{
public:
typedef basic_num_token<CharT> num_token;
typedef basic_re_tokeniser_state<CharT> state;
typedef basic_string_token<CharT> string_token;
typedef typename string_token::string string;
typedef std::map<string_token, std::size_t> token_map;
typedef std::pair<string_token, std::size_t> token_pair;
static void next (state &state_, token_map &map_, num_token &token_)
{
CharT ch_ = 0;
bool eos_ = state_.next (ch_);
token_.min_max (0, false, 0);
while (!eos_ && ch_ == '"')
{
state_._in_string ^= 1;
eos_ = state_.next (ch_);
}
if (eos_)
{
if (state_._in_string)
{
throw runtime_error ("Unexpected end of regex "
"(missing '\"').");
}
if (state_._paren_count)
{
throw runtime_error ("Unexpected end of regex "
"(missing ')').");
}
token_.set (num_token::END, null_token);
}
else
{
if (ch_ == '\\')
{
// Even if we are in a string, respect escape sequences...
escape (state_, map_, token_);
}
else if (state_._in_string)
{
// All other meta characters lose their special meaning
// inside a string.
create_charset_token (string (1, ch_), false, map_, token_);
}
else
{
// Not an escape sequence and not inside a string, so
// check for meta characters.
switch (ch_)
{
case '(':
token_.set (num_token::OPENPAREN, null_token);
++state_._paren_count;
read_options (state_);
break;
case ')':
--state_._paren_count;
if (state_._paren_count < 0)
{
std::ostringstream ss_;
ss_ << "Number of open parenthesis < 0 at index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
token_.set (num_token::CLOSEPAREN, null_token);
if (!state_._flags_stack.empty ())
{
state_._flags = state_._flags_stack.top ();
state_._flags_stack.pop ();
}
break;
case '?':
if (!state_.eos () && *state_._curr == '?')
{
token_.set (num_token::AOPT, null_token);
state_.increment ();
}
else
{
token_.set (num_token::OPT, null_token);
}
break;
case '*':
if (!state_.eos () && *state_._curr == '?')
{
token_.set (num_token::AZEROORMORE, null_token);
state_.increment ();
}
else
{
token_.set (num_token::ZEROORMORE, null_token);
}
break;
case '+':
if (!state_.eos () && *state_._curr == '?')
{
token_.set (num_token::AONEORMORE, null_token);
state_.increment ();
}
else
{
token_.set (num_token::ONEORMORE, null_token);
}
break;
case '{':
open_curly (state_, token_);
break;
case '|':
token_.set (num_token::OR, null_token);
break;
case '^':
if (state_._curr - 1 == state_._start)
{
token_.set (num_token::CHARSET, bol_token);
state_._seen_BOL_assertion = true;
}
else
{
create_charset_token (string (1, ch_), false,
map_, token_);
}
break;
case '$':
if (state_._curr == state_._end)
{
token_.set (num_token::CHARSET, eol_token);
state_._seen_EOL_assertion = true;
}
else
{
create_charset_token (string (1, ch_), false,
map_, token_);
}
break;
case '.':
{
string dot_;
if (state_._flags & dot_not_newline)
{
dot_ = '\n';
}
create_charset_token (dot_, true, map_, token_);
break;
}
case '[':
{
charset (state_, map_, token_);
break;
}
case '/':
throw runtime_error("Lookahead ('/') is not supported yet.");
break;
default:
if ((state_._flags & icase) &&
(std::isupper (ch_, state_._locale) ||
std::islower (ch_, state_._locale)))
{
CharT upper_ = std::toupper (ch_, state_._locale);
CharT lower_ = std::tolower (ch_, state_._locale);
string str_ (1, upper_);
str_ += lower_;
create_charset_token (str_, false, map_, token_);
}
else
{
create_charset_token (string (1, ch_), false,
map_, token_);
}
break;
}
}
}
}
private:
typedef basic_re_tokeniser_helper<CharT> tokeniser_helper;
static void read_options (state &state_)
{
if (!state_.eos () && *state_._curr == '?')
{
CharT ch_ = 0;
bool eos_ = false;
bool negate_ = false;
state_.increment ();
eos_ = state_.next (ch_);
state_._flags_stack.push (state_._flags);
while (!eos_ && ch_ != ':')
{
switch (ch_)
{
case '-':
negate_ ^= 1;
break;
case 'i':
if (negate_)
{
state_._flags = static_cast<regex_flags>
(state_._flags & ~icase);
}
else
{
state_._flags = static_cast<regex_flags>
(state_._flags | icase);
}
negate_ = false;
break;
case 's':
if (negate_)
{
state_._flags = static_cast<regex_flags>
(state_._flags | dot_not_newline);
}
else
{
state_._flags = static_cast<regex_flags>
(state_._flags & ~dot_not_newline);
}
negate_ = false;
break;
default:
{
std::ostringstream ss_;
ss_ << "Unknown option at index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
}
eos_ = state_.next (ch_);
}
// End of string handler will handle early termination
}
else if (!state_._flags_stack.empty ())
{
state_._flags_stack.push (state_._flags);
}
}
static void escape (state &state_, token_map &map_, num_token &token_)
{
CharT ch_ = 0;
std::size_t str_len_ = 0;
const CharT *str_ = tokeniser_helper::escape_sequence (state_,
ch_, str_len_);
if (str_)
{
state state2_ (str_ + 1, str_ + str_len_, state_._flags,
state_._locale);
charset (state2_, map_, token_);
}
else
{
create_charset_token (string (1, ch_), false, map_, token_);
}
}
static void charset (state &state_, token_map &map_, num_token &token_)
{
string chars_;
bool negated_ = false;
tokeniser_helper::charset (state_, chars_, negated_);
create_charset_token (chars_, negated_, map_, token_);
}
static void create_charset_token (const string &charset_,
const bool negated_, token_map &map_, num_token &token_)
{
std::size_t id_ = null_token;
string_token stok_ (negated_, charset_);
stok_.remove_duplicates ();
stok_.normalise ();
typename token_map::const_iterator iter_ = map_.find (stok_);
if (iter_ == map_.end ())
{
id_ = map_.size ();
map_.insert (token_pair (stok_, id_));
}
else
{
id_ = iter_->second;
}
token_.set (num_token::CHARSET, id_);
}
static void open_curly (state &state_, num_token &token_)
{
if (state_.eos ())
{
throw runtime_error ("Unexpected end of regex "
"(missing '}').");
}
else if (*state_._curr >= '0' && *state_._curr <= '9')
{
repeat_n (state_, token_);
if (!state_.eos () && *state_._curr == '?')
{
token_._type = num_token::AREPEATN;
state_.increment ();
}
}
else
{
macro (state_, token_);
}
}
// SYNTAX:
// {n[,[n]]}
// SEMANTIC RULES:
// {0} - INVALID (throw exception)
// {0,} = *
// {0,0} - INVALID (throw exception)
// {0,1} = ?
// {1,} = +
// {min,max} where min == max - {min}
// {min,max} where max < min - INVALID (throw exception)
static void repeat_n (state &state_, num_token &token_)
{
CharT ch_ = 0;
bool eos_ = state_.next (ch_);
while (!eos_ && ch_ >= '0' && ch_ <= '9')
{
token_._min *= 10;
token_._min += ch_ - '0';
eos_ = state_.next (ch_);
}
if (eos_)
{
throw runtime_error ("Unexpected end of regex "
"(missing '}').");
}
bool min_max_ = false;
bool repeatn_ = true;
token_._comma = ch_ == ',';
if (token_._comma)
{
eos_ = state_.next (ch_);
if (eos_)
{
throw runtime_error ("Unexpected end of regex "
"(missing '}').");
}
if (ch_ == '}')
{
// Small optimisation: Check for '*' equivalency.
if (token_._min == 0)
{
token_.set (num_token::ZEROORMORE, null_token);
repeatn_ = false;
}
// Small optimisation: Check for '+' equivalency.
else if (token_._min == 1)
{
token_.set (num_token::ONEORMORE, null_token);
repeatn_ = false;
}
}
else
{
if (ch_ < '0' || ch_ > '9')
{
std::ostringstream ss_;
ss_ << "Missing '}' at index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
min_max_ = true;
do
{
token_._max *= 10;
token_._max += ch_ - '0';
eos_ = state_.next (ch_);
} while (!eos_ && ch_ >= '0' && ch_ <= '9');
if (eos_)
{
throw runtime_error ("Unexpected end of regex "
"(missing '}').");
}
// Small optimisation: Check for '?' equivalency.
if (token_._min == 0 && token_._max == 1)
{
token_.set (num_token::OPT, null_token);
repeatn_ = false;
}
// Small optimisation: if min == max, then min.
else if (token_._min == token_._max)
{
token_._comma = false;
min_max_ = false;
token_._max = 0;
}
}
}
if (ch_ != '}')
{
std::ostringstream ss_;
ss_ << "Missing '}' at index " << state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
if (repeatn_)
{
// SEMANTIC VALIDATION follows:
// NOTE: {0,} has already become *
// therefore we don't check for a comma.
if (token_._min == 0 && token_._max == 0)
{
std::ostringstream ss_;
ss_ << "Cannot have exactly zero repeats preceding index " <<
state_.index () << '.';
throw runtime_error (ss_.str ().c_str ());
}
if (min_max_ && token_._max < token_._min)
{
std::ostringstream ss_;
ss_ << "Max less than min preceding index " <<
state_.index () << '.';
throw runtime_error (ss_.str ().c_str ());
}
token_.set (num_token::REPEATN, null_token);
}
}
static void macro (state &state_, num_token &token_)
{
CharT ch_ = 0;
bool eos_ = false;
const CharT *start_ = state_._curr;
state_.next (ch_);
if (ch_ != '_' && !(ch_ >= 'A' && ch_ <= 'Z') &&
!(ch_ >= 'a' && ch_ <= 'z'))
{
std::ostringstream ss_;
ss_ << "Invalid MACRO name at index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
do
{
eos_ = state_.next (ch_);
if (eos_)
{
throw runtime_error ("Unexpected end of regex "
"(missing '}').");
}
} while (ch_ == '_' || ch_ == '-' || (ch_ >= 'A' && ch_ <= 'Z') ||
(ch_ >= 'a' && ch_ <= 'z') || (ch_ >= '0' && ch_ <= '9'));
if (ch_ != '}')
{
std::ostringstream ss_;
ss_ << "Missing '}' at index " << state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
std::size_t len_ = state_._curr - 1 - start_;
if (len_ > max_macro_len)
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "MACRO name '";
while (len_)
{
os_ << ss_.narrow (*start_++, ' ');
--len_;
}
os_ << "' too long.";
throw runtime_error (os_.str ());
}
token_.set (num_token::MACRO, null_token);
// Some systems have memcpy in namespace std.
using namespace std;
memcpy (token_._macro, start_, len_ * sizeof (CharT));
token_._macro[len_] = 0;
}
};
}
}
}
#endif

View File

@@ -0,0 +1,549 @@
// tokeniser_helper.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_RE_TOKENISER_HELPER_H
#define BOOST_LEXER_RE_TOKENISER_HELPER_H
#include "../../char_traits.hpp"
// strlen()
#include <cstring>
#include "../../size_t.hpp"
#include "re_tokeniser_state.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT, typename Traits = char_traits<CharT> >
class basic_re_tokeniser_helper
{
public:
typedef basic_re_tokeniser_state<CharT> state;
typedef std::basic_string<CharT> string;
static const CharT *escape_sequence (state &state_, CharT &ch_,
std::size_t &str_len_)
{
bool eos_ = state_.eos ();
if (eos_)
{
throw runtime_error ("Unexpected end of regex "
"following '\\'.");
}
const CharT *str_ = charset_shortcut (*state_._curr, str_len_);
if (str_)
{
state_.increment ();
}
else
{
ch_ = chr (state_);
}
return str_;
}
// This function can call itself.
static void charset (state &state_, string &chars_, bool &negated_)
{
CharT ch_ = 0;
bool eos_ = state_.next (ch_);
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex "
"following '['.");
}
negated_ = ch_ == '^';
if (negated_)
{
eos_ = state_.next (ch_);
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex "
"following '^'.");
}
}
bool chset_ = false;
CharT prev_ = 0;
while (ch_ != ']')
{
if (ch_ == '\\')
{
std::size_t str_len_ = 0;
const CharT *str_ = escape_sequence (state_, prev_, str_len_);
chset_ = str_ != 0;
if (chset_)
{
state temp_state_ (str_ + 1, str_ + str_len_,
state_._flags, state_._locale);
string temp_chars_;
bool temp_negated_ = false;
charset (temp_state_, temp_chars_, temp_negated_);
if (negated_ != temp_negated_)
{
std::ostringstream ss_;
ss_ << "Mismatch in charset negation preceding "
"index " << state_.index () << '.';
throw runtime_error (ss_.str ().c_str ());
}
chars_ += temp_chars_;
}
}
/*
else if (ch_ == '[' && !state_.eos () && *state_._curr == ':')
{
// TODO: POSIX charsets
}
*/
else
{
chset_ = false;
prev_ = ch_;
}
eos_ = state_.next (ch_);
// Covers preceding if, else if and else
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex "
"(missing ']').");
}
if (ch_ == '-')
{
charset_range (chset_, state_, eos_, ch_, prev_, chars_);
}
else if (!chset_)
{
if ((state_._flags & icase) &&
(std::isupper (prev_, state_._locale) ||
std::islower (prev_, state_._locale)))
{
CharT upper_ = std::toupper (prev_, state_._locale);
CharT lower_ = std::tolower (prev_, state_._locale);
chars_ += upper_;
chars_ += lower_;
}
else
{
chars_ += prev_;
}
}
}
if (!negated_ && chars_.empty ())
{
throw runtime_error ("Empty charsets not allowed.");
}
}
static CharT chr (state &state_)
{
CharT ch_ = 0;
// eos_ has already been checked for.
switch (*state_._curr)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
ch_ = decode_octal (state_);
break;
case 'a':
ch_ = '\a';
state_.increment ();
break;
case 'b':
ch_ = '\b';
state_.increment ();
break;
case 'c':
ch_ = decode_control_char (state_);
break;
case 'e':
ch_ = 27; // '\e' not recognised by compiler
state_.increment ();
break;
case 'f':
ch_ = '\f';
state_.increment ();
break;
case 'n':
ch_ = '\n';
state_.increment ();
break;
case 'r':
ch_ = '\r';
state_.increment ();
break;
case 't':
ch_ = '\t';
state_.increment ();
break;
case 'v':
ch_ = '\v';
state_.increment ();
break;
case 'x':
ch_ = decode_hex (state_);
break;
default:
ch_ = *state_._curr;
state_.increment ();
break;
}
return ch_;
}
private:
static const char *charset_shortcut (const char ch_,
std::size_t &str_len_)
{
const char *str_ = 0;
switch (ch_)
{
case 'd':
str_ = "[0-9]";
break;
case 'D':
str_ = "[^0-9]";
break;
case 's':
str_ = "[ \t\n\r\f\v]";
break;
case 'S':
str_ = "[^ \t\n\r\f\v]";
break;
case 'w':
str_ = "[_0-9A-Za-z]";
break;
case 'W':
str_ = "[^_0-9A-Za-z]";
break;
}
if (str_)
{
// Some systems have strlen in namespace std.
using namespace std;
str_len_ = strlen (str_);
}
else
{
str_len_ = 0;
}
return str_;
}
static const wchar_t *charset_shortcut (const wchar_t ch_,
std::size_t &str_len_)
{
const wchar_t *str_ = 0;
switch (ch_)
{
case 'd':
str_ = L"[0-9]";
break;
case 'D':
str_ = L"[^0-9]";
break;
case 's':
str_ = L"[ \t\n\r\f\v]";
break;
case 'S':
str_ = L"[^ \t\n\r\f\v]";
break;
case 'w':
str_ = L"[_0-9A-Za-z]";
break;
case 'W':
str_ = L"[^_0-9A-Za-z]";
break;
}
if (str_)
{
// Some systems have wcslen in namespace std.
using namespace std;
str_len_ = wcslen (str_);
}
else
{
str_len_ = 0;
}
return str_;
}
static CharT decode_octal (state &state_)
{
std::size_t accumulator_ = 0;
CharT ch_ = *state_._curr;
unsigned short count_ = 3;
bool eos_ = false;
for (;;)
{
accumulator_ *= 8;
accumulator_ += ch_ - '0';
--count_;
state_.increment ();
eos_ = state_.eos ();
if (!count_ || eos_) break;
ch_ = *state_._curr;
// Don't consume invalid chars!
if (ch_ < '0' || ch_ > '7')
{
break;
}
}
return static_cast<CharT> (accumulator_);
}
static CharT decode_control_char (state &state_)
{
// Skip over 'c'
state_.increment ();
CharT ch_ = 0;
bool eos_ = state_.next (ch_);
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex following \\c.");
}
else
{
if (ch_ >= 'a' && ch_ <= 'z')
{
ch_ -= 'a' - 1;
}
else if (ch_ >= 'A' && ch_ <= 'Z')
{
ch_ -= 'A' - 1;
}
else if (ch_ == '@')
{
// Apparently...
ch_ = 0;
}
else
{
std::ostringstream ss_;
ss_ << "Invalid control char at index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
}
return ch_;
}
static CharT decode_hex (state &state_)
{
// Skip over 'x'
state_.increment ();
CharT ch_ = 0;
bool eos_ = state_.next (ch_);
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex following \\x.");
}
if (!((ch_ >= '0' && ch_ <= '9') || (ch_ >= 'a' && ch_ <= 'f') ||
(ch_ >= 'A' && ch_ <= 'F')))
{
std::ostringstream ss_;
ss_ << "Illegal char following \\x at index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
std::size_t hex_ = 0;
do
{
hex_ *= 16;
if (ch_ >= '0' && ch_ <= '9')
{
hex_ += ch_ - '0';
}
else if (ch_ >= 'a' && ch_ <= 'f')
{
hex_ += 10 + (ch_ - 'a');
}
else
{
hex_ += 10 + (ch_ - 'A');
}
eos_ = state_.eos ();
if (!eos_)
{
ch_ = *state_._curr;
// Don't consume invalid chars!
if (((ch_ >= '0' && ch_ <= '9') ||
(ch_ >= 'a' && ch_ <= 'f') || (ch_ >= 'A' && ch_ <= 'F')))
{
state_.increment ();
}
else
{
eos_ = true;
}
}
} while (!eos_);
return static_cast<CharT> (hex_);
}
static void charset_range (const bool chset_, state &state_, bool &eos_,
CharT &ch_, const CharT prev_, string &chars_)
{
if (chset_)
{
std::ostringstream ss_;
ss_ << "Charset cannot form start of range preceding "
"index " << state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
eos_ = state_.next (ch_);
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex "
"following '-'.");
}
CharT curr_ = 0;
if (ch_ == '\\')
{
std::size_t str_len_ = 0;
if (escape_sequence (state_, curr_, str_len_))
{
std::ostringstream ss_;
ss_ << "Charset cannot form end of range preceding index "
<< state_.index () << '.';
throw runtime_error (ss_.str ().c_str ());
}
}
/*
else if (ch_ == '[' && !state_.eos () && *state_._curr == ':')
{
std::ostringstream ss_;
ss_ << "POSIX char class cannot form end of range at "
"index " << state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
*/
else
{
curr_ = ch_;
}
eos_ = state_.next (ch_);
// Covers preceding if and else
if (eos_)
{
// Pointless returning index if at end of string
throw runtime_error ("Unexpected end of regex "
"(missing ']').");
}
std::size_t start_ = static_cast<typename Traits::index_type> (prev_);
std::size_t end_ = static_cast<typename Traits::index_type> (curr_);
// Semanic check
if (end_ < start_)
{
std::ostringstream ss_;
ss_ << "Invalid range in charset preceding index " <<
state_.index () - 1 << '.';
throw runtime_error (ss_.str ().c_str ());
}
chars_.reserve (chars_.size () + (end_ + 1 - start_));
for (; start_ <= end_; ++start_)
{
CharT ch_ = static_cast<CharT> (start_);
if ((state_._flags & icase) &&
(std::isupper (ch_, state_._locale) ||
std::islower (ch_, state_._locale)))
{
CharT upper_ = std::toupper (ch_, state_._locale);
CharT lower_ = std::tolower (ch_, state_._locale);
chars_ += (upper_);
chars_ += (lower_);
}
else
{
chars_ += (ch_);
}
}
}
};
}
}
}
#endif

View File

@@ -0,0 +1,98 @@
// tokeniser_state.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_RE_TOKENISER_STATE_HPP
#define BOOST_LEXER_RE_TOKENISER_STATE_HPP
#include "../../consts.hpp"
#include <locale>
#include "../../size_t.hpp"
#include <stack>
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT>
struct basic_re_tokeniser_state
{
const CharT * const _start;
const CharT * const _end;
const CharT *_curr;
regex_flags _flags;
std::stack<regex_flags> _flags_stack;
std::locale _locale;
long _paren_count;
bool _in_string;
bool _seen_BOL_assertion;
bool _seen_EOL_assertion;
basic_re_tokeniser_state (const CharT *start_, const CharT * const end_,
const regex_flags flags_, const std::locale locale_) :
_start (start_),
_end (end_),
_curr (start_),
_flags (flags_),
_locale (locale_),
_paren_count (0),
_in_string (false),
_seen_BOL_assertion (false),
_seen_EOL_assertion (false)
{
}
// prevent VC++ 7.1 warning:
const basic_re_tokeniser_state &operator =
(const basic_re_tokeniser_state &rhs_)
{
_start = rhs_._start;
_end = rhs_._end;
_curr = rhs_._curr;
_flags = rhs_._flags;
_locale = rhs_._locale;
_paren_count = rhs_._paren_count;
_in_string = rhs_._in_string;
_seen_BOL_assertion = rhs_._seen_BOL_assertion;
_seen_EOL_assertion = rhs_._seen_EOL_assertion;
return this;
}
inline bool next (CharT &ch_)
{
if (_curr >= _end)
{
ch_ = 0;
return true;
}
else
{
ch_ = *_curr;
increment ();
return false;
}
}
inline void increment ()
{
++_curr;
}
inline std::size_t index ()
{
return _curr - _start;
}
inline bool eos ()
{
return _curr >= _end;
}
};
}
}
}
#endif

View File

@@ -0,0 +1,90 @@
// end_node.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_END_NODE_HPP
#define BOOST_LEXER_END_NODE_HPP
#include "node.hpp"
#include "../../size_t.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
class end_node : public node
{
public:
end_node (const std::size_t id_, const std::size_t unique_id_,
const std::size_t lexer_state_) :
node (false),
_id (id_),
_unique_id (unique_id_),
_lexer_state (lexer_state_)
{
node::_firstpos.push_back (this);
node::_lastpos.push_back (this);
}
virtual ~end_node ()
{
}
virtual type what_type () const
{
return END;
}
virtual bool traverse (const_node_stack &/*node_stack_*/,
bool_stack &/*perform_op_stack_*/) const
{
return false;
}
virtual const node_vector &followpos () const
{
// _followpos is always empty..!
return _followpos;
}
virtual bool end_state () const
{
return true;
}
virtual std::size_t id () const
{
return _id;
}
virtual std::size_t unique_id () const
{
return _unique_id;
}
virtual std::size_t lexer_state () const
{
return _lexer_state;
}
private:
std::size_t _id;
std::size_t _unique_id;
std::size_t _lexer_state;
node_vector _followpos;
virtual void copy_node (node_ptr_vector &/*node_ptr_vector_*/,
node_stack &/*new_node_stack_*/, bool_stack &/*perform_op_stack_*/,
bool &/*down_*/) const
{
// Nothing to do, as end_nodes are not copied.
}
};
}
}
}
#endif

View File

@@ -0,0 +1,90 @@
// iteration_node.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_ITERATION_NODE_HPP
#define BOOST_LEXER_ITERATION_NODE_HPP
#include "node.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
class iteration_node : public node
{
public:
iteration_node (node *next_, const bool greedy_) :
node (true),
_next (next_),
_greedy (greedy_)
{
node_vector::iterator iter_;
node_vector::iterator end_;
_next->append_firstpos (_firstpos);
_next->append_lastpos (_lastpos);
for (iter_ = _lastpos.begin (), end_ = _lastpos.end ();
iter_ != end_; ++iter_)
{
(*iter_)->append_followpos (_firstpos);
}
for (iter_ = _firstpos.begin (), end_ = _firstpos.end ();
iter_ != end_; ++iter_)
{
(*iter_)->greedy (greedy_);
}
}
virtual ~iteration_node ()
{
}
virtual type what_type () const
{
return ITERATION;
}
virtual bool traverse (const_node_stack &node_stack_,
bool_stack &perform_op_stack_) const
{
perform_op_stack_.push (true);
node_stack_.push (_next);
return true;
}
private:
// Not owner of this pointer...
node *_next;
bool _greedy;
virtual void copy_node (node_ptr_vector &node_ptr_vector_,
node_stack &new_node_stack_, bool_stack &perform_op_stack_,
bool &down_) const
{
if (perform_op_stack_.top ())
{
node *ptr_ = new_node_stack_.top ();
node_ptr_vector_->push_back (static_cast<iteration_node *>(0));
node_ptr_vector_->back () = new iteration_node (ptr_, _greedy);
new_node_stack_.top () = node_ptr_vector_->back ();
}
else
{
down_ = true;
}
perform_op_stack_.pop ();
}
};
}
}
}
#endif

View File

@@ -0,0 +1,107 @@
// leaf_node.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_LEAF_NODE_HPP
#define BOOST_LEXER_LEAF_NODE_HPP
#include "../../consts.hpp" // null_token
#include "node.hpp"
#include "../../size_t.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
class leaf_node : public node
{
public:
leaf_node (const std::size_t token_, const bool greedy_) :
node (token_ == null_token),
_token (token_),
_set_greedy (!greedy_),
_greedy (greedy_)
{
if (!_nullable)
{
_firstpos.push_back (this);
_lastpos.push_back (this);
}
}
virtual ~leaf_node ()
{
}
virtual void append_followpos (const node_vector &followpos_)
{
for (node_vector::const_iterator iter_ = followpos_.begin (),
end_ = followpos_.end (); iter_ != end_; ++iter_)
{
_followpos.push_back (*iter_);
}
}
virtual type what_type () const
{
return LEAF;
}
virtual bool traverse (const_node_stack &/*node_stack_*/,
bool_stack &/*perform_op_stack_*/) const
{
return false;
}
virtual std::size_t token () const
{
return _token;
}
virtual void greedy (const bool greedy_)
{
if (!_set_greedy)
{
_greedy = greedy_;
_set_greedy = true;
}
}
virtual bool greedy () const
{
return _greedy;
}
virtual const node_vector &followpos () const
{
return _followpos;
}
virtual node_vector &followpos ()
{
return _followpos;
}
private:
std::size_t _token;
bool _set_greedy;
bool _greedy;
node_vector _followpos;
virtual void copy_node (node_ptr_vector &node_ptr_vector_,
node_stack &new_node_stack_, bool_stack &/*perform_op_stack_*/,
bool &/*down_*/) const
{
node_ptr_vector_->push_back (static_cast<leaf_node *>(0));
node_ptr_vector_->back () = new leaf_node (_token, _greedy);
new_node_stack_.push (node_ptr_vector_->back ());
}
};
}
}
}
#endif

View File

@@ -0,0 +1,188 @@
// node.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_NODE_HPP
#define BOOST_LEXER_NODE_HPP
#include <boost/assert.hpp>
#include "../../containers/ptr_vector.hpp"
#include "../../runtime_error.hpp"
#include "../../size_t.hpp"
#include <stack>
#include <vector>
namespace boost
{
namespace lexer
{
namespace detail
{
class node
{
public:
enum type {LEAF, SEQUENCE, SELECTION, ITERATION, END};
typedef std::stack<bool> bool_stack;
typedef std::stack<node *> node_stack;
// stack and vector not owner of node pointers
typedef std::stack<const node *> const_node_stack;
typedef std::vector<node *> node_vector;
typedef ptr_vector<node> node_ptr_vector;
node () :
_nullable (false)
{
}
node (const bool nullable_) :
_nullable (nullable_)
{
}
virtual ~node ()
{
}
bool nullable () const
{
return _nullable;
}
void append_firstpos (node_vector &firstpos_) const
{
firstpos_.insert (firstpos_.end (),
_firstpos.begin (), _firstpos.end ());
}
void append_lastpos (node_vector &lastpos_) const
{
lastpos_.insert (lastpos_.end (),
_lastpos.begin (), _lastpos.end ());
}
virtual void append_followpos (const node_vector &/*followpos_*/)
{
throw runtime_error ("Internal error node::append_followpos()");
}
node *copy (node_ptr_vector &node_ptr_vector_) const
{
node *new_root_ = 0;
const_node_stack node_stack_;
bool_stack perform_op_stack_;
bool down_ = true;
node_stack new_node_stack_;
node_stack_.push (this);
while (!node_stack_.empty ())
{
while (down_)
{
down_ = node_stack_.top ()->traverse (node_stack_,
perform_op_stack_);
}
while (!down_ && !node_stack_.empty ())
{
const node *top_ = node_stack_.top ();
top_->copy_node (node_ptr_vector_, new_node_stack_,
perform_op_stack_, down_);
if (!down_) node_stack_.pop ();
}
}
BOOST_ASSERT(new_node_stack_.size () == 1);
new_root_ = new_node_stack_.top ();
new_node_stack_.pop ();
return new_root_;
}
virtual type what_type () const = 0;
virtual bool traverse (const_node_stack &node_stack_,
bool_stack &perform_op_stack_) const = 0;
node_vector &firstpos ()
{
return _firstpos;
}
const node_vector &firstpos () const
{
return _firstpos;
}
// _lastpos modified externally, so not const &
node_vector &lastpos ()
{
return _lastpos;
}
virtual bool end_state () const
{
return false;
}
virtual std::size_t id () const
{
throw runtime_error ("Internal error node::id()");
}
virtual std::size_t unique_id () const
{
throw runtime_error ("Internal error node::unique_id()");
}
virtual std::size_t lexer_state () const
{
throw runtime_error ("Internal error node::state()");
}
virtual std::size_t token () const
{
throw runtime_error ("Internal error node::token()");
}
virtual void greedy (const bool /*greedy_*/)
{
throw runtime_error ("Internal error node::token(bool)");
}
virtual bool greedy () const
{
throw runtime_error ("Internal error node::token()");
}
virtual const node_vector &followpos () const
{
throw runtime_error ("Internal error node::followpos()");
}
virtual node_vector &followpos ()
{
throw runtime_error ("Internal error node::followpos()");
}
protected:
const bool _nullable;
node_vector _firstpos;
node_vector _lastpos;
virtual void copy_node (node_ptr_vector &node_ptr_vector_,
node_stack &new_node_stack_, bool_stack &perform_op_stack_,
bool &down_) const = 0;
private:
node (node const &); // No copy construction.
node &operator = (node const &); // No assignment.
};
}
}
}
#endif

View File

@@ -0,0 +1,94 @@
// selection_node.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_SELECTION_NODE_HPP
#define BOOST_LEXER_SELECTION_NODE_HPP
#include "node.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
class selection_node : public node
{
public:
selection_node (node *left_, node *right_) :
node (left_->nullable () || right_->nullable ()),
_left (left_),
_right (right_)
{
_left->append_firstpos (_firstpos);
_right->append_firstpos (_firstpos);
_left->append_lastpos (_lastpos);
_right->append_lastpos (_lastpos);
}
virtual ~selection_node ()
{
}
virtual type what_type () const
{
return SELECTION;
}
virtual bool traverse (const_node_stack &node_stack_,
bool_stack &perform_op_stack_) const
{
perform_op_stack_.push (true);
switch (_right->what_type ())
{
case SEQUENCE:
case SELECTION:
case ITERATION:
perform_op_stack_.push (false);
break;
default:
break;
}
node_stack_.push (_right);
node_stack_.push (_left);
return true;
}
private:
// Not owner of these pointers...
node *_left;
node *_right;
virtual void copy_node (node_ptr_vector &node_ptr_vector_,
node_stack &new_node_stack_, bool_stack &perform_op_stack_,
bool &down_) const
{
if (perform_op_stack_.top ())
{
node *rhs_ = new_node_stack_.top ();
new_node_stack_.pop ();
node *lhs_ = new_node_stack_.top ();
node_ptr_vector_->push_back (static_cast<selection_node *>(0));
node_ptr_vector_->back () = new selection_node (lhs_, rhs_);
new_node_stack_.top () = node_ptr_vector_->back ();
}
else
{
down_ = true;
}
perform_op_stack_.pop ();
}
};
}
}
}
#endif

View File

@@ -0,0 +1,112 @@
// sequence_node.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_SEQUENCE_NODE_HPP
#define BOOST_LEXER_SEQUENCE_NODE_HPP
#include "node.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
class sequence_node : public node
{
public:
sequence_node (node *left_, node *right_) :
node (left_->nullable () && right_->nullable ()),
_left (left_),
_right (right_)
{
_left->append_firstpos (_firstpos);
if (_left->nullable ())
{
_right->append_firstpos (_firstpos);
}
if (_right->nullable ())
{
_left->append_lastpos (_lastpos);
}
_right->append_lastpos (_lastpos);
node_vector &lastpos_ = _left->lastpos ();
const node_vector &firstpos_ = _right->firstpos ();
for (node_vector::iterator iter_ = lastpos_.begin (),
end_ = lastpos_.end (); iter_ != end_; ++iter_)
{
(*iter_)->append_followpos (firstpos_);
}
}
virtual ~sequence_node ()
{
}
virtual type what_type () const
{
return SEQUENCE;
}
virtual bool traverse (const_node_stack &node_stack_,
bool_stack &perform_op_stack_) const
{
perform_op_stack_.push (true);
switch (_right->what_type ())
{
case SEQUENCE:
case SELECTION:
case ITERATION:
perform_op_stack_.push (false);
break;
default:
break;
}
node_stack_.push (_right);
node_stack_.push (_left);
return true;
}
private:
// Not owner of these pointers...
node *_left;
node *_right;
virtual void copy_node (node_ptr_vector &node_ptr_vector_,
node_stack &new_node_stack_, bool_stack &perform_op_stack_,
bool &down_) const
{
if (perform_op_stack_.top ())
{
node *rhs_ = new_node_stack_.top ();
new_node_stack_.pop ();
node *lhs_ = new_node_stack_.top ();
node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node (lhs_, rhs_);
new_node_stack_.top () = node_ptr_vector_->back ();
}
else
{
down_ = true;
}
perform_op_stack_.pop ();
}
};
}
}
}
#endif

View File

@@ -0,0 +1,81 @@
// charset.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_CHARSET_HPP
#define BOOST_LEXER_CHARSET_HPP
#include <set>
#include "../size_t.hpp"
#include "../string_token.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
template<typename CharT>
struct basic_charset
{
typedef basic_string_token<CharT> token;
typedef std::set<std::size_t> index_set;
token _token;
index_set _index_set;
basic_charset ()
{
}
basic_charset (const token &token_, const std::size_t index_) :
_token (token_)
{
_index_set.insert (index_);
}
bool empty () const
{
return _token.empty () && _index_set.empty ();
}
void intersect (basic_charset &rhs_, basic_charset &overlap_)
{
_token.intersect (rhs_._token, overlap_._token);
if (!overlap_._token.empty ())
{
typename index_set::const_iterator iter_ = _index_set.begin ();
typename index_set::const_iterator end_ = _index_set.end ();
for (; iter_ != end_; ++iter_)
{
overlap_._index_set.insert (*iter_);
}
iter_ = rhs_._index_set.begin ();
end_ = rhs_._index_set.end ();
for (; iter_ != end_; ++iter_)
{
overlap_._index_set.insert (*iter_);
}
if (_token.empty ())
{
_index_set.clear ();
}
if (rhs_._token.empty ())
{
rhs_._index_set.clear ();
}
}
}
};
}
}
}
#endif

View File

@@ -0,0 +1,140 @@
// equivset.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_EQUIVSET_HPP
#define BOOST_LEXER_EQUIVSET_HPP
#include <algorithm>
#include "../parser/tree/node.hpp"
#include <set>
#include "../size_t.hpp"
namespace boost
{
namespace lexer
{
namespace detail
{
struct equivset
{
typedef std::set<std::size_t> index_set;
typedef std::vector<std::size_t> index_vector;
// Not owner of nodes:
typedef std::vector<node *> node_vector;
index_vector _index_vector;
bool _greedy;
std::size_t _id;
node_vector _followpos;
equivset () :
_greedy (true),
_id (0)
{
}
equivset (const index_set &index_set_, const bool greedy_,
const std::size_t id_, const node_vector &followpos_) :
_greedy (greedy_),
_id (id_),
_followpos (followpos_)
{
index_set::const_iterator iter_ = index_set_.begin ();
index_set::const_iterator end_ = index_set_.end ();
for (; iter_ != end_; ++iter_)
{
_index_vector.push_back (*iter_);
}
}
bool empty () const
{
return _index_vector.empty () && _followpos.empty ();
}
void intersect (equivset &rhs_, equivset &overlap_)
{
intersect_indexes (rhs_._index_vector, overlap_._index_vector);
if (!overlap_._index_vector.empty ())
{
// Note that the LHS takes priority in order to
// respect rule ordering priority in the lex spec.
overlap_._id = _id;
overlap_._greedy = _greedy;
overlap_._followpos = _followpos;
node_vector::const_iterator overlap_begin_ =
overlap_._followpos.begin ();
node_vector::const_iterator overlap_end_ =
overlap_._followpos.end ();
node_vector::const_iterator rhs_iter_ =
rhs_._followpos.begin ();
node_vector::const_iterator rhs_end_ =
rhs_._followpos.end ();
for (; rhs_iter_ != rhs_end_; ++rhs_iter_)
{
node *node_ = *rhs_iter_;
if (std::find (overlap_begin_, overlap_end_, node_) ==
overlap_end_)
{
overlap_._followpos.push_back (node_);
overlap_begin_ = overlap_._followpos.begin ();
overlap_end_ = overlap_._followpos.end ();
}
}
if (_index_vector.empty ())
{
_followpos.clear ();
}
if (rhs_._index_vector.empty ())
{
rhs_._followpos.clear ();
}
}
}
private:
void intersect_indexes (index_vector &rhs_, index_vector &overlap_)
{
index_vector::iterator iter_ = _index_vector.begin ();
index_vector::iterator end_ = _index_vector.end ();
index_vector::iterator rhs_iter_ = rhs_.begin ();
index_vector::iterator rhs_end_ = rhs_.end ();
while (iter_ != end_ && rhs_iter_ != rhs_end_)
{
const std::size_t index_ = *iter_;
const std::size_t rhs_index_ = *rhs_iter_;
if (index_ < rhs_index_)
{
++iter_;
}
else if (index_ > rhs_index_)
{
++rhs_iter_;
}
else
{
overlap_.push_back (index_);
iter_ = _index_vector.erase (iter_);
end_ = _index_vector.end ();
rhs_iter_ = rhs_.erase (rhs_iter_);
rhs_end_ = rhs_.end ();
}
}
}
};
}
}
}
#endif

View File

@@ -0,0 +1,806 @@
// rules.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_RULES_HPP
#define BOOST_LEXER_RULES_HPP
#include "consts.hpp"
#include <deque>
#include <locale>
#include <map>
#include "runtime_error.hpp"
#include <set>
#include "size_t.hpp"
#include <sstream>
#include <string>
#include <vector>
namespace boost
{
namespace lexer
{
namespace detail
{
// return name of initial state
template <typename CharT>
struct strings;
template <>
struct strings<char>
{
static const char *initial ()
{
return "INITIAL";
}
static const char *dot ()
{
return ".";
}
static const char *all_states ()
{
return "*";
}
static const char *char_name ()
{
return "char";
}
static const char *char_prefix ()
{
return "";
}
};
template <>
struct strings<wchar_t>
{
static const wchar_t *initial ()
{
return L"INITIAL";
}
static const wchar_t *dot ()
{
return L".";
}
static const wchar_t *all_states ()
{
return L"*";
}
static const char *char_name ()
{
return "wchar_t";
}
static const char *char_prefix ()
{
return "L";
}
};
}
template<typename CharT>
class basic_rules
{
public:
typedef std::vector<std::size_t> id_vector;
typedef std::deque<id_vector> id_vector_deque;
typedef std::basic_string<CharT> string;
typedef std::deque<string> string_deque;
typedef std::deque<string_deque> string_deque_deque;
typedef std::set<string> string_set;
typedef std::pair<string, string> string_pair;
typedef std::deque<string_pair> string_pair_deque;
typedef std::map<string, std::size_t> string_size_t_map;
typedef std::pair<string, std::size_t> string_size_t_pair;
basic_rules (const regex_flags flags_ = dot_not_newline,
std::size_t (*counter_ptr_) () = 0) :
_flags (flags_),
_counter (0),
_counter_ptr (counter_ptr_)
{
add_state (initial ());
}
void clear ()
{
_statemap.clear ();
_macrodeque.clear ();
_macroset.clear ();
_regexes.clear ();
_ids.clear ();
_unique_ids.clear ();
_states.clear ();
_flags = dot_not_newline;
_locale = std::locale ();
add_state (initial ());
}
void clear (const CharT *state_name_)
{
std::size_t state_ = state (state_name_);
if (state_ != npos)
{
_regexes[state_].clear ();
_ids[state_].clear ();
_unique_ids[state_].clear ();
_states[state_].clear ();
}
}
void flags (const regex_flags flags_)
{
_flags = flags_;
}
regex_flags flags () const
{
return _flags;
}
std::size_t next_unique_id ()
{
return _counter_ptr ? _counter_ptr () : _counter++;
}
std::locale imbue (std::locale &locale_)
{
std::locale loc_ = _locale;
_locale = locale_;
return loc_;
}
const std::locale &locale () const
{
return _locale;
}
std::size_t state (const CharT *name_) const
{
std::size_t state_ = npos;
typename string_size_t_map::const_iterator iter_ =
_statemap.find (name_);
if (iter_ != _statemap.end ())
{
state_ = iter_->second;
}
return state_;
}
const CharT *state (const std::size_t index_) const
{
if (index_ == 0)
{
return initial ();
}
else
{
const std::size_t vec_index_ = index_ - 1;
if (vec_index_ > _lexer_state_names.size () - 1)
{
return 0;
}
else
{
return _lexer_state_names[vec_index_].c_str ();
}
}
}
std::size_t add_state (const CharT *name_)
{
validate (name_);
if (_statemap.insert (string_size_t_pair (name_,
_statemap.size ())).second)
{
_regexes.push_back (string_deque ());
_ids.push_back (id_vector ());
_unique_ids.push_back (id_vector ());
_states.push_back (id_vector ());
if (string (name_) != initial ())
{
_lexer_state_names.push_back (name_);
}
}
// Initial is not stored, so no need to - 1.
return _lexer_state_names.size ();
}
void add_macro (const CharT *name_, const CharT *regex_)
{
add_macro (name_, string (regex_));
}
void add_macro (const CharT *name_, const CharT *regex_start_,
const CharT *regex_end_)
{
add_macro (name_, string (regex_start_, regex_end_));
}
void add_macro (const CharT *name_, const string &regex_)
{
validate (name_);
typename string_set::const_iterator iter_ = _macroset.find (name_);
if (iter_ == _macroset.end ())
{
_macrodeque.push_back (string_pair (name_, regex_));
_macroset.insert (name_);
}
else
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Attempt to redefine MACRO '";
while (*name_)
{
os_ << ss_.narrow (*name_++, static_cast<CharT> (' '));
}
os_ << "'.";
throw runtime_error (os_.str ());
}
}
void add_macros (const basic_rules<CharT> &rules_)
{
const string_pair_deque &macros_ = rules_.macrodeque ();
typename string_pair_deque::const_iterator macro_iter_ =
macros_.begin ();
typename string_pair_deque::const_iterator macro_end_ =
macros_.end ();
for (; macro_iter_ != macro_end_; ++macro_iter_)
{
add_macro (macro_iter_->first.c_str (),
macro_iter_->second.c_str ());
}
}
void merge_macros (const basic_rules<CharT> &rules_)
{
const string_pair_deque &macros_ = rules_.macrodeque ();
typename string_pair_deque::const_iterator macro_iter_ =
macros_.begin ();
typename string_pair_deque::const_iterator macro_end_ =
macros_.end ();
typename string_set::const_iterator macro_dest_iter_;
typename string_set::const_iterator macro_dest_end_ = _macroset.end ();
for (; macro_iter_ != macro_end_; ++macro_iter_)
{
macro_dest_iter_ = _macroset.find (macro_iter_->first);
if (macro_dest_iter_ == macro_dest_end_)
{
add_macro (macro_iter_->first.c_str (),
macro_iter_->second.c_str ());
}
}
}
std::size_t add (const CharT *regex_, const std::size_t id_)
{
return add (string (regex_), id_);
}
std::size_t add (const CharT *regex_start_, const CharT *regex_end_,
const std::size_t id_)
{
return add (string (regex_start_, regex_end_), id_);
}
std::size_t add (const string &regex_, const std::size_t id_)
{
const std::size_t counter_ = next_unique_id ();
check_for_invalid_id (id_);
_regexes.front ().push_back (regex_);
_ids.front ().push_back (id_);
_unique_ids.front ().push_back (counter_);
_states.front ().push_back (0);
return counter_;
}
std::size_t add (const CharT *curr_state_, const CharT *regex_,
const CharT *new_state_)
{
return add (curr_state_, string (regex_), new_state_);
}
std::size_t add (const CharT *curr_state_, const CharT *regex_start_,
const CharT *regex_end_, const CharT *new_state_)
{
return add (curr_state_, string (regex_start_, regex_end_),
new_state_);
}
std::size_t add (const CharT *curr_state_, const string &regex_,
const CharT *new_state_)
{
return add (curr_state_, regex_, 0, new_state_, false);
}
std::size_t add (const CharT *curr_state_, const CharT *regex_,
const std::size_t id_, const CharT *new_state_)
{
return add (curr_state_, string (regex_), id_, new_state_);
}
std::size_t add (const CharT *curr_state_, const CharT *regex_start_,
const CharT *regex_end_, const std::size_t id_,
const CharT *new_state_)
{
return add (curr_state_, string (regex_start_, regex_end_), id_,
new_state_);
}
std::size_t add (const CharT *curr_state_, const string &regex_,
const std::size_t id_, const CharT *new_state_)
{
return add (curr_state_, regex_, id_, new_state_, true);
}
void add (const CharT *source_, const basic_rules<CharT> &rules_,
const CharT *dest_, const CharT *to_ = detail::strings<CharT>::dot ())
{
const bool star_ = *source_ == '*' && *(source_ + 1) == 0;
const bool dest_dot_ = *dest_ == '.' && *(dest_ + 1) == 0;
const bool to_dot_ = *to_ == '.' && *(to_ + 1) == 0;
std::size_t state_ = 0;
const string_deque_deque &all_regexes_ = rules_.regexes ();
const id_vector_deque &all_ids_ = rules_.ids ();
const id_vector_deque &all_unique_ids_ = rules_.unique_ids ();
const id_vector_deque &all_states_ = rules_.states ();
typename string_deque::const_iterator regex_iter_;
typename string_deque::const_iterator regex_end_;
typename id_vector::const_iterator id_iter_;
typename id_vector::const_iterator uid_iter_;
typename id_vector::const_iterator state_iter_;
if (star_)
{
typename string_deque_deque::const_iterator all_regexes_iter_ =
all_regexes_.begin ();
typename string_deque_deque::const_iterator all_regexes_end_ =
all_regexes_.end ();
typename id_vector_deque::const_iterator all_ids_iter_ =
all_ids_.begin ();
typename id_vector_deque::const_iterator all_uids_iter_ =
all_unique_ids_.begin ();
typename id_vector_deque::const_iterator all_states_iter_ =
all_states_.begin ();
for (; all_regexes_iter_ != all_regexes_end_;
++state_, ++all_regexes_iter_, ++all_ids_iter_,
++all_uids_iter_, ++all_states_iter_)
{
regex_iter_ = all_regexes_iter_->begin ();
regex_end_ = all_regexes_iter_->end ();
id_iter_ = all_ids_iter_->begin ();
uid_iter_ = all_uids_iter_->begin ();
state_iter_ = all_states_iter_->begin ();
for (; regex_iter_ != regex_end_; ++regex_iter_, ++id_iter_,
++uid_iter_, ++state_iter_)
{
// If ..._dot_ then lookup state name from rules_; otherwise
// pass name through.
add (dest_dot_ ? rules_.state (state_) : dest_, *regex_iter_,
*id_iter_, to_dot_ ? rules_.state (*state_iter_) : to_, true,
*uid_iter_);
}
}
}
else
{
const CharT *start_ = source_;
string state_name_;
while (*source_)
{
while (*source_ && *source_ != ',')
{
++source_;
}
state_name_.assign (start_, source_);
if (*source_)
{
++source_;
start_ = source_;
}
state_ = rules_.state (state_name_.c_str ());
if (state_ == npos)
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Unknown state name '";
source_ = state_name_.c_str ();
while (*source_)
{
os_ << ss_.narrow (*source_++, ' ');
}
os_ << "'.";
throw runtime_error (os_.str ());
}
regex_iter_ = all_regexes_[state_].begin ();
regex_end_ = all_regexes_[state_].end ();
id_iter_ = all_ids_[state_].begin ();
uid_iter_ = all_unique_ids_[state_].begin ();
state_iter_ = all_states_[state_].begin ();
for (; regex_iter_ != regex_end_; ++regex_iter_, ++id_iter_,
++uid_iter_, ++state_iter_)
{
// If ..._dot_ then lookup state name from rules_; otherwise
// pass name through.
add (dest_dot_ ? state_name_.c_str () : dest_, *regex_iter_,
*id_iter_, to_dot_ ? rules_.state (*state_iter_) : to_, true,
*uid_iter_);
}
}
}
}
/*
void add (const CharT *curr_state_, const basic_rules<CharT> &rules_)
{
const string_deque_deque &regexes_ = rules_.regexes ();
const id_vector_deque &ids_ = rules_.ids ();
const id_vector_deque &unique_ids_ = rules_.unique_ids ();
typename string_deque_deque::const_iterator state_regex_iter_ =
regexes_.begin ();
typename string_deque_deque::const_iterator state_regex_end_ =
regexes_.end ();
typename id_vector_deque::const_iterator state_id_iter_ =
ids_.begin ();
typename id_vector_deque::const_iterator state_uid_iter_ =
unique_ids_.begin ();
typename string_deque::const_iterator regex_iter_;
typename string_deque::const_iterator regex_end_;
typename id_vector::const_iterator id_iter_;
typename id_vector::const_iterator uid_iter_;
for (; state_regex_iter_ != state_regex_end_; ++state_regex_iter_)
{
regex_iter_ = state_regex_iter_->begin ();
regex_end_ = state_regex_iter_->end ();
id_iter_ = state_id_iter_->begin ();
uid_iter_ = state_uid_iter_->begin ();
for (; regex_iter_ != regex_end_; ++regex_iter_, ++id_iter_,
++uid_iter_)
{
add (curr_state_, *regex_iter_, *id_iter_, curr_state_, true,
*uid_iter_);
}
}
}
*/
const string_size_t_map &statemap () const
{
return _statemap;
}
const string_pair_deque &macrodeque () const
{
return _macrodeque;
}
const string_deque_deque &regexes () const
{
return _regexes;
}
const id_vector_deque &ids () const
{
return _ids;
}
const id_vector_deque &unique_ids () const
{
return _unique_ids;
}
const id_vector_deque &states () const
{
return _states;
}
bool empty () const
{
typename string_deque_deque::const_iterator iter_ = _regexes.begin ();
typename string_deque_deque::const_iterator end_ = _regexes.end ();
bool empty_ = true;
for (; iter_ != end_; ++iter_)
{
if (!iter_->empty ())
{
empty_ = false;
break;
}
}
return empty_;
}
static const CharT *initial ()
{
return detail::strings<CharT>::initial ();
}
static const CharT *all_states ()
{
return detail::strings<CharT>::all_states ();
}
static const CharT *dot ()
{
return detail::strings<CharT>::dot ();
}
private:
string_size_t_map _statemap;
string_pair_deque _macrodeque;
string_set _macroset;
string_deque_deque _regexes;
id_vector_deque _ids;
id_vector_deque _unique_ids;
id_vector_deque _states;
regex_flags _flags;
std::size_t _counter;
std::size_t (*_counter_ptr) ();
std::locale _locale;
string_deque _lexer_state_names;
std::size_t add (const CharT *curr_state_, const string &regex_,
const std::size_t id_, const CharT *new_state_, const bool check_,
const std::size_t uid_ = npos)
{
const bool star_ = *curr_state_ == '*' && *(curr_state_ + 1) == 0;
const bool dot_ = *new_state_ == '.' && *(new_state_ + 1) == 0;
if (check_)
{
check_for_invalid_id (id_);
}
if (!dot_)
{
validate (new_state_);
}
std::size_t new_ = string::npos;
typename string_size_t_map::const_iterator iter_;
typename string_size_t_map::const_iterator end_ = _statemap.end ();
id_vector states_;
if (!dot_)
{
iter_ = _statemap.find (new_state_);
if (iter_ == end_)
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Unknown state name '";
while (*new_state_)
{
os_ << ss_.narrow (*new_state_++, ' ');
}
os_ << "'.";
throw runtime_error (os_.str ());
}
new_ = iter_->second;
}
if (star_)
{
const std::size_t size_ = _statemap.size ();
for (std::size_t i_ = 0; i_ < size_; ++i_)
{
states_.push_back (i_);
}
}
else
{
const CharT *start_ = curr_state_;
string state_;
while (*curr_state_)
{
while (*curr_state_ && *curr_state_ != ',')
{
++curr_state_;
}
state_.assign (start_, curr_state_);
if (*curr_state_)
{
++curr_state_;
start_ = curr_state_;
}
validate (state_.c_str ());
iter_ = _statemap.find (state_.c_str ());
if (iter_ == end_)
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Unknown state name '";
curr_state_ = state_.c_str ();
while (*curr_state_)
{
os_ << ss_.narrow (*curr_state_++, ' ');
}
os_ << "'.";
throw runtime_error (os_.str ());
}
states_.push_back (iter_->second);
}
}
std::size_t first_counter_ = npos;
for (std::size_t i_ = 0, size_ = states_.size (); i_ < size_; ++i_)
{
const std::size_t curr_ = states_[i_];
_regexes[curr_].push_back (regex_);
_ids[curr_].push_back (id_);
if (uid_ == npos)
{
const std::size_t counter_ = next_unique_id ();
if (first_counter_ == npos)
{
first_counter_ = counter_;
}
_unique_ids[curr_].push_back (counter_);
}
else
{
if (first_counter_ == npos)
{
first_counter_ = uid_;
}
_unique_ids[curr_].push_back (uid_);
}
_states[curr_].push_back (dot_ ? curr_ : new_);
}
return first_counter_;
}
void validate (const CharT *name_) const
{
const CharT *start_ = name_;
if (*name_ != '_' && !(*name_ >= 'A' && *name_ <= 'Z') &&
!(*name_ >= 'a' && *name_ <= 'z'))
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Invalid name '";
while (*name_)
{
os_ << ss_.narrow (*name_++, ' ');
}
os_ << "'.";
throw runtime_error (os_.str ());
}
else if (*name_)
{
++name_;
}
while (*name_)
{
if (*name_ != '_' && *name_ != '-' &&
!(*name_ >= 'A' && *name_ <= 'Z') &&
!(*name_ >= 'a' && *name_ <= 'z') &&
!(*name_ >= '0' && *name_ <= '9'))
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Invalid name '";
name_ = start_;
while (*name_)
{
os_ << ss_.narrow (*name_++, ' ');
}
os_ << "'.";
throw runtime_error (os_.str ());
}
++name_;
}
if (name_ - start_ > static_cast<std::ptrdiff_t>(max_macro_len))
{
std::basic_stringstream<CharT> ss_;
std::ostringstream os_;
os_ << "Name '";
name_ = start_;
while (*name_)
{
os_ << ss_.narrow (*name_++, ' ');
}
os_ << "' too long.";
throw runtime_error (os_.str ());
}
}
void check_for_invalid_id (const std::size_t id_) const
{
switch (id_)
{
case 0:
throw runtime_error ("id 0 is reserved for EOF.");
case npos:
throw runtime_error ("id npos is reserved for the "
"UNKNOWN token.");
default:
// OK
break;
}
}
};
typedef basic_rules<char> rules;
typedef basic_rules<wchar_t> wrules;
}
}
#endif

View File

@@ -0,0 +1,26 @@
// runtime_error.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_RUNTIME_ERROR_HPP
#define BOOST_LEXER_RUNTIME_ERROR_HPP
#include <stdexcept>
namespace boost
{
namespace lexer
{
class runtime_error : public std::runtime_error
{
public:
runtime_error (const std::string &what_arg_) :
std::runtime_error (what_arg_)
{
}
};
}
}
#endif

View File

@@ -0,0 +1,35 @@
// examples/serialise.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_SERIALISE_HPP
#define BOOST_LEXER_SERIALISE_HPP
#include "internals.hpp"
#include "state_machine.hpp"
#include <boost/serialization/vector.hpp>
namespace boost
{
namespace lexer
{
// IMPORTANT! This won't work if you don't enable RTTI!
template<typename CharT, class Archive>
void serialise (basic_state_machine<CharT> &sm_, Archive &ar_,
unsigned int version_ = 1)
{
detail::internals &internals_ = const_cast<detail::internals &>
(sm_.data ());
ar_ & version_;
ar_ & *internals_._lookup;
ar_ & internals_._dfa_alphabet;
ar_ & *internals_._dfa;
ar_ & internals_._seen_BOL_assertion;
ar_ & internals_._seen_EOL_assertion;
}
}
}
#endif

View File

@@ -0,0 +1,13 @@
// size_t.h
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_SIZE_T_H
#define BOOST_LEXER_SIZE_T_H
#include <stddef.h> // ptrdiff_t
#include <string>
#endif

View File

@@ -0,0 +1,431 @@
// state_machine.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_STATE_MACHINE_HPP
#define BOOST_LEXER_STATE_MACHINE_HPP
#include <algorithm>
#include "conversion/char_state_machine.hpp"
#include "consts.hpp"
#include <deque>
#include "internals.hpp"
#include <map>
#include "containers/ptr_vector.hpp"
#include "size_t.hpp"
#include <string>
namespace boost
{
namespace lexer
{
template<typename CharT>
class basic_state_machine
{
public:
typedef CharT char_type;
class iterator
{
public:
friend class basic_state_machine;
struct data
{
// Current iterator info
std::size_t dfa;
std::size_t states;
std::size_t state;
std::size_t transitions;
std::size_t transition;
// Current state info
bool end_state;
std::size_t id;
std::size_t unique_id;
std::size_t goto_dfa;
std::size_t bol_index;
std::size_t eol_index;
// Current transition info
basic_string_token<CharT> token;
std::size_t goto_state;
data () :
dfa (npos),
states (0),
state (npos),
transitions (0),
transition (npos),
end_state (false),
id (npos),
unique_id (npos),
goto_dfa (npos),
bol_index (npos),
eol_index (npos),
goto_state (npos)
{
}
bool operator == (const data &rhs_) const
{
return dfa == rhs_.dfa &&
states == rhs_.states &&
state == rhs_.state &&
transitions == rhs_.transitions &&
transition == rhs_.transition &&
end_state == rhs_.end_state &&
id == rhs_.id &&
unique_id == rhs_.unique_id &&
goto_dfa == rhs_.goto_dfa &&
bol_index == rhs_.bol_index &&
eol_index == rhs_.eol_index &&
token == rhs_.token &&
goto_state == rhs_.goto_state;
}
};
iterator () :
_sm (0),
_dfas (0),
_dfa (npos),
_states (0),
_state (npos),
_transitions (0),
_transition (npos)
{
}
bool operator == (const iterator &rhs_) const
{
return _dfas == rhs_._dfas && _dfa == rhs_._dfa &&
_states == rhs_._states && _state == rhs_._state &&
_transitions == rhs_._transitions &&
_transition == rhs_._transition;
}
bool operator != (const iterator &rhs_) const
{
return !(*this == rhs_);
}
data &operator * ()
{
return _data;
}
data *operator -> ()
{
return &_data;
}
// Let compiler generate operator = ().
// prefix version
iterator &operator ++ ()
{
next ();
return *this;
}
// postfix version
iterator operator ++ (int)
{
iterator iter_ = *this;
next ();
return iter_;
}
void clear ()
{
_dfas = _states = _transitions = 0;
_dfa = _state = _transition = npos;
}
private:
basic_state_machine *_sm;
data _data;
std::size_t _dfas;
std::size_t _dfa;
std::size_t _states;
std::size_t _state;
std::size_t _transitions;
std::size_t _transition;
typename detail::basic_char_state_machine<CharT>::state::
size_t_string_token_map::const_iterator _token_iter;
typename detail::basic_char_state_machine<CharT>::state::
size_t_string_token_map::const_iterator _token_end;
void next ()
{
bool reset_state_ = false;
if (_transition >= _transitions)
{
_transition = _data.transition = 0;
_data.state = ++_state;
reset_state_ = true;
if (_state >= _states)
{
++_dfa;
if (_dfa >= _dfas)
{
clear ();
reset_state_ = false;
}
else
{
_states = _data.states =
_sm->_csm._sm_vector[_dfa].size ();
_state = _data.state = 0;
}
}
}
else
{
_data.transition = _transition;
}
if (reset_state_)
{
const typename detail::basic_char_state_machine<CharT>::
state *ptr_ = &_sm->_csm._sm_vector[_dfa][_state];
_transitions = _data.transitions = ptr_->_transitions.size ();
_data.end_state = ptr_->_end_state;
_data.id = ptr_->_id;
_data.unique_id = ptr_->_unique_id;
_data.goto_dfa = ptr_->_state;
_data.bol_index = ptr_->_bol_index;
_data.eol_index = ptr_->_eol_index;
_token_iter = ptr_->_transitions.begin ();
_token_end = ptr_->_transitions.end ();
}
if (_token_iter != _token_end)
{
_data.token = _token_iter->second;
_data.goto_state = _token_iter->first;
++_token_iter;
++_transition;
}
else
{
_data.token.clear ();
_data.goto_state = npos;
}
}
};
friend class iterator;
basic_state_machine ()
{
}
void clear ()
{
_internals.clear ();
_csm.clear ();
}
bool empty () const
{
// Don't include _csm in this test, as irrelevant to state.
return _internals._lookup->empty () &&
_internals._dfa_alphabet.empty () &&
_internals._dfa->empty ();
}
std::size_t size () const
{
return _internals._dfa->size ();
}
bool operator == (const basic_state_machine &rhs_) const
{
// Don't include _csm in this test, as irrelevant to state.
return _internals._lookup == rhs_._internals._lookup &&
_internals._dfa_alphabet == rhs_._internals._dfa_alphabet &&
_internals._dfa == rhs_._internals._dfa &&
_internals._seen_BOL_assertion ==
rhs_._internals._seen_BOL_assertion &&
_internals._seen_EOL_assertion ==
rhs_._internals._seen_EOL_assertion;
}
iterator begin () const
{
iterator iter_;
iter_._sm = const_cast<basic_state_machine *>(this);
check_for_csm ();
if (!_csm.empty ())
{
const typename detail::basic_char_state_machine<CharT>::
state_vector *ptr_ = &_csm._sm_vector.front ();
iter_._dfas = _csm._sm_vector.size ();
iter_._states = iter_._data.states = ptr_->size ();
iter_._transitions = iter_._data.transitions =
ptr_->front ()._transitions.size ();
iter_._dfa = iter_._data.dfa = 0;
iter_._state = iter_._data.state = 0;
iter_._transition = 0;
iter_._data.end_state = ptr_->front ()._end_state;
iter_._data.id = ptr_->front ()._id;
iter_._data.unique_id = ptr_->front ()._unique_id;
iter_._data.goto_dfa = ptr_->front ()._state;
iter_._data.bol_index = ptr_->front ()._bol_index;
iter_._data.eol_index = ptr_->front ()._eol_index;
iter_._token_iter = ptr_->front ()._transitions.begin ();
iter_._token_end = ptr_->front ()._transitions.end ();
// Deal with case where there is only a bol or eol
// but no other transitions.
if (iter_._transitions)
{
++iter_;
}
}
return iter_;
}
iterator end () const
{
iterator iter_;
iter_._sm = const_cast<basic_state_machine *>(this);
return iter_;
}
void swap (basic_state_machine &sm_)
{
_internals.swap (sm_._internals);
_csm.swap (sm_._csm);
}
const detail::internals &data () const
{
return _internals;
}
private:
detail::internals _internals;
mutable detail::basic_char_state_machine<CharT> _csm;
void check_for_csm () const
{
if (_csm.empty ())
{
human_readable (_csm);
}
}
void human_readable (detail::basic_char_state_machine<CharT> &sm_) const
{
const std::size_t max_ = sizeof (CharT) == 1 ?
num_chars : num_wchar_ts;
const std::size_t start_states_ = _internals._dfa->size ();
sm_.clear ();
sm_._sm_vector.resize (start_states_);
for (std::size_t start_state_index_ = 0;
start_state_index_ < start_states_; ++start_state_index_)
{
const detail::internals::size_t_vector *lu_ =
_internals._lookup[start_state_index_];
const std::size_t alphabet_ =
_internals._dfa_alphabet[start_state_index_] - dfa_offset;
std::vector<std::basic_string<CharT> > chars_ (alphabet_);
const std::size_t states_ = _internals._dfa[start_state_index_]->
size () / (alphabet_ + dfa_offset);
const std::size_t *read_ptr_ = &_internals.
_dfa[start_state_index_]->front () + alphabet_ + dfa_offset;
sm_._sm_vector[start_state_index_].resize (states_ - 1);
for (std::size_t alpha_index_ = 0; alpha_index_ < max_;
++alpha_index_)
{
const std::size_t col_ = lu_->at (alpha_index_);
if (col_ != static_cast<std::size_t>(dead_state_index))
{
chars_[col_ - dfa_offset] += static_cast<CharT>
(alpha_index_);
}
}
for (std::size_t state_index_ = 1; state_index_ < states_;
++state_index_)
{
typename detail::basic_char_state_machine<CharT>::state
*state_ = &sm_._sm_vector[start_state_index_]
[state_index_ - 1];
state_->_end_state = *read_ptr_ != 0;
state_->_id = *(read_ptr_ + id_index);
state_->_unique_id = *(read_ptr_ + unique_id_index);
state_->_state = *(read_ptr_ + state_index);
state_->_bol_index = *(read_ptr_ + bol_index) - 1;
state_->_eol_index = *(read_ptr_ + eol_index) - 1;
read_ptr_ += dfa_offset;
for (std::size_t col_index_ = 0; col_index_ < alphabet_;
++col_index_, ++read_ptr_)
{
const std::size_t transition_ = *read_ptr_;
if (transition_ != 0)
{
const std::size_t i_ = transition_ - 1;
typename detail::basic_char_state_machine<CharT>::
state::size_t_string_token_map::iterator iter_ =
state_->_transitions.find (i_);
if (iter_ == state_->_transitions.end ())
{
basic_string_token<CharT> token_
(false, chars_[col_index_]);
typename detail::basic_char_state_machine<CharT>::
state::size_t_string_token_pair pair_
(i_, token_);
state_->_transitions.insert (pair_);
}
else
{
iter_->second._charset += chars_[col_index_];
}
}
}
for (typename detail::basic_char_state_machine<CharT>::state::
size_t_string_token_map::iterator iter_ =
state_->_transitions.begin (),
end_ = state_->_transitions.end ();
iter_ != end_; ++iter_)
{
std::sort (iter_->second._charset.begin (),
iter_->second._charset.end ());
iter_->second.normalise ();
}
}
}
}
};
typedef basic_state_machine<char> state_machine;
typedef basic_state_machine<wchar_t> wstate_machine;
}
}
#endif

View File

@@ -0,0 +1,406 @@
// string_token.hpp
// Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_LEXER_STRING_TOKEN_HPP
#define BOOST_LEXER_STRING_TOKEN_HPP
#include <algorithm>
#include "size_t.hpp"
#include "consts.hpp" // num_chars, num_wchar_ts
#include <string>
#include <limits>
namespace boost
{
namespace lexer
{
template<typename CharT>
struct basic_string_token
{
typedef std::basic_string<CharT> string;
bool _negated;
string _charset;
basic_string_token () :
_negated (false)
{
}
basic_string_token (const bool negated_, const string &charset_) :
_negated (negated_),
_charset (charset_)
{
}
void remove_duplicates ()
{
const CharT *start_ = _charset.c_str ();
const CharT *end_ = start_ + _charset.size ();
// Optimisation for very large charsets:
// sorting via pointers is much quicker than
// via iterators...
std::sort (const_cast<CharT *> (start_), const_cast<CharT *> (end_));
_charset.erase (std::unique (_charset.begin (), _charset.end ()),
_charset.end ());
}
void normalise ()
{
const std::size_t max_chars_ = sizeof (CharT) == 1 ?
num_chars : num_wchar_ts;
if (_charset.length () == max_chars_)
{
_negated = !_negated;
_charset.clear ();
}
else if (_charset.length () > max_chars_ / 2)
{
negate ();
}
}
void negate ()
{
const std::size_t max_chars_ = sizeof (CharT) == 1 ?
num_chars : num_wchar_ts;
CharT curr_char_ = (std::numeric_limits<CharT>::min)();
string temp_;
const CharT *curr_ = _charset.c_str ();
const CharT *chars_end_ = curr_ + _charset.size ();
_negated = !_negated;
temp_.resize (max_chars_ - _charset.size ());
CharT *ptr_ = const_cast<CharT *> (temp_.c_str ());
std::size_t i_ = 0;
while (curr_ < chars_end_)
{
while (*curr_ > curr_char_)
{
*ptr_ = curr_char_;
++ptr_;
++curr_char_;
++i_;
}
++curr_char_;
++curr_;
++i_;
}
for (; i_ < max_chars_; ++i_)
{
*ptr_ = curr_char_;
++ptr_;
++curr_char_;
}
_charset = temp_;
}
bool operator < (const basic_string_token &rhs_) const
{
return _negated < rhs_._negated ||
(_negated == rhs_._negated && _charset < rhs_._charset);
}
bool empty () const
{
return _charset.empty () && !_negated;
}
bool any () const
{
return _charset.empty () && _negated;
}
void clear ()
{
_negated = false;
_charset.clear ();
}
void intersect (basic_string_token &rhs_, basic_string_token &overlap_)
{
if ((any () && rhs_.any ()) || (_negated == rhs_._negated &&
!any () && !rhs_.any ()))
{
intersect_same_types (rhs_, overlap_);
}
else
{
intersect_diff_types (rhs_, overlap_);
}
}
static void escape_char (const CharT ch_, string &out_)
{
switch (ch_)
{
case '\0':
out_ += '\\';
out_ += '0';
break;
case '\a':
out_ += '\\';
out_ += 'a';
break;
case '\b':
out_ += '\\';
out_ += 'b';
break;
case 27:
out_ += '\\';
out_ += 'x';
out_ += '1';
out_ += 'b';
break;
case '\f':
out_ += '\\';
out_ += 'f';
break;
case '\n':
out_ += '\\';
out_ += 'n';
break;
case '\r':
out_ += '\\';
out_ += 'r';
break;
case '\t':
out_ += '\\';
out_ += 't';
break;
case '\v':
out_ += '\\';
out_ += 'v';
break;
case '\\':
out_ += '\\';
out_ += '\\';
break;
case '"':
out_ += '\\';
out_ += '"';
break;
case '\'':
out_ += '\\';
out_ += '\'';
break;
default:
{
if (ch_ < 32 && ch_ >= 0)
{
std::basic_stringstream<CharT> ss_;
out_ += '\\';
out_ += 'x';
ss_ << std::hex <<
static_cast<std::size_t> (ch_);
out_ += ss_.str ();
}
else
{
out_ += ch_;
}
break;
}
}
}
private:
void intersect_same_types (basic_string_token &rhs_,
basic_string_token &overlap_)
{
if (any ())
{
clear ();
overlap_._negated = true;
rhs_.clear ();
}
else
{
typename string::iterator iter_ = _charset.begin ();
typename string::iterator end_ = _charset.end ();
typename string::iterator rhs_iter_ = rhs_._charset.begin ();
typename string::iterator rhs_end_ = rhs_._charset.end ();
overlap_._negated = _negated;
while (iter_ != end_ && rhs_iter_ != rhs_end_)
{
if (*iter_ < *rhs_iter_)
{
++iter_;
}
else if (*iter_ > *rhs_iter_)
{
++rhs_iter_;
}
else
{
overlap_._charset += *iter_;
iter_ = _charset.erase (iter_);
end_ = _charset.end ();
rhs_iter_ = rhs_._charset.erase (rhs_iter_);
rhs_end_ = rhs_._charset.end ();
}
}
if (_negated)
{
// duplicates already merged, so safe to merge
// using std lib.
// src, dest
merge (_charset, overlap_._charset);
// duplicates already merged, so safe to merge
// using std lib.
// src, dest
merge (rhs_._charset, overlap_._charset);
_negated = false;
rhs_._negated = false;
std::swap (_charset, rhs_._charset);
normalise ();
overlap_.normalise ();
rhs_.normalise ();
}
else if (!overlap_._charset.empty ())
{
normalise ();
overlap_.normalise ();
rhs_.normalise ();
}
}
}
void intersect_diff_types (basic_string_token &rhs_,
basic_string_token &overlap_)
{
if (any ())
{
intersect_any (rhs_, overlap_);
}
else if (_negated)
{
intersect_negated (rhs_, overlap_);
}
else // _negated == false
{
intersect_charset (rhs_, overlap_);
}
}
void intersect_any (basic_string_token &rhs_, basic_string_token &overlap_)
{
if (rhs_._negated)
{
rhs_.intersect_negated (*this, overlap_);
}
else // rhs._negated == false
{
rhs_.intersect_charset (*this, overlap_);
}
}
void intersect_negated (basic_string_token &rhs_,
basic_string_token &overlap_)
{
if (rhs_.any ())
{
overlap_._negated = true;
overlap_._charset = _charset;
rhs_._negated = false;
rhs_._charset = _charset;
clear ();
}
else // rhs._negated == false
{
rhs_.intersect_charset (*this, overlap_);
}
}
void intersect_charset (basic_string_token &rhs_,
basic_string_token &overlap_)
{
if (rhs_.any ())
{
overlap_._charset = _charset;
rhs_._negated = true;
rhs_._charset = _charset;
clear ();
}
else // rhs_._negated == true
{
typename string::iterator iter_ = _charset.begin ();
typename string::iterator end_ = _charset.end ();
typename string::iterator rhs_iter_ = rhs_._charset.begin ();
typename string::iterator rhs_end_ = rhs_._charset.end ();
while (iter_ != end_ && rhs_iter_ != rhs_end_)
{
if (*iter_ < *rhs_iter_)
{
overlap_._charset += *iter_;
rhs_iter_ = rhs_._charset.insert (rhs_iter_, *iter_);
++rhs_iter_;
rhs_end_ = rhs_._charset.end ();
iter_ = _charset.erase (iter_);
end_ = _charset.end ();
}
else if (*iter_ > *rhs_iter_)
{
++rhs_iter_;
}
else
{
++iter_;
++rhs_iter_;
}
}
if (iter_ != end_)
{
// nothing bigger in rhs_ than iter_,
// so safe to merge using std lib.
string temp_ (iter_, end_);
// src, dest
merge (temp_, overlap_._charset);
_charset.erase (iter_, end_);
}
if (!overlap_._charset.empty ())
{
merge (overlap_._charset, rhs_._charset);
// possible duplicates, so check for any and erase.
rhs_._charset.erase (std::unique (rhs_._charset.begin (),
rhs_._charset.end ()), rhs_._charset.end ());
normalise ();
overlap_.normalise ();
rhs_.normalise ();
}
}
}
void merge (string &src_, string &dest_)
{
string tmp_ (src_.size () + dest_.size (), 0);
std::merge (src_.begin (), src_.end (), dest_.begin (), dest_.end (),
tmp_.begin ());
dest_ = tmp_;
}
};
}
}
#endif

View File

@@ -0,0 +1,85 @@
/*=============================================================================
Copyright (c) 2001-2011 Joel de Guzman
http://spirit.sourceforge.net/
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/
#ifndef BOOST_SPIRIT_MAKE_CONS_OCTOBER_16_2008_1252PM
#define BOOST_SPIRIT_MAKE_CONS_OCTOBER_16_2008_1252PM
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/spirit/include/phoenix_limits.hpp> // needs to be included before proto
#include <boost/proto/proto.hpp>
#include <boost/mpl/eval_if.hpp>
#include <boost/fusion/include/cons.hpp>
#include <boost/type_traits/remove_const.hpp>
#include <boost/type_traits/is_abstract.hpp>
#include <boost/type_traits/is_function.hpp>
#include <boost/type_traits/add_reference.hpp>
#include <boost/utility/enable_if.hpp>
namespace boost { namespace spirit { namespace detail
{
template <typename T>
struct as_meta_element
: mpl::eval_if_c<is_abstract<T>::value || is_function<T>::value
, add_reference<T>, remove_const<T> >
{};
template <typename T>
struct as_meta_element<T&> : as_meta_element<T> // always store by value
{};
template <typename T, int N>
struct as_meta_element<T[N]>
{
typedef const T(&type)[N];
};
namespace result_of
{
template <typename Car, typename Cdr = fusion::nil_>
struct make_cons
{
typedef typename as_meta_element<Car>::type car_type; typedef typename fusion::cons<car_type, Cdr> type;
};
}
template <typename Car, typename Cdr>
fusion::cons<typename as_meta_element<Car>::type, Cdr>
make_cons(Car const& car, Cdr const& cdr)
{
typedef typename as_meta_element<Car>::type car_type;
typedef typename fusion::cons<car_type, Cdr> result;
return result(car, cdr);
}
template <typename Car>
fusion::cons<typename as_meta_element<Car>::type>
make_cons(Car const& car)
{
typedef typename as_meta_element<Car>::type car_type;
typedef typename fusion::cons<car_type> result;
return result(car);
}
#if defined(__GNUC__) && (__GNUC__ == 4) && (__GNUC_MINOR__ == 0)
// workaround for gcc-4.0 bug where illegal function types
// can be formed (const is added to function type)
// description: http://lists.boost.org/Archives/boost/2009/04/150743.php
template <typename Car>
fusion::cons<typename as_meta_element<Car>::type>
make_cons(Car& car, typename enable_if<is_function<Car> >::type* = 0)
{
typedef typename as_meta_element<Car>::type car_type;
typedef typename fusion::cons<car_type> result;
return result(car);
}
#endif
}}}
#endif

View File

@@ -0,0 +1,114 @@
/*=============================================================================
Copyright (c) 2001-2010 Joel de Guzman
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
==============================================================================*/
#include <boost/version.hpp>
// This is the same as the one in fusion in Boost 1.41. This is provided
// for compatibility with Boost 1.40 and below.
#if (BOOST_VERSION > 104000)
#include <boost/fusion/include/make_vector.hpp>
namespace boost { namespace spirit { namespace detail
{
namespace result_of
{
using fusion::result_of::make_vector;
}
using fusion::make_vector;
}}}
#else
#ifndef BOOST_PP_IS_ITERATING
#if !defined(SPIRIT_MAKE_VECTOR_07162005_0243)
#define SPIRIT_MAKE_VECTOR_07162005_0243
#include <boost/preprocessor/iterate.hpp>
#include <boost/preprocessor/repetition/enum_params.hpp>
#include <boost/preprocessor/repetition/enum_binary_params.hpp>
#include <boost/preprocessor/repetition/enum_params_with_a_default.hpp>
#include <boost/preprocessor/repetition/repeat_from_to.hpp>
#include <boost/fusion/container/vector/vector.hpp>
#include <boost/fusion/support/detail/as_fusion_element.hpp>
namespace boost { namespace fusion
{
struct void_;
}}
namespace boost { namespace spirit { namespace detail
{
namespace result_of
{
template <
BOOST_PP_ENUM_PARAMS_WITH_A_DEFAULT(
FUSION_MAX_VECTOR_SIZE, typename T, fusion::void_)
, typename Extra = fusion::void_
>
struct make_vector;
template <>
struct make_vector<>
{
typedef fusion::vector0 type;
};
}
inline fusion::vector0
make_vector()
{
return fusion::vector0();
}
#define BOOST_FUSION_AS_FUSION_ELEMENT(z, n, data) \
typename fusion::detail::as_fusion_element<BOOST_PP_CAT(T, n)>::type
#define BOOST_PP_FILENAME_1 <boost/spirit/home/support/detail/make_vector.hpp>
#define BOOST_PP_ITERATION_LIMITS (1, FUSION_MAX_VECTOR_SIZE)
#include BOOST_PP_ITERATE()
#undef BOOST_FUSION_AS_FUSION_ELEMENT
}}}
#endif
#else // defined(BOOST_PP_IS_ITERATING)
///////////////////////////////////////////////////////////////////////////////
//
// Preprocessor vertical repetition code
//
///////////////////////////////////////////////////////////////////////////////
#define N BOOST_PP_ITERATION()
namespace result_of
{
template <BOOST_PP_ENUM_PARAMS(N, typename T)>
#if defined(BOOST_NO_PARTIAL_SPECIALIZATION_IMPLICIT_DEFAULT_ARGS)
#define TEXT(z, n, text) , text
struct make_vector< BOOST_PP_ENUM_PARAMS(N, T) BOOST_PP_REPEAT_FROM_TO(BOOST_PP_DEC(N), FUSION_MAX_VECTOR_SIZE, TEXT, fusion::void_) >
#undef TEXT
#else
struct make_vector<BOOST_PP_ENUM_PARAMS(N, T)>
#endif
{
typedef BOOST_PP_CAT(fusion::vector, N)<BOOST_PP_ENUM(N, BOOST_FUSION_AS_FUSION_ELEMENT, _)> type;
};
}
template <BOOST_PP_ENUM_PARAMS(N, typename T)>
inline BOOST_PP_CAT(fusion::vector, N)<BOOST_PP_ENUM(N, BOOST_FUSION_AS_FUSION_ELEMENT, _)>
make_vector(BOOST_PP_ENUM_BINARY_PARAMS(N, T, const& _))
{
return BOOST_PP_CAT(fusion::vector, N)<BOOST_PP_ENUM(N, BOOST_FUSION_AS_FUSION_ELEMENT, _)>(
BOOST_PP_ENUM_PARAMS(N, _));
}
#undef N
#endif // defined(BOOST_PP_IS_ITERATING)
#endif // (BOOST_VERSION > 103800)

View File

@@ -0,0 +1,583 @@
// fp_traits.hpp
#ifndef BOOST_SPIRIT_MATH_FP_TRAITS_HPP
#define BOOST_SPIRIT_MATH_FP_TRAITS_HPP
// Copyright (c) 2006 Johan Rade
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt
// or copy at http://www.boost.org/LICENSE_1_0.txt)
#if defined(__vms) && defined(__DECCXX) && !__IEEE_FLOAT
# error The VAX floating point mode on VMS is not supported.
#endif
#if defined(_MSC_VER)
#pragma once
#endif
#include <cstring>
#include <boost/assert.hpp>
#include <boost/cstdint.hpp>
#include <boost/detail/endian.hpp>
#include <boost/static_assert.hpp>
#include <boost/type_traits/is_floating_point.hpp>
//------------------------------------------------------------------------------
namespace boost {
namespace spirit {
namespace math {
namespace detail {
//------------------------------------------------------------------------------
/*
Most processors support three different floating point precisions:
single precision (32 bits), double precision (64 bits)
and extended double precision (>64 bits)
Note that the C++ type long double can be implemented
both as double precision and extended double precision.
*/
struct single_precision_tag {};
struct double_precision_tag {};
struct extended_double_precision_tag {};
//------------------------------------------------------------------------------
/*
template<class T, class U> struct fp_traits_impl;
This is traits class that describes the binary structure of floating
point numbers of C++ type T and precision U
Requirements:
T = float, double or long double
U = single_precision_tag, double_precision_tag
or extended_double_precision_tag
Typedef members:
bits -- the target type when copying the leading bytes of a floating
point number. It is a typedef for uint32_t or uint64_t.
coverage -- tells us whether all bytes are copied or not.
It is a typedef for all_bits or not_all_bits.
Static data members:
sign, exponent, flag, mantissa -- bit masks that give the meaning of the bits
in the leading bytes.
Static function members:
init() -- initializes the static data members, if needed.
(Is a no-op in the specialized versions of the template.)
get_bits(), set_bits() -- provide access to the leading bytes.
*/
struct all_bits {};
struct not_all_bits {};
// Generic version -------------------------------------------------------------
// The generic version uses run time initialization to determine the floating
// point format. It is capable of handling most formats,
// but not the Motorola 68K extended double precision format.
// Currently the generic version is used only for extended double precision
// on Itanium. In all other cases there are specializations of the template
// that use compile time initialization.
template<class T> struct uint32_t_coverage
{
typedef not_all_bits type;
};
template<> struct uint32_t_coverage<single_precision_tag>
{
typedef all_bits type;
};
template<class T, class U> struct fp_traits_impl
{
typedef uint32_t bits;
typedef BOOST_DEDUCED_TYPENAME uint32_t_coverage<U>::type coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
static uint32_t exponent;
static uint32_t flag;
static uint32_t mantissa;
static void init()
{
if(is_init_) return;
do_init_();
is_init_ = true;
}
static void get_bits(T x, uint32_t& a)
{
memcpy(&a, reinterpret_cast<const unsigned char*>(&x) + offset_, 4);
}
static void set_bits(T& x, uint32_t a)
{
memcpy(reinterpret_cast<unsigned char*>(&x) + offset_, &a, 4);
}
private:
static size_t offset_;
static bool is_init_;
static void do_init_();
};
//..............................................................................
template<class T, class U> uint32_t fp_traits_impl<T,U>::exponent;
template<class T, class U> uint32_t fp_traits_impl<T,U>::flag;
template<class T, class U> uint32_t fp_traits_impl<T,U>::mantissa;
template<class T, class U> size_t fp_traits_impl<T,U>::offset_;
template<class T, class U> bool fp_traits_impl<T,U>::is_init_;
// In a single-threaded program, do_init will be called exactly once.
// In a multi-threaded program, do_init may be called simultaneously
// by more then one thread. That should not be a problem.
//..............................................................................
template<class T, class U> void fp_traits_impl<T,U>::do_init_()
{
T x = static_cast<T>(3) / static_cast<T>(4);
// sign bit = 0
// exponent: first and last bit = 0, all other bits = 1
// flag bit (if present) = 1
// mantissa: first bit = 1, all other bits = 0
uint32_t a;
for(size_t k = 0; k <= sizeof(T) - 4; ++k) {
memcpy(&a, reinterpret_cast<unsigned char*>(&x) + k, 4);
switch(a) {
case 0x3f400000: // IEEE single precision format
offset_ = k;
exponent = 0x7f800000;
flag = 0x00000000;
mantissa = 0x007fffff;
return;
case 0x3fe80000: // IEEE double precision format
// and PowerPC extended double precision format
offset_ = k;
exponent = 0x7ff00000;
flag = 0x00000000;
mantissa = 0x000fffff;
return;
case 0x3ffe0000: // Motorola extended double precision format
// Must not get here. Must be handled by specialization.
// To get accurate cutoff between normals and subnormals
// we must use the flag bit that is in the 5th byte.
// Otherwise this cutoff will be off by a factor 2.
// If we do get here, then we have failed to detect the Motorola
// processor at compile time.
BOOST_ASSERT(false &&
"Failed to detect the Motorola processor at compile time");
return;
case 0x3ffe8000: // IEEE extended double precision format
// with 15 exponent bits
offset_ = k;
exponent = 0x7fff0000;
flag = 0x00000000;
mantissa = 0x0000ffff;
return;
case 0x3ffec000: // Intel extended double precision format
offset_ = k;
exponent = 0x7fff0000;
flag = 0x00008000;
mantissa = 0x00007fff;
return;
default:
continue;
}
}
BOOST_ASSERT(false);
// Unknown format.
}
// float (32 bits) -------------------------------------------------------------
template<> struct fp_traits_impl<float, single_precision_tag>
{
typedef uint32_t bits;
typedef all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7f800000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0x00000000);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x007fffff);
static void init() {}
static void get_bits(float x, uint32_t& a) { memcpy(&a, &x, 4); }
static void set_bits(float& x, uint32_t a) { memcpy(&x, &a, 4); }
};
// double (64 bits) ------------------------------------------------------------
#if defined(BOOST_NO_INT64_T) || defined(BOOST_NO_INCLASS_MEMBER_INITIALIZATION)
template<> struct fp_traits_impl<double, double_precision_tag>
{
typedef uint32_t bits;
typedef not_all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7ff00000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x000fffff);
static void init() {}
static void get_bits(double x, uint32_t& a)
{
memcpy(&a, reinterpret_cast<const unsigned char*>(&x) + offset_, 4);
}
static void set_bits(double& x, uint32_t a)
{
memcpy(reinterpret_cast<unsigned char*>(&x) + offset_, &a, 4);
}
private:
#if defined(BOOST_BIG_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 0);
#elif defined(BOOST_LITTLE_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 4);
#else
BOOST_STATIC_ASSERT(false);
#endif
};
//..............................................................................
#else
template<> struct fp_traits_impl<double, double_precision_tag>
{
typedef uint64_t bits;
typedef all_bits coverage;
static const uint64_t sign = (uint64_t)0x80000000 << 32;
static const uint64_t exponent = (uint64_t)0x7ff00000 << 32;
static const uint64_t flag = 0;
static const uint64_t mantissa
= ((uint64_t)0x000fffff << 32) + (uint64_t)0xffffffff;
static void init() {}
static void get_bits(double x, uint64_t& a) { memcpy(&a, &x, 8); }
static void set_bits(double& x, uint64_t a) { memcpy(&x, &a, 8); }
};
#endif
// long double (64 bits) -------------------------------------------------------
#if defined(BOOST_NO_INT64_T) || defined(BOOST_NO_INCLASS_MEMBER_INITIALIZATION)
template<> struct fp_traits_impl<long double, double_precision_tag>
{
typedef uint32_t bits;
typedef not_all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7ff00000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x000fffff);
static void init() {}
static void get_bits(long double x, uint32_t& a)
{
memcpy(&a, reinterpret_cast<const unsigned char*>(&x) + offset_, 4);
}
static void set_bits(long double& x, uint32_t a)
{
memcpy(reinterpret_cast<unsigned char*>(&x) + offset_, &a, 4);
}
private:
#if defined(BOOST_BIG_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 0);
#elif defined(BOOST_LITTLE_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 4);
#else
BOOST_STATIC_ASSERT(false);
#endif
};
//..............................................................................
#else
template<> struct fp_traits_impl<long double, double_precision_tag>
{
typedef uint64_t bits;
typedef all_bits coverage;
static const uint64_t sign = (uint64_t)0x80000000 << 32;
static const uint64_t exponent = (uint64_t)0x7ff00000 << 32;
static const uint64_t flag = 0;
static const uint64_t mantissa
= ((uint64_t)0x000fffff << 32) + (uint64_t)0xffffffff;
static void init() {}
static void get_bits(long double x, uint64_t& a) { memcpy(&a, &x, 8); }
static void set_bits(long double& x, uint64_t a) { memcpy(&x, &a, 8); }
};
#endif
// long double (>64 bits), x86 and x64 -----------------------------------------
#if defined(__i386) || defined(__i386__) || defined(_M_IX86) \
|| defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) \
|| defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)
// Intel extended double precision format (80 bits)
template<> struct fp_traits_impl<long double, extended_double_precision_tag>
{
typedef uint32_t bits;
typedef not_all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7fff0000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0x00008000);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x00007fff);
static void init() {}
static void get_bits(long double x, uint32_t& a)
{
memcpy(&a, reinterpret_cast<const unsigned char*>(&x) + 6, 4);
}
static void set_bits(long double& x, uint32_t a)
{
memcpy(reinterpret_cast<unsigned char*>(&x) + 6, &a, 4);
}
};
// long double (>64 bits), Itanium ---------------------------------------------
#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
// The floating point format is unknown at compile time
// No template specialization is provided.
// The generic definition is used.
// The Itanium supports both
// the Intel extended double precision format (80 bits) and
// the IEEE extended double precision format with 15 exponent bits (128 bits).
// long double (>64 bits), PowerPC ---------------------------------------------
#elif defined(__powerpc) || defined(__powerpc__) || defined(__POWERPC__) \
|| defined(__ppc) || defined(__ppc__) || defined(__PPC__)
// PowerPC extended double precision format (128 bits)
template<> struct fp_traits_impl<long double, extended_double_precision_tag>
{
typedef uint32_t bits;
typedef not_all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7ff00000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0x00000000);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x000fffff);
static void init() {}
static void get_bits(long double x, uint32_t& a)
{
memcpy(&a, reinterpret_cast<const unsigned char*>(&x) + offset_, 4);
}
static void set_bits(long double& x, uint32_t a)
{
memcpy(reinterpret_cast<unsigned char*>(&x) + offset_, &a, 4);
}
private:
#if defined(BOOST_BIG_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 0);
#elif defined(BOOST_LITTLE_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 12);
#else
BOOST_STATIC_ASSERT(false);
#endif
};
// long double (>64 bits), Motorola 68K ----------------------------------------
#elif defined(__m68k) || defined(__m68k__) \
|| defined(__mc68000) || defined(__mc68000__) \
// Motorola extended double precision format (96 bits)
// It is the same format as the Intel extended double precision format,
// except that 1) it is big-endian, 2) the 3rd and 4th byte are padding, and
// 3) the flag bit is not set for infinity
template<> struct fp_traits_impl<long double, extended_double_precision_tag>
{
typedef uint32_t bits;
typedef not_all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7fff0000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0x00008000);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x00007fff);
static void init() {}
// copy 1st, 2nd, 5th and 6th byte. 3rd and 4th byte are padding.
static void get_bits(long double x, uint32_t& a)
{
memcpy(&a, &x, 2);
memcpy(reinterpret_cast<unsigned char*>(&a) + 2,
reinterpret_cast<const unsigned char*>(&x) + 4, 2);
}
static void set_bits(long double& x, uint32_t a)
{
memcpy(&x, &a, 2);
memcpy(reinterpret_cast<unsigned char*>(&x) + 4,
reinterpret_cast<const unsigned char*>(&a) + 2, 2);
}
};
// long double (>64 bits), All other processors --------------------------------
#else
// IEEE extended double precision format with 15 exponent bits (128 bits)
template<> struct fp_traits_impl<long double, extended_double_precision_tag>
{
typedef uint32_t bits;
typedef not_all_bits coverage;
BOOST_STATIC_CONSTANT(uint32_t, sign = 0x80000000);
BOOST_STATIC_CONSTANT(uint32_t, exponent = 0x7fff0000);
BOOST_STATIC_CONSTANT(uint32_t, flag = 0x00000000);
BOOST_STATIC_CONSTANT(uint32_t, mantissa = 0x0000ffff);
static void init() {}
static void get_bits(long double x, uint32_t& a)
{
memcpy(&a, reinterpret_cast<const unsigned char*>(&x) + offset_, 4);
}
static void set_bits(long double& x, uint32_t a)
{
memcpy(reinterpret_cast<unsigned char*>(&x) + offset_, &a, 4);
}
private:
#if defined(BOOST_BIG_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 0);
#elif defined(BOOST_LITTLE_ENDIAN)
BOOST_STATIC_CONSTANT(int, offset_ = 12);
#else
BOOST_STATIC_ASSERT(false);
#endif
};
#endif
//------------------------------------------------------------------------------
// size_to_precision is a type switch for converting a C++ floating point type
// to the corresponding precision type.
template<int n> struct size_to_precision;
template<> struct size_to_precision<4>
{
typedef single_precision_tag type;
};
template<> struct size_to_precision<8>
{
typedef double_precision_tag type;
};
template<> struct size_to_precision<10>
{
typedef extended_double_precision_tag type;
};
template<> struct size_to_precision<12>
{
typedef extended_double_precision_tag type;
};
template<> struct size_to_precision<16>
{
typedef extended_double_precision_tag type;
};
// fp_traits is a type switch that selects the right fp_traits_impl
template<class T> struct fp_traits
{
BOOST_STATIC_ASSERT(boost::is_floating_point<T>::value);
typedef BOOST_DEDUCED_TYPENAME size_to_precision<sizeof(T)>::type precision;
typedef fp_traits_impl<T, precision> type;
};
//------------------------------------------------------------------------------
} // namespace detail
} // namespace math
} // namespace spirit
} // namespace boost
#endif

View File

@@ -0,0 +1,235 @@
// fpclassify.hpp
#ifndef BOOST_SPIRIT_MATH_FPCLASSIFY_HPP
#define BOOST_SPIRIT_MATH_FPCLASSIFY_HPP
// Copyright (c) 2006 Johan Rade
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt
// or copy at http://www.boost.org/LICENSE_1_0.txt)
/*
The following algorithm is used:
If all exponent bits, the flag bit (if there is one),
and all mantissa bits are 0, then the number is zero.
If all exponent bits and the flag bit (if there is one) are 0,
and at least one mantissa bit is 1, then the number is subnormal.
If all exponent bits are 1 and all mantissa bits are 0,
then the number is infinity.
If all exponent bits are 1 and at least one mantissa bit is 1,
then the number is a not-a-number.
Otherwise the number is normal.
(Note that the binary representation of infinity
has flag bit 0 for Motorola 68K extended double precision,
and flag bit 1 for Intel extended double precision.)
To get the bits, the four or eight most significant bytes are copied
into an uint32_t or uint64_t and bit masks are applied.
This covers all the exponent bits and the flag bit (if there is one),
but not always all the mantissa bits.
Some of the functions below have two implementations,
depending on whether all the mantissa bits are copied or not.
*/
#if defined(_MSC_VER)
#pragma once
#endif
#include <cmath>
#ifndef FP_INFINITE
# define FP_INFINITE 0
# define FP_NAN 1
# define FP_NORMAL 2
# define FP_SUBNORMAL 3
# define FP_ZERO 4
#endif
#include <boost/spirit/home/support/detail/math/detail/fp_traits.hpp>
namespace boost {
namespace spirit {
namespace math {
//------------------------------------------------------------------------------
template<class T> bool (isfinite)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent;
return a != traits::exponent;
}
//------------------------------------------------------------------------------
template<class T> bool (isnormal)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::flag;
return (a != 0) && (a < traits::exponent);
}
//------------------------------------------------------------------------------
namespace detail {
template<class T> bool isinf_impl(T x, all_bits)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::mantissa;
return a == traits::exponent;
}
template<class T> bool isinf_impl(T x, not_all_bits)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::mantissa;
if(a != traits::exponent)
return false;
traits::set_bits(x,0);
return x == 0;
}
} // namespace detail
template<class T> bool (isinf)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
return detail::isinf_impl(x, BOOST_DEDUCED_TYPENAME traits::coverage());
}
//------------------------------------------------------------------------------
namespace detail {
template<class T> bool isnan_impl(T x, all_bits)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::mantissa;
return a > traits::exponent;
}
template<class T> bool isnan_impl(T x, not_all_bits)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::mantissa;
if(a < traits::exponent)
return false;
a &= traits::mantissa;
traits::set_bits(x,a);
return x != 0;
}
} // namespace detail
template<class T> bool (isnan)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
return detail::isnan_impl(x, BOOST_DEDUCED_TYPENAME traits::coverage());
}
//------------------------------------------------------------------------------
namespace detail {
template<class T> int fpclassify_impl(T x, all_bits)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::flag | traits::mantissa;
if(a <= traits::mantissa) {
if(a == 0)
return FP_ZERO;
else
return FP_SUBNORMAL;
}
if(a < traits::exponent)
return FP_NORMAL;
a &= traits::mantissa;
if(a == 0)
return FP_INFINITE;
return FP_NAN;
}
template<class T> int fpclassify_impl(T x, not_all_bits)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::exponent | traits::flag | traits::mantissa;
if(a <= traits::mantissa) {
if(x == 0)
return FP_ZERO;
else
return FP_SUBNORMAL;
}
if(a < traits::exponent)
return FP_NORMAL;
a &= traits::mantissa;
traits::set_bits(x,a);
if(x == 0)
return FP_INFINITE;
return FP_NAN;
}
} // namespace detail
template<class T> int (fpclassify)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
return detail::fpclassify_impl(x, BOOST_DEDUCED_TYPENAME traits::coverage());
}
//------------------------------------------------------------------------------
} // namespace math
} // namespace spirit
} // namespace boost
#endif

View File

@@ -0,0 +1,92 @@
// signbit.hpp
#ifndef BOOST_SPIRIT_MATH_SIGNBIT_HPP
#define BOOST_SPIRIT_MATH_SIGNBIT_HPP
// Copyright (c) 2006 Johan Rade
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt
// or copy at http://www.boost.org/LICENSE_1_0.txt)
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/spirit/home/support/detail/math/detail/fp_traits.hpp>
namespace boost {
namespace spirit {
namespace math {
//------------------------------------------------------------------------------
template<class T> bool (signbit)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= traits::sign;
return a != 0;
}
//------------------------------------------------------------------------------
namespace detail {
template<class T> T copysign_impl(T x, T y)
{
typedef BOOST_DEDUCED_TYPENAME fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a &= ~traits::sign;
BOOST_DEDUCED_TYPENAME traits::bits b;
traits::get_bits(y,b);
b &= traits::sign;
traits::set_bits(x,a|b);
return x;
}
}
inline float (copysign)(float x, float y) // magnitude of x and sign of y
{
return detail::copysign_impl(x,y);
}
inline double (copysign)(double x, double y)
{
return detail::copysign_impl(x,y);
}
inline long double (copysign)(long double x, long double y)
{
return detail::copysign_impl(x,y);
}
//------------------------------------------------------------------------------
template<class T> T (changesign)(T x)
{
typedef BOOST_DEDUCED_TYPENAME detail::fp_traits<T>::type traits;
traits::init();
BOOST_DEDUCED_TYPENAME traits::bits a;
traits::get_bits(x,a);
a ^= traits::sign;
traits::set_bits(x,a);
return x;
}
//------------------------------------------------------------------------------
} // namespace math
} // namespace spirit
} // namespace boost
#endif

View File

@@ -0,0 +1,113 @@
/*=============================================================================
Copyright (c) 2001-2011 Joel de Guzman
Copyright (c) 2001-2011 Hartmut Kaiser
http://spirit.sourceforge.net/
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/
#if !defined(SPIRIT_POW10_DECEMBER_26_2008_1118AM)
#define SPIRIT_POW10_DECEMBER_26_2008_1118AM
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/config/no_tr1/cmath.hpp>
#include <boost/limits.hpp>
#include <boost/spirit/home/support/unused.hpp>
#include <boost/spirit/home/support/numeric_traits.hpp>
#if BOOST_WORKAROUND(BOOST_MSVC, >= 1400)
# pragma warning(push)
# pragma warning(disable: 4244) // conversion from 'double' to 'float', possible loss of data
#endif
namespace boost { namespace spirit { namespace traits
{
template <typename T, typename Enable/* = void*/>
struct pow10_helper
{
static T call(unsigned dim)
{
using namespace std; // allow for ADL to find the correct overload
return pow(T(10), T(dim));
}
};
template <>
struct pow10_helper<unused_type>
{
static unused_type call(unused_type)
{
return unused;
}
};
#if (DBL_MAX_10_EXP == 308) // for IEEE-754
template <>
struct pow10_helper<double>
{
static double call(unsigned dim)
{
static double const exponents[] =
{
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
1e20, 1e21, 1e22, 1e23, 1e24, 1e25, 1e26, 1e27, 1e28, 1e29,
1e30, 1e31, 1e32, 1e33, 1e34, 1e35, 1e36, 1e37, 1e38, 1e39,
1e40, 1e41, 1e42, 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49,
1e50, 1e51, 1e52, 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59,
1e60, 1e61, 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69,
1e70, 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, 1e79,
1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, 1e88, 1e89,
1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, 1e97, 1e98, 1e99,
1e100, 1e101, 1e102, 1e103, 1e104, 1e105, 1e106, 1e107, 1e108, 1e109,
1e110, 1e111, 1e112, 1e113, 1e114, 1e115, 1e116, 1e117, 1e118, 1e119,
1e120, 1e121, 1e122, 1e123, 1e124, 1e125, 1e126, 1e127, 1e128, 1e129,
1e130, 1e131, 1e132, 1e133, 1e134, 1e135, 1e136, 1e137, 1e138, 1e139,
1e140, 1e141, 1e142, 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149,
1e150, 1e151, 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159,
1e160, 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, 1e169,
1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, 1e178, 1e179,
1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, 1e187, 1e188, 1e189,
1e190, 1e191, 1e192, 1e193, 1e194, 1e195, 1e196, 1e197, 1e198, 1e199,
1e200, 1e201, 1e202, 1e203, 1e204, 1e205, 1e206, 1e207, 1e208, 1e209,
1e210, 1e211, 1e212, 1e213, 1e214, 1e215, 1e216, 1e217, 1e218, 1e219,
1e220, 1e221, 1e222, 1e223, 1e224, 1e225, 1e226, 1e227, 1e228, 1e229,
1e230, 1e231, 1e232, 1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239,
1e240, 1e241, 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249,
1e250, 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, 1e259,
1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, 1e268, 1e269,
1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, 1e277, 1e278, 1e279,
1e280, 1e281, 1e282, 1e283, 1e284, 1e285, 1e286, 1e287, 1e288, 1e289,
1e290, 1e291, 1e292, 1e293, 1e294, 1e295, 1e296, 1e297, 1e298, 1e299,
1e300, 1e301, 1e302, 1e303, 1e304, 1e305, 1e306, 1e307, 1e308,
};
BOOST_ASSERT(dim < sizeof(exponents)/sizeof(double));
return exponents[dim];
}
};
template <>
struct pow10_helper<float>
{
static float call(unsigned dim)
{
return pow10_helper<double>::call(dim);
}
};
#endif // for IEEE-754
template <typename T>
inline T pow10(unsigned dim)
{
return pow10_helper<T>::call(dim);
}
}}}
#if BOOST_WORKAROUND(BOOST_MSVC, >= 1400)
# pragma warning(pop)
#endif
#endif

View File

@@ -0,0 +1,28 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
// http://spirit.sourceforge.net/
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#ifndef BOOST_SPIRIT_SCOPED_ENUM_EMULATION_HPP
#define BOOST_SPIRIT_SCOPED_ENUM_EMULATION_HPP
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/version.hpp>
#include <boost/config.hpp>
#if BOOST_VERSION >= 104000
# include <boost/detail/scoped_enum_emulation.hpp>
#else
# if !defined(BOOST_NO_CXX11_SCOPED_ENUMS)
# define BOOST_NO_CXX11_SCOPED_ENUMS
# endif
# define BOOST_SCOPED_ENUM_START(name) struct name { enum enum_type
# define BOOST_SCOPED_ENUM_END };
# define BOOST_SCOPED_ENUM(name) name::enum_type
#endif
#endif

View File

@@ -0,0 +1,71 @@
/*=============================================================================
Copyright (c) 2001-2011 Joel de Guzman
Copyright (c) 2001-2011 Hartmut Kaiser
http://spirit.sourceforge.net/
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/
#if !defined(SPIRIT_SIGN_MAR_11_2009_0734PM)
#define SPIRIT_SIGN_MAR_11_2009_0734PM
#if defined(_MSC_VER)
#pragma once
#endif
#include <boost/config/no_tr1/cmath.hpp>
#include <boost/version.hpp>
#if BOOST_VERSION < 104000
#include <boost/spirit/home/support/detail/math/fpclassify.hpp>
#include <boost/spirit/home/support/detail/math/signbit.hpp>
#else
#include <boost/math/special_functions/fpclassify.hpp>
#include <boost/math/special_functions/sign.hpp>
#endif
namespace boost { namespace spirit { namespace detail
{
#if BOOST_VERSION < 104000
// signbit(-NAN) is broken for versions of Boost earlier than 1.40.0
// This routine has been taken and adapted from Johan Rade's fp_traits
// library
template<typename T>
inline bool (signbit)(T x)
{
return (boost::spirit::math::signbit)(x);
}
template<typename T>
inline T (changesign)(T x)
{
return (boost::spirit::math::changesign)(x);
}
#else
template<typename T>
inline bool (signbit)(T x)
{
return (boost::math::signbit)(x) ? true : false;
}
// This routine has been taken and adapted from Johan Rade's fp_traits
// library
template<typename T>
inline T (changesign)(T x)
{
#if defined(BOOST_MATH_USE_STD_FPCLASSIFY) && !defined(BOOST_MATH_DISABLE_STD_FPCLASSIFY)
return -x;
#else
typedef typename math::detail::fp_traits<T>::type traits_type;
typename traits_type::bits a;
traits_type::get_bits(x, a);
a ^= traits_type::sign;
traits_type::set_bits(x, a);
return x;
#endif
}
#endif
}}}
#endif

View File

@@ -0,0 +1,48 @@
/*=============================================================================
Copyright (c) 2001-2011 Joel de Guzman
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/
#if !defined(SPIRIT_WHAT_FUNCTION_APRIL_22_2007_0236PM)
#define SPIRIT_WHAT_FUNCTION_APRIL_22_2007_0236PM
#if defined(_MSC_VER)
#pragma once
#endif
#include <string>
#include <boost/spirit/home/support/info.hpp>
#include <boost/detail/workaround.hpp>
namespace boost { namespace spirit { namespace detail
{
template <typename Context>
struct what_function
{
what_function(info& what_, Context& context_)
: what(what_), context(context_)
{
what.value = std::list<info>();
}
template <typename Component>
void operator()(Component const& component) const
{
#if BOOST_WORKAROUND(BOOST_MSVC, BOOST_TESTED_AT(1600))
component; // suppresses warning: C4100: 'component' : unreferenced formal parameter
#endif
boost::get<std::list<info> >(what.value).
push_back(component.what(context));
}
info& what;
Context& context;
private:
// silence MSVC warning C4512: assignment operator could not be generated
what_function& operator= (what_function const&);
};
}}}
#endif