diff --git a/doc/Changelog.md b/doc/Changelog.md index bee93d2d5..126cbcb48 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -17,6 +17,7 @@ * Added new atomic rule [`consume`](Rule-Reference.md#consume-count-). * Added new atomic rule [`everything`](Rule-Reference.md#everything). * Added new rule [`invert`](Rule-Reference.md#invert-r-) to convert between `one` and `not_one` etc. +* Added new rule [`is_buffer`](Rule-Reference.md#is_buffer) to allow grammars to change when using a buffer input. * Added new convenience rule [`partial`](Rule-Reference.md#partial-r-). * Added new convenience rule [`star_partial`](Rule-Reference.md#star_partial-r-). * Added new convenience rule [`strict`](Rule-Reference.md#strict-r-). diff --git a/doc/Development.md b/doc/Development.md new file mode 100644 index 000000000..45b36f032 --- /dev/null +++ b/doc/Development.md @@ -0,0 +1,56 @@ +# Development + +### C++ Standard + +Version 0.x of the PEGTL requires at least C++11. + +Version 1.x of the PEGTL requires at least C++11. + +Version 2.x of the PEGTL requires at least C++11. + +Version 3.x of the PEGTL requires at least C++17. + +Version 4.x of the PEGTL requires at least C++17. + +Version 5.x of the PEGTL will make the jump to C++20 or even C++23. + + * Add support for C++20 `char8_t` where appropriate. + * Use C++20 `std::span` in inputs and everywhere else it makes sense. + * Investigate whether there is anything useful we can do with Ranges in the PEGTL. + * Use C++20 Concepts instead of all the SFINAE and meta-programming where possible. + * Give examples for C++20 "lambdas in unevaluated contexts" in conjunction with `tao::pegtl::function`. + * Keep an open eye for opportunities to use C++20 spaceship operator. Spaceship! + * Keep an open eye for opportunities to use C++20 defaulted comparison operators. + * Keep an open eye for opportunities to use C++20 `[[likely]]` and `[[unlikely]]`. + * Keep an open eye for opportunities to use C++20 `constinit` and `consteval`, and + * keep an open eye for opportunities to use the extended `constexpr` facilities. + * keep an open eye for opportunities to use the extended CTAD facilities from C++20. + * Keep an open eye for opportunities to use class types as non-type template parameters. + * Replace the hand-crafted endian facilities with C++20 `std::endian` and C++23 `std::byteswap`. + * Investigate how C++20 and C++23 compile-time facilities can help with compile-time strings. + * Investigate whether we can use C++20 `std::bit_cast` to improve some of the low-level code. + * Use C++23 "deducing this" feature to let base class `make_rewind_guard()` return a rewind guard for a derived class. + * Can we assume the C++17 `charconv` facilities are universally available? Can we do this for 4.x? + +### Other Things + + * Build a compile-time facility to convert Unicode code points to UTF8 sequences! + * Investigate whether we are crazy enough to attempt parsing linked lists or trees. + +### Buffer Inputs + +A couple of things that could be done in the area of buffer inputs. + + * Optional automatic discard. + * Use the double-mmap ring-buffer to prevent `discard()` having to copy data within the buffer. + * Debug input and related facilities that detect when data in the input buffer is accessed after being discarded and/or moved by another discard. + * Investigate the use of ("stackful") coroutines for parsing from a network socket, and + * investigate whether this can also be used for incremental parsing that keeps everything. + +--- + +This document is part of the [PEGTL](https://github.com/taocpp/PEGTL). + +Copyright (c) 2023 Dr. Colin Hirsch and Daniel Frey +Distributed under the Boost Software License, Version 1.0
+See accompanying file [LICENSE_1_0.txt](../LICENSE_1_0.txt) or copy at https://www.boost.org/LICENSE_1_0.txt diff --git a/doc/README.md b/doc/README.md index f85cef0d8..184de7e9f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -121,6 +121,7 @@ * [Limitations](Grammar-Analysis.md#limitations) * [Changelog](Changelog.md) * [Migration Guide](Migration-Guide.md) +* [Development](Development.md) ### Rule Reference Index diff --git a/include/tao/pegtl/buffer.hpp b/include/tao/pegtl/buffer.hpp new file mode 100644 index 000000000..0b2484898 --- /dev/null +++ b/include/tao/pegtl/buffer.hpp @@ -0,0 +1,106 @@ +// Copyright (c) 2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_BUFFER_HPP +#define TAO_PEGTL_BUFFER_HPP + +#include +#include + +#include "config.hpp" + +#include "internal/buffer_input.hpp" +#include "internal/dynamic_buffer.hpp" +#include "internal/static_buffer.hpp" + +#include "internal/input_with_peeks.hpp" +#include "internal/input_with_source.hpp" + +#include "internal/cstream_reader.hpp" +#include "internal/cstring_reader.hpp" +#include "internal/istream_reader.hpp" + +#if !defined TAO_PEGTL_DEFAULT_BUFFER_SIZE +#define TAO_PEGTL_DEFAULT_BUFFER_SIZE 4000 +#endif + +#if !defined TAO_PEGTL_DEFAULT_CHUNK_SIZE +#define TAO_PEGTL_DEFAULT_CHUNK_SIZE 1000 +#endif + +namespace TAO_PEGTL_NAMESPACE +{ + template< typename Reader > + struct dynamic_input + : internal::input_with_peeks< internal::buffer_input< internal::dynamic_buffer< char, Reader > > > + { + using internal::input_with_peeks< internal::buffer_input< internal::dynamic_buffer< char, Reader > > >::input_with_peeks; + }; + + dynamic_input( const std::size_t, const std::size_t, std::FILE* ) -> dynamic_input< internal::cstream_reader >; + dynamic_input( const std::size_t, const std::size_t, const char* ) -> dynamic_input< internal::cstring_reader >; + dynamic_input( const std::size_t, const std::size_t, std::istream& ) -> dynamic_input< internal::istream_reader >; + + using dynamic_cstream_input = dynamic_input< internal::cstream_reader >; + using dynamic_cstring_input = dynamic_input< internal::cstring_reader >; + using dynamic_istream_input = dynamic_input< internal::istream_reader >; + + template< typename Reader, std::size_t BufferSize = TAO_PEGTL_DEFAULT_BUFFER_SIZE, std::size_t ChunkSize = TAO_PEGTL_DEFAULT_CHUNK_SIZE > + struct static_input + : internal::input_with_peeks< internal::buffer_input< internal::static_buffer< char, Reader, BufferSize, ChunkSize > > > + { + using internal::input_with_peeks< internal::buffer_input< internal::static_buffer< char, Reader, BufferSize, ChunkSize > > >::input_with_peeks; + }; + + static_input( std::FILE* ) -> static_input< internal::cstream_reader >; + static_input( const char* ) -> static_input< internal::cstring_reader >; + static_input( std::istream& ) -> static_input< internal::istream_reader >; + + template< std::size_t BufferSize = TAO_PEGTL_DEFAULT_BUFFER_SIZE, std::size_t ChunkSize = TAO_PEGTL_DEFAULT_CHUNK_SIZE > + using static_cstream_input = static_input< internal::cstream_reader, BufferSize, ChunkSize >; + + template< std::size_t BufferSize = TAO_PEGTL_DEFAULT_BUFFER_SIZE, std::size_t ChunkSize = TAO_PEGTL_DEFAULT_CHUNK_SIZE > + using static_cstring_input = static_input< internal::cstring_reader, BufferSize, ChunkSize >; + + template< std::size_t BufferSize = TAO_PEGTL_DEFAULT_BUFFER_SIZE, std::size_t ChunkSize = TAO_PEGTL_DEFAULT_CHUNK_SIZE > + using static_istream_input = static_input< internal::istream_reader, BufferSize, ChunkSize >; + +} // namespace TAO_PEGTL_NAMESPACE + +#include "analyze_traits.hpp" + +#include "internal/discard.hpp" +#include "internal/is_buffer.hpp" +#include "internal/require.hpp" + +namespace TAO_PEGTL_NAMESPACE +{ + // clang-format off + struct discard : internal::discard {}; + struct is_buffer : internal::is_buffer {}; + template< unsigned Amount > struct require : internal::require< Amount > {}; + // clang-format on + + template< typename Name > + struct analyze_traits< Name, internal::discard > + : analyze_opt_traits<> + {}; + + template< typename Name > + struct analyze_traits< Name, internal::is_buffer > + : analyze_opt_traits<> + {}; + + template< typename Name, unsigned Amount > + struct analyze_traits< Name, internal::require< Amount > > + : analyze_opt_traits<> + {}; + +} // namespace TAO_PEGTL_NAMESPACE + +#include "discard_input.hpp" +#include "discard_input_on_failure.hpp" +#include "discard_input_on_success.hpp" + +#endif diff --git a/include/tao/pegtl/demangle.hpp b/include/tao/pegtl/demangle.hpp index aa62100d0..cbcfc2711 100644 --- a/include/tao/pegtl/demangle.hpp +++ b/include/tao/pegtl/demangle.hpp @@ -11,7 +11,8 @@ namespace TAO_PEGTL_NAMESPACE { - // ensure a consistent interface + // Ensure a consistent interface. + template< typename T > [[nodiscard]] constexpr std::string_view demangle() noexcept; @@ -35,6 +36,7 @@ template< typename T > namespace TAO_PEGTL_NAMESPACE::internal { // When using libstdc++ with clang, std::string_view::find is not constexpr :( + template< char C > constexpr const char* string_view_find( const char* p, std::size_t n ) noexcept { @@ -73,10 +75,11 @@ template< typename T > // GCC 9.1 and 9.2 have a bug that leads to truncated __PRETTY_FUNCTION__ names, // see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=91155 + template< typename T > [[nodiscard]] constexpr std::string_view TAO_PEGTL_NAMESPACE::demangle() noexcept { - // fallback: requires RTTI, no demangling + // Fallback: Requires RTTI, no demangling. return typeid( T ).name(); } @@ -113,8 +116,8 @@ template< typename T > template< typename T > [[nodiscard]] constexpr std::string_view TAO_PEGTL_NAMESPACE::demangle() noexcept { - // we can not add static_assert for additional safety, - // see issues #296, #301 and #308 + // We can not add static_assert for additional safety, + // see issues #296, #301 and #308. constexpr std::string_view sv = __FUNCSIG__; constexpr auto begin = sv.find( "demangle<" ); constexpr auto tmp = sv.substr( begin + 9 ); @@ -133,7 +136,7 @@ template< typename T > template< typename T > [[nodiscard]] constexpr std::string_view TAO_PEGTL_NAMESPACE::demangle() noexcept { - // fallback: requires RTTI, no demangling + // Fallback: Requires RTTI, no demangling. return typeid( T ).name(); } diff --git a/include/tao/pegtl/discard_input.hpp b/include/tao/pegtl/discard_input.hpp new file mode 100644 index 000000000..0cac8ecc6 --- /dev/null +++ b/include/tao/pegtl/discard_input.hpp @@ -0,0 +1,38 @@ +// Copyright (c) 2019-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_DISCARD_INPUT_HPP +#define TAO_PEGTL_DISCARD_INPUT_HPP + +#include "apply_mode.hpp" +#include "config.hpp" +#include "match.hpp" +#include "nothing.hpp" +#include "rewind_mode.hpp" + +namespace TAO_PEGTL_NAMESPACE +{ + struct discard_input + : maybe_nothing + { + template< typename Rule, + apply_mode A, + rewind_mode M, + template< typename... > + class Action, + template< typename... > + class Control, + typename ParseInput, + typename... States > + [[nodiscard]] static bool match( ParseInput& in, States&&... st ) + { + const bool result = TAO_PEGTL_NAMESPACE::match< Rule, A, M, Action, Control >( in, st... ); + in.discard(); + return result; + } + }; + +} // namespace TAO_PEGTL_NAMESPACE + +#endif diff --git a/include/tao/pegtl/discard_input_on_failure.hpp b/include/tao/pegtl/discard_input_on_failure.hpp new file mode 100644 index 000000000..12cea02e1 --- /dev/null +++ b/include/tao/pegtl/discard_input_on_failure.hpp @@ -0,0 +1,40 @@ +// Copyright (c) 2019-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_DISCARD_INPUT_ON_FAILURE_HPP +#define TAO_PEGTL_DISCARD_INPUT_ON_FAILURE_HPP + +#include "apply_mode.hpp" +#include "config.hpp" +#include "match.hpp" +#include "nothing.hpp" +#include "rewind_mode.hpp" + +namespace TAO_PEGTL_NAMESPACE +{ + struct discard_input_on_failure + : maybe_nothing + { + template< typename Rule, + apply_mode A, + rewind_mode M, + template< typename... > + class Action, + template< typename... > + class Control, + typename ParseInput, + typename... States > + [[nodiscard]] static bool match( ParseInput& in, States&&... st ) + { + const bool result = TAO_PEGTL_NAMESPACE::match< Rule, A, M, Action, Control >( in, st... ); + if( !result ) { + in.discard(); + } + return result; + } + }; + +} // namespace TAO_PEGTL_NAMESPACE + +#endif diff --git a/include/tao/pegtl/discard_input_on_success.hpp b/include/tao/pegtl/discard_input_on_success.hpp new file mode 100644 index 000000000..0ed60d8bd --- /dev/null +++ b/include/tao/pegtl/discard_input_on_success.hpp @@ -0,0 +1,40 @@ +// Copyright (c) 2019-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_DISCARD_INPUT_ON_SUCCESS_HPP +#define TAO_PEGTL_DISCARD_INPUT_ON_SUCCESS_HPP + +#include "apply_mode.hpp" +#include "config.hpp" +#include "match.hpp" +#include "nothing.hpp" +#include "rewind_mode.hpp" + +namespace TAO_PEGTL_NAMESPACE +{ + struct discard_input_on_success + : maybe_nothing + { + template< typename Rule, + apply_mode A, + rewind_mode M, + template< typename... > + class Action, + template< typename... > + class Control, + typename ParseInput, + typename... States > + [[nodiscard]] static bool match( ParseInput& in, States&&... st ) + { + const bool result = TAO_PEGTL_NAMESPACE::match< Rule, A, M, Action, Control >( in, st... ); + if( result ) { + in.discard(); + } + return result; + } + }; + +} // namespace TAO_PEGTL_NAMESPACE + +#endif diff --git a/include/tao/pegtl/internal/buffer_input.hpp b/include/tao/pegtl/internal/buffer_input.hpp new file mode 100644 index 000000000..64843d838 --- /dev/null +++ b/include/tao/pegtl/internal/buffer_input.hpp @@ -0,0 +1,198 @@ +// Copyright (c) 2022-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_BUFFER_INPUT_HPP +#define TAO_PEGTL_INTERNAL_BUFFER_INPUT_HPP + +#include +#include +#include + +#if defined( __cpp_exceptions ) +#include +#include +#else +#include +#endif + +#include "../config.hpp" +#include "../count_position.hpp" +#include "../pointer_position.hpp" + +#include "rewind_guard.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + template< typename Base > + class buffer_input + : public Base + { + public: + using data_t = typename Base::data_t; + using error_position_t = count_position; + using rewind_position_t = pointer_position< data_t >; + + template< typename... As > + buffer_input( As&&... as ) + : Base( std::forward< As >( as )... ), + m_current( this->mutable_begin() ), + m_end( this->mutable_begin() ) + { + // assert( buffer_chunk_size() > 0 ); + // assert( buffer_chunk_size() < buffer_capacity() ); + } + + buffer_input( buffer_input&& ) = delete; + buffer_input( const buffer_input& ) = delete; + + ~buffer_input() = default; + + void operator=( buffer_input&& ) = delete; + void operator=( const buffer_input& ) = delete; + + [[nodiscard]] bool empty() + { + return size( 1 ) == 0; + } + + [[nodiscard]] std::size_t size( const std::size_t amount ) + { + require( amount ); + return this->buffer_size(); + } + + [[nodiscard]] std::size_t size( const std::size_t minimum, const std::size_t maximum ) + { + require( maximum ); + buffer_check_size( minimum ); + return this->buffer_size(); + } + + [[nodiscard]] const data_t* current( const std::size_t offset = 0 ) const noexcept + { + return m_current + offset; + } + + [[nodiscard]] const data_t* previous( const rewind_position_t saved ) const noexcept + { + return saved.data; + } + + [[nodiscard]] const data_t* end( const std::size_t amount ) + { + require( amount ); + return m_end; + } + + [[nodiscard]] const data_t* end( const std::size_t minimum, const std::size_t maximum ) + { + require( maximum ); + buffer_check_size( minimum ); + return m_end; + } + + template< typename Rule > + void consume( const std::size_t count ) noexcept + { + m_current += count; + m_position.count += count; + } + + void discard() noexcept + { + const std::size_t s = m_end - m_current; + + if( ( s > 0 ) && ( m_current > this->buffer_begin() + this->buffer_chunk_size() ) ) { + std::memmove( this->mutable_begin(), m_current, s ); + } + m_current = this->mutable_begin(); + m_end = m_current + s; + } + + void require( const std::size_t amount ) + { + if( m_current + amount <= m_end ) { + return; + } + if( ( m_current + amount ) > ( this->buffer_begin() + this->buffer_capacity() ) ) { +#if defined( __cpp_exceptions ) + throw std::overflow_error( "require() beyond end of buffer" ); +#else + std::fputs( "overflow error: require() beyond end of buffer\n", stderr ); + std::terminate(); +#endif + } + m_end += this->m_reader( const_cast< data_t* >( m_end ), ( std::min )( buffer_free_after_end(), ( std::max )( amount - buffer_size(), this->buffer_chunk_size() ) ) ); + } + + void buffer_check_size( const std::size_t amount ) + { + if( this->buffer_size() < amount ) { +#if defined( __cpp_exceptions ) + throw std::overflow_error( "require() beyond end of reader" ); +#else + std::fputs( "overflow error: require() beyond end of reader\n", stderr ); + std::terminate(); +#endif + } + } + + [[nodiscard]] std::size_t buffer_size() const noexcept + { + // assert( m_end >= m_current ); + return m_end - m_current; + } + + [[nodiscard]] std::size_t buffer_free_before_current() const noexcept + { + // assert( m_current >= this->buffer_begin() ); + return std::size_t( m_current - this->buffer_begin() ); + } + + [[nodiscard]] std::size_t buffer_free_after_end() const noexcept + { + // assert( this->buffer_begin() + m_size >= m_end ); + return std::size_t( this->buffer_begin() + this->buffer_capacity() - m_end ); + } + + template< rewind_mode M > + [[nodiscard]] auto make_rewind_guard() noexcept + { + return rewind_guard< M, buffer_input >( this ); + } + + [[nodiscard]] auto rewind_position() const noexcept + { + return rewind_position_t( m_current ); + } + + void rewind_to_position( const rewind_position_t saved ) noexcept + { + m_current = saved.data; + } + + [[nodiscard]] error_position_t current_position() const noexcept + { + return m_position; + } + + [[nodiscard]] error_position_t previous_position( const rewind_position_t saved ) const noexcept + { + return error_position_t( m_position.count + saved.data - m_current ); + } + + [[nodiscard]] error_position_t direct_position() const noexcept + { + return m_position; + } + + protected: + error_position_t m_position; + const data_t* m_current; + const data_t* m_end; + }; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/copy_input.hpp b/include/tao/pegtl/internal/copy_input.hpp index e86675228..bbc26f81a 100644 --- a/include/tao/pegtl/internal/copy_input.hpp +++ b/include/tao/pegtl/internal/copy_input.hpp @@ -100,7 +100,7 @@ namespace TAO_PEGTL_NAMESPACE::internal template< rewind_mode M > [[nodiscard]] auto make_rewind_guard() noexcept { - return rewind_guard< M, copy_input >( this ); // TODO: With C++23 "deducing this" we don't need to re-implement this in derived classes that change the rewind position. + return rewind_guard< M, copy_input >( this ); } [[nodiscard]] auto rewind_position() const noexcept diff --git a/include/tao/pegtl/internal/cstream_reader.hpp b/include/tao/pegtl/internal/cstream_reader.hpp new file mode 100644 index 000000000..ace1a07bb --- /dev/null +++ b/include/tao/pegtl/internal/cstream_reader.hpp @@ -0,0 +1,59 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_CSTREAM_READER_HPP +#define TAO_PEGTL_INTERNAL_CSTREAM_READER_HPP + +#include +#include +#include + +#if defined( __cpp_exceptions ) +#include +#else +#include +#endif + +#include "../config.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + struct cstream_reader + { + explicit cstream_reader( std::FILE* s ) noexcept + : m_cstream( s ) + { + assert( m_cstream != nullptr ); + } + + [[nodiscard]] std::size_t operator()( char* buffer, const std::size_t length ) const + { + if( const auto r = std::fread( buffer, 1, length, m_cstream ) ) { + return r; + } + if( std::feof( m_cstream ) != 0 ) { + return 0; + } + + // Please contact us if you know how to provoke the following exception. + // The example on cppreference.com doesn't work, at least not on macOS. + + // LCOV_EXCL_START +#if defined( __cpp_exceptions ) + const auto ec = std::ferror( m_cstream ); + assert( ec != 0 ); + throw std::system_error( ec, std::system_category(), "std::fread() failed" ); +#else + std::fputs( "std::fread() failed\n", stderr ); + std::terminate(); +#endif + // LCOV_EXCL_STOP + } + + std::FILE* m_cstream; + }; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/cstring_reader.hpp b/include/tao/pegtl/internal/cstring_reader.hpp new file mode 100644 index 000000000..4a680cd92 --- /dev/null +++ b/include/tao/pegtl/internal/cstring_reader.hpp @@ -0,0 +1,41 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_CSTRING_READER_HPP +#define TAO_PEGTL_INTERNAL_CSTRING_READER_HPP + +#include +#include + +#include "../config.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + struct cstring_reader + { + explicit cstring_reader( const char* zero_terminated ) noexcept + : m_cstring( zero_terminated ) + { + assert( m_cstring != nullptr ); + } + + [[nodiscard]] std::size_t operator()( char* buffer, const std::size_t length ) noexcept + { + std::size_t i = 0; + char c; + + while( ( i < length ) && ( ( c = m_cstring[ i ] ) != 0 ) ) { + *buffer++ = c; + ++i; + } + m_cstring += i; + return i; + } + + const char* m_cstring; + }; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/discard.hpp b/include/tao/pegtl/internal/discard.hpp new file mode 100644 index 000000000..978e39727 --- /dev/null +++ b/include/tao/pegtl/internal/discard.hpp @@ -0,0 +1,33 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_DISCARD_HPP +#define TAO_PEGTL_INTERNAL_DISCARD_HPP + +#include "../config.hpp" +#include "../type_list.hpp" + +#include "enable_control.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + struct discard + { + using rule_t = discard; + using subs_t = empty_list; + + template< typename ParseInput > + [[nodiscard]] static bool match( ParseInput& in ) noexcept( noexcept( in.discard() ) ) + { + in.discard(); + return true; + } + }; + + template<> + inline constexpr bool enable_control< discard > = false; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/dynamic_buffer.hpp b/include/tao/pegtl/internal/dynamic_buffer.hpp new file mode 100644 index 000000000..b6d8c032e --- /dev/null +++ b/include/tao/pegtl/internal/dynamic_buffer.hpp @@ -0,0 +1,72 @@ +// Copyright (c) 2022-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_DYNAMIC_BUFFER_HPP +#define TAO_PEGTL_INTERNAL_DYNAMIC_BUFFER_HPP + +#include +#include +#include +#include + +#include "../config.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + template< typename Data, typename Reader > + class dynamic_buffer + { + public: + using data_t = Data; + + template< typename... As > + dynamic_buffer( const std::size_t size, const std::size_t chunk, As&&... as ) + : m_size( size ), + m_chunk( chunk ), + m_buffer( new Data[ size ] ), + m_reader( std::forward< As >( as )... ) + { + assert( chunk > 0 ); + assert( size > chunk ); + } + + dynamic_buffer( dynamic_buffer&& ) = delete; + dynamic_buffer( const dynamic_buffer& ) = delete; + + ~dynamic_buffer() = default; + + void operator=( dynamic_buffer&& ) = delete; + void operator=( const dynamic_buffer& ) = delete; + + [[nodiscard]] Data* mutable_begin() noexcept + { + return m_buffer.get(); + } + + [[nodiscard]] const Data* buffer_begin() const noexcept + { + return m_buffer.get(); + } + + [[nodiscard]] std::size_t buffer_capacity() const noexcept + { + return m_size; + } + + [[nodiscard]] std::size_t buffer_chunk_size() const noexcept + { + return m_chunk; + } + + protected: + const std::size_t m_size; + const std::size_t m_chunk; + const std::unique_ptr< Data[] > m_buffer; + + Reader m_reader; + }; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/function.hpp b/include/tao/pegtl/internal/function.hpp index 52c339634..68f4c7eed 100644 --- a/include/tao/pegtl/internal/function.hpp +++ b/include/tao/pegtl/internal/function.hpp @@ -19,6 +19,19 @@ namespace TAO_PEGTL_NAMESPACE::internal template< typename Func, Func Tion, typename Peek > struct function; + template< bool E, typename Input, bool ( *Tion )( Input& ) noexcept( E ) > + struct function< bool ( * )( Input& ) noexcept( E ), Tion, void > + { + using rule_t = function; + using subs_t = empty_list; + + template< typename ParseInput > + [[nodiscard]] static bool match( ParseInput& in ) noexcept( E ) + { + return Tion( in ); + } + }; + template< bool E, typename Input, typename... States, bool ( *Tion )( Input&, States... ) noexcept( E ) > struct function< bool ( * )( Input&, States... ) noexcept( E ), Tion, void > { diff --git a/include/tao/pegtl/internal/has_buffer_size.hpp b/include/tao/pegtl/internal/has_buffer_size.hpp new file mode 100644 index 000000000..e4ebe11c5 --- /dev/null +++ b/include/tao/pegtl/internal/has_buffer_size.hpp @@ -0,0 +1,24 @@ +// Copyright (c) 2022-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_HAS_BUFFER_SIZE_HPP +#define TAO_PEGTL_INTERNAL_HAS_BUFFER_SIZE_HPP + +#include + +#include "../config.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + template< typename, typename = void > + inline constexpr bool has_buffer_size = false; + + template< typename C > + inline constexpr bool has_buffer_size< C, decltype( (void)std::declval< C >().buffer_size(), void() ) > = true; + + // The (void) is to shut up a warning from GCC 9 and 10 about the return value of the nodiscard-function buffer_size() being ignored. + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/is_buffer.hpp b/include/tao/pegtl/internal/is_buffer.hpp new file mode 100644 index 000000000..7451aacaf --- /dev/null +++ b/include/tao/pegtl/internal/is_buffer.hpp @@ -0,0 +1,33 @@ +// Copyright (c) 2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_IS_BUFFER_HPP +#define TAO_PEGTL_INTERNAL_IS_BUFFER_HPP + +#include "../config.hpp" +#include "../type_list.hpp" + +#include "enable_control.hpp" +#include "has_buffer_size.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + struct is_buffer + { + using rule_t = is_buffer; + using subs_t = empty_list; + + template< typename ParseInput > + [[nodiscard]] static bool match( ParseInput& /*unused*/ ) noexcept + { + return has_buffer_size< ParseInput >; + } + }; + + template<> + inline constexpr bool enable_control< is_buffer > = false; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/istream_reader.hpp b/include/tao/pegtl/internal/istream_reader.hpp new file mode 100644 index 000000000..8c4356e7b --- /dev/null +++ b/include/tao/pegtl/internal/istream_reader.hpp @@ -0,0 +1,51 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_ISTREAM_READER_HPP +#define TAO_PEGTL_INTERNAL_ISTREAM_READER_HPP + +#include + +#if defined( __cpp_exceptions ) +#include +#else +#include +#include +#endif + +#include "../config.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + struct istream_reader + { + explicit istream_reader( std::istream& s ) noexcept + : m_istream( s ) + {} + + [[nodiscard]] std::size_t operator()( char* buffer, const std::size_t length ) + { + m_istream.read( buffer, static_cast< std::streamsize >( length ) ); + + if( const auto r = m_istream.gcount() ) { + return static_cast< std::size_t >( r ); + } + if( m_istream.eof() ) { + return 0; + } +#if defined( __cpp_exceptions ) + const auto ec = errno; + throw std::system_error( ec, std::system_category(), "std::istream::read() failed" ); +#else + std::fputs( "std::istream::read() failed\n", stderr ); + std::terminate(); +#endif + } + + std::istream& m_istream; + }; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/require.hpp b/include/tao/pegtl/internal/require.hpp new file mode 100644 index 000000000..8fd2a372f --- /dev/null +++ b/include/tao/pegtl/internal/require.hpp @@ -0,0 +1,42 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_REQUIRE_HPP +#define TAO_PEGTL_INTERNAL_REQUIRE_HPP + +#include "../config.hpp" +#include "../type_list.hpp" + +#include "enable_control.hpp" +#include "success.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + template< unsigned Amount > + struct require; + + template<> + struct require< 0 > + : success + {}; + + template< unsigned Amount > + struct require + { + using rule_t = require; + using subs_t = empty_list; + + template< typename ParseInput > + [[nodiscard]] static bool match( ParseInput& in ) noexcept( noexcept( in.size( 1 ) ) ) + { + return in.size( Amount ) >= Amount; + } + }; + + template< unsigned Amount > + inline constexpr bool enable_control< require< Amount > > = false; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/static_buffer.hpp b/include/tao/pegtl/internal/static_buffer.hpp new file mode 100644 index 000000000..73839ac50 --- /dev/null +++ b/include/tao/pegtl/internal/static_buffer.hpp @@ -0,0 +1,66 @@ +// Copyright (c) 2022-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#ifndef TAO_PEGTL_INTERNAL_STATIC_BUFFER_HPP +#define TAO_PEGTL_INTERNAL_STATIC_BUFFER_HPP + +#include +#include +#include + +#include "../config.hpp" + +namespace TAO_PEGTL_NAMESPACE::internal +{ + template< typename Data, typename Reader, std::size_t Size, std::size_t Chunk > + class static_buffer + { + public: + using data_t = Data; + + static_assert( Chunk > 0 ); + static_assert( Chunk < Size ); + + template< typename... As > + static_buffer( As&&... as ) + : m_reader( std::forward< As >( as )... ) + {} + + static_buffer( static_buffer&& ) = delete; + static_buffer( const static_buffer& ) = delete; + + ~static_buffer() = default; + + void operator=( static_buffer&& ) = delete; + void operator=( const static_buffer& ) = delete; + + [[nodiscard]] Data* mutable_begin() noexcept + { + return m_buffer.data(); + } + + [[nodiscard]] const Data* buffer_begin() const noexcept + { + return m_buffer.data(); + } + + [[nodiscard]] std::size_t buffer_capacity() const noexcept + { + return m_buffer.size(); + } + + [[nodiscard]] std::size_t buffer_chunk_size() const noexcept + { + return Chunk; + } + + protected: + std::array< Data, Size > m_buffer; + + Reader m_reader; + }; + +} // namespace TAO_PEGTL_NAMESPACE::internal + +#endif diff --git a/include/tao/pegtl/internal/view_input.hpp b/include/tao/pegtl/internal/view_input.hpp index c3320a9d6..234fea48c 100644 --- a/include/tao/pegtl/internal/view_input.hpp +++ b/include/tao/pegtl/internal/view_input.hpp @@ -114,7 +114,7 @@ namespace TAO_PEGTL_NAMESPACE::internal template< rewind_mode M > [[nodiscard]] auto make_rewind_guard() noexcept { - return rewind_guard< M, view_input >( this ); // TODO: With C++23 "deducing this" we don't need to re-implement this in derived classes that change the rewind position. + return rewind_guard< M, view_input >( this ); } [[nodiscard]] auto rewind_position() const noexcept diff --git a/src/test/pegtl/argv_input.cc b/src/test/pegtl/argv_input.cpp similarity index 84% rename from src/test/pegtl/argv_input.cc rename to src/test/pegtl/argv_input.cpp index b6788378f..c94d96ffe 100644 --- a/src/test/pegtl/argv_input.cc +++ b/src/test/pegtl/argv_input.cpp @@ -13,8 +13,8 @@ namespace TAO_PEGTL_NAMESPACE char data[ 12 ]; std::memcpy( data, "foo\0bar\0baz", 12 ); char* argv[] = { data, data + 4, data + 8 }; - argv_input in( argv, 1 ); - TAO_PEGTL_TEST_ASSERT( in.source() == "argv[1]" ); + argv_input_with_source in( argv, 1 ); + TAO_PEGTL_TEST_ASSERT( in.direct_source() == "argv[1]" ); const auto result = parse< string< 'b', 'a', 'r' > >( in ); TAO_PEGTL_TEST_ASSERT( result ); } diff --git a/src/test/pegtl/contrib_coverage.cc b/src/test/pegtl/contrib_coverage.cpp similarity index 96% rename from src/test/pegtl/contrib_coverage.cc rename to src/test/pegtl/contrib_coverage.cpp index ff9d3a3d1..3718812cd 100644 --- a/src/test/pegtl/contrib_coverage.cc +++ b/src/test/pegtl/contrib_coverage.cpp @@ -5,6 +5,7 @@ #include #include "test.hpp" +#include "test_inputs.hpp" #include #include @@ -30,7 +31,7 @@ namespace TAO_PEGTL_NAMESPACE { const std::string data = "F"; coverage_result result; - memory_input in( data, __FILE__ ); + test::text_input< ascii::lf > in( data ); const bool success = coverage< grammar >( in, result ); std::cout << result; // To manually see that printing does the right thing, too. TAO_PEGTL_TEST_ASSERT( success ); @@ -50,7 +51,7 @@ namespace TAO_PEGTL_NAMESPACE { const std::string data = "F"; coverage_result result; - memory_input in( data, __FILE__ ); + test::text_input< ascii::lf > in( data ); const bool success = coverage< grammar >( in, result ); std::cout << result; // To manually see that printing does the right thing, too. TAO_PEGTL_TEST_ASSERT( success ); diff --git a/src/test/pegtl/contrib_separated_seq.cc b/src/test/pegtl/contrib_separated_seq.cpp similarity index 97% rename from src/test/pegtl/contrib_separated_seq.cc rename to src/test/pegtl/contrib_separated_seq.cpp index 870237d36..8274e4002 100644 --- a/src/test/pegtl/contrib_separated_seq.cc +++ b/src/test/pegtl/contrib_separated_seq.cpp @@ -2,10 +2,10 @@ // Distributed under the Boost Software License, Version 1.0. // (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) -#include - #include +#include + // clang-format off struct A {}; struct B {}; @@ -16,10 +16,13 @@ struct S {}; // clang-format on using namespace TAO_PEGTL_NAMESPACE; + static_assert( std::is_base_of_v< internal::seq<>, separated_seq< S > > ); static_assert( std::is_base_of_v< internal::seq< A >, separated_seq< S, A > > ); static_assert( std::is_base_of_v< internal::seq< A, S, B >, separated_seq< S, A, B > > ); static_assert( std::is_base_of_v< internal::seq< A, S, B, S, C, S, D >, separated_seq< S, A, B, C, D > > ); int main() -{} +{ + return 0; +} diff --git a/src/test/pegtl/contrib_state_control.cc b/src/test/pegtl/contrib_state_control.cpp similarity index 98% rename from src/test/pegtl/contrib_state_control.cc rename to src/test/pegtl/contrib_state_control.cpp index b4b6084e6..40f131a87 100644 --- a/src/test/pegtl/contrib_state_control.cc +++ b/src/test/pegtl/contrib_state_control.cpp @@ -15,6 +15,7 @@ int main() #include #include "test.hpp" +#include "test_inputs.hpp" #include @@ -158,7 +159,7 @@ namespace TAO_PEGTL_NAMESPACE // must< sor< one< 'a' >, try_catch_return_false< seq< one< 'b' >, must< one< 'c' > > > >, two< 'b' > >, eof > { test_state< true > st; - memory_input in( "bb", __FUNCTION__ ); + test::text_input< ascii::lf > in( "bb" ); std::size_t b = 0; const bool result = parse< test_grammar, test_action, state_control< normal >::type >( in, -1, b, st ); TAO_PEGTL_TEST_ASSERT( result ); @@ -200,7 +201,7 @@ namespace TAO_PEGTL_NAMESPACE } { test_state< false > st; - memory_input in( "bb", __FUNCTION__ ); + test::text_input< ascii::lf > in( "bb" ); std::size_t b = 0; const bool result = parse< test_grammar, test_action, state_control< normal >::type >( in, -1, b, st ); TAO_PEGTL_TEST_ASSERT( result ); @@ -212,7 +213,7 @@ namespace TAO_PEGTL_NAMESPACE // not_at< try_catch_return_false< try_catch_raise_nested< must< one< 'x' > > > > > { test_state< true > st; - memory_input in( "a", __FUNCTION__ ); + test::text_input< ascii::lf > in( "a" ); std::size_t b = 0; const bool result = parse< test_nested, nothing, state_control< normal >::type >( in, -1, b, st ); TAO_PEGTL_TEST_ASSERT( result ); diff --git a/src/test/pegtl/contrib_to_string.cc b/src/test/pegtl/contrib_to_string.cpp similarity index 94% rename from src/test/pegtl/contrib_to_string.cc rename to src/test/pegtl/contrib_to_string.cpp index 3d94ee8fa..1997c1012 100644 --- a/src/test/pegtl/contrib_to_string.cc +++ b/src/test/pegtl/contrib_to_string.cpp @@ -4,7 +4,6 @@ #include "test.hpp" -#include #include namespace TAO_PEGTL_NAMESPACE @@ -27,6 +26,7 @@ namespace TAO_PEGTL_NAMESPACE // to_string does *not* care about the outer class template TAO_PEGTL_TEST_ASSERT( ( to_string< one< 'a', 'b', 'c' > >() == "abc" ) ); + TAO_PEGTL_TEST_ASSERT( ( to_string< not_one< 'a', 'b', 'c' > >() == "abc" ) ); } } // namespace TAO_PEGTL_NAMESPACE diff --git a/src/test/pegtl/contrib_trace1.cc b/src/test/pegtl/contrib_trace_1.cc similarity index 93% rename from src/test/pegtl/contrib_trace1.cc rename to src/test/pegtl/contrib_trace_1.cc index 9d9ca9e4d..92a23edd7 100644 --- a/src/test/pegtl/contrib_trace1.cc +++ b/src/test/pegtl/contrib_trace_1.cc @@ -5,6 +5,7 @@ #include #include "test.hpp" +#include "test_inputs.hpp" #include @@ -25,7 +26,7 @@ namespace TAO_PEGTL_NAMESPACE void unit_test() { const std::string data = "F"; - memory_input in( data, __FILE__ ); + test::text_input< ascii::lf > in( data ); // Just enough to see that it compiles and nothing explodes; // the output format probabaly changes between compilers and // versions making a proper test difficult. diff --git a/src/test/pegtl/contrib_trace2.cc b/src/test/pegtl/contrib_trace_2.cc similarity index 86% rename from src/test/pegtl/contrib_trace2.cc rename to src/test/pegtl/contrib_trace_2.cc index 78c5b1648..5655c4496 100644 --- a/src/test/pegtl/contrib_trace2.cc +++ b/src/test/pegtl/contrib_trace_2.cc @@ -3,6 +3,7 @@ // (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) #include "test.hpp" +#include "test_inputs.hpp" #include @@ -51,41 +52,41 @@ namespace TAO_PEGTL_NAMESPACE void unit_test() { { - memory_input in( "ab", "trace test please ignore" ); + test::text_input< ascii::lf > in( "ab" ); const auto result = standard_trace< test::GRAMMAR1 >( in ); TAO_PEGTL_TEST_ASSERT( result ); TAO_PEGTL_TEST_ASSERT( test::a0 == 0 ); TAO_PEGTL_TEST_ASSERT( test::a == 0 ); } { - memory_input in( "ab", "trace test please ignore" ); + test::lazy_input< ascii::lf > in( "ab" ); const auto result = standard_trace< test::GRAMMAR1, test::trace_action >( in ); TAO_PEGTL_TEST_ASSERT( result ); TAO_PEGTL_TEST_ASSERT( test::a0 == 1 ); TAO_PEGTL_TEST_ASSERT( test::a == 1 ); } { - memory_input in( "a\r\n\t\0b", 6, "trace test please ignore" ); + test::text_input< ascii::lf > in( "a\r\n\t\0b", 6 ); const auto result = standard_trace< test::GRAMMAR2 >( in ); TAO_PEGTL_TEST_ASSERT( result ); TAO_PEGTL_TEST_ASSERT( test::a0 == 1 ); TAO_PEGTL_TEST_ASSERT( test::a == 1 ); } { - memory_input in( "a\r\n\t\0b", 6, "trace test please ignore" ); + test::text_input< ascii::lf > in( "a\r\n\t\0b", 6 ); const auto result = standard_trace< test::GRAMMAR2, test::trace_action >( in ); TAO_PEGTL_TEST_ASSERT( result ); TAO_PEGTL_TEST_ASSERT( test::a0 == 2 ); TAO_PEGTL_TEST_ASSERT( test::a == 1 ); } { - memory_input in( "c", "trace test please ignore" ); + test::text_input< ascii::lf > in( "c" ); const auto result = standard_trace< test::GRAMMAR3 >( in ); TAO_PEGTL_TEST_ASSERT( !result ); } #if defined( __cpp_exceptions ) { - memory_input in( "c", "trace test please ignore" ); + test::text_input< ascii::lf > in( "c" ); const auto result = standard_trace< test::GRAMMAR4 >( in ); TAO_PEGTL_TEST_ASSERT( !result ); } diff --git a/src/test/pegtl/contrib_partial_trace.cc b/src/test/pegtl/contrib_trace_3.cc similarity index 88% rename from src/test/pegtl/contrib_partial_trace.cc rename to src/test/pegtl/contrib_trace_3.cc index a643026b6..75c997de2 100644 --- a/src/test/pegtl/contrib_partial_trace.cc +++ b/src/test/pegtl/contrib_trace_3.cc @@ -3,6 +3,7 @@ // (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) #include "test.hpp" +#include "test_inputs.hpp" #include @@ -19,7 +20,7 @@ namespace TAO_PEGTL_NAMESPACE void unit_test() { - memory_input in( "xaacy", "trace test please ignore" ); + test::text_input< ascii::lf > in( "xaacy", "trace test please ignore" ); const auto result = parse< outer, partial_action >( in ); TAO_PEGTL_TEST_ASSERT( result ); } diff --git a/src/test/pegtl/control_unwind.cc b/src/test/pegtl/control_unwind.cpp similarity index 94% rename from src/test/pegtl/control_unwind.cc rename to src/test/pegtl/control_unwind.cpp index d69eb6914..de040ca6d 100644 --- a/src/test/pegtl/control_unwind.cc +++ b/src/test/pegtl/control_unwind.cpp @@ -10,9 +10,8 @@ int main() } #else -#include - #include "test.hpp" +#include "test_inputs.hpp" namespace TAO_PEGTL_NAMESPACE { @@ -63,7 +62,7 @@ namespace TAO_PEGTL_NAMESPACE void unit_test() { - memory_input in( "a1", __FUNCTION__ ); + test::text_input< ascii::lf > in( "a1" ); try { parse< r, a, c >( in ); TAO_PEGTL_TEST_UNREACHABLE; // LCOV_EXCL_LINE diff --git a/src/test/pegtl/cstream_input.cpp b/src/test/pegtl/cstream_input.cpp new file mode 100644 index 000000000..4ebbf73eb --- /dev/null +++ b/src/test/pegtl/cstream_input.cpp @@ -0,0 +1,54 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +#include "test.hpp" + +#include + +namespace TAO_PEGTL_NAMESPACE +{ + struct file_content + : seq< TAO_PEGTL_STRING( "dummy content" ), one< '\n' >, discard > + {}; + + struct file_grammar + : seq< rep_min_max< 11, 11, file_content >, eof > + {}; + + void unit_test() + { + const char* const filename = "src/test/pegtl/file_data.txt"; + { +#if defined( _MSC_VER ) + std::FILE* stream; + ::fopen_s( &stream, filename, "rb" ); +#else + std::FILE* stream = std::fopen( filename, "rb" ); +#endif + TAO_PEGTL_TEST_ASSERT( stream != nullptr ); + static_input in( stream ); + TAO_PEGTL_TEST_ASSERT( parse< file_grammar >( in ) ); + TAO_PEGTL_TEST_ASSERT( in.empty() ); + std::fclose( stream ); + } { +#if defined( _MSC_VER ) + std::FILE* stream; + ::fopen_s( &stream, filename, "rb" ); +#else + std::FILE* stream = std::fopen( filename, "rb" ); +#endif + TAO_PEGTL_TEST_ASSERT( stream != nullptr ); + dynamic_input in( 500, 10, stream ); + TAO_PEGTL_TEST_ASSERT( parse< file_grammar >( in ) ); + TAO_PEGTL_TEST_ASSERT( in.empty() ); + std::fclose( stream ); + } + } + +} // namespace TAO_PEGTL_NAMESPACE + +#include "main.hpp" diff --git a/src/test/pegtl/cstring_input.cpp b/src/test/pegtl/cstring_input.cpp new file mode 100644 index 000000000..8b5333980 --- /dev/null +++ b/src/test/pegtl/cstring_input.cpp @@ -0,0 +1,48 @@ +// Copyright (c) 2019-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#include + +#include "test.hpp" + +#include + +namespace TAO_PEGTL_NAMESPACE +{ + template< typename Rule, template< typename... > class Action = nothing > + bool parse_cstring( const char* string, const std::size_t size, const std::size_t chunk ) + { + dynamic_input in( size, chunk, string ); + return parse< Rule, Action >( in ); + } + + void unit_test() + { + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 4, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 4, 3 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 8, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 8, 4 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 8, 7 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, discard, eof > >( "abc", 3, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, discard, eof > >( "abc", 3, 2 ) ); +#if defined( __cpp_exceptions ) + TAO_PEGTL_TEST_THROWS( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 3, 1 ) ); + TAO_PEGTL_TEST_THROWS( parse_cstring< seq< string< 'a', 'b', 'c' >, eof > >( "abc", 3, 2 ) ); +#endif + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' > >, eof > >( "abcabcabc", 12, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' > >, eof > >( "abcabcabc", 12, 2 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' > >, eof > >( "abcabcabc", 12, 8 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' > >, eof > >( "abcabcabc", 12, 11 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' >, discard >, eof > >( "abcabcabc", 4, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' >, discard >, eof > >( "abcabcabc", 4, 2 ) ); +#if defined( __cpp_exceptions ) + TAO_PEGTL_TEST_THROWS( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' > >, eof > >( "abcabcabc", 4, 1 ) ); + TAO_PEGTL_TEST_THROWS( parse_cstring< seq< rep< 3, string< 'a', 'b', 'c' > >, eof > >( "abcabcabc", 4, 2 ) ); +#endif + TAO_PEGTL_TEST_ASSERT( parse_cstring< seq< string< 'a', 'b', 'c' >, discard, string< 'd', 'e', 'f' >, eof > >( "abcdef", 4, 1 ) ); + } + +} // namespace TAO_PEGTL_NAMESPACE + +#include "main.hpp" diff --git a/src/test/pegtl/discard_input.cpp b/src/test/pegtl/discard_input.cpp new file mode 100644 index 000000000..0fd58d78e --- /dev/null +++ b/src/test/pegtl/discard_input.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2019-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#include + +#include "test.hpp" + +#include + +namespace TAO_PEGTL_NAMESPACE +{ + template< typename Rule, template< typename... > class Action > + bool parse_cstring( const char* string, const std::size_t size, const std::size_t chunk ) + { + dynamic_input in( size, chunk, string ); + return parse< Rule, Action >( in ); + } + + // clang-format off + struct n : one< 'n' > {}; + struct a : one< 'a' > {}; + struct f : one< 'f' > {}; + struct s : one< 's' > {}; + + template< typename Rule > struct my_action {}; + template<> struct my_action< a > : discard_input {}; + template<> struct my_action< f > : discard_input_on_failure {}; + template<> struct my_action< s > : discard_input_on_success {}; + // clang-format on + + void unit_test() + { +#if defined( __cpp_exceptions ) + TAO_PEGTL_TEST_THROWS( parse_cstring< rep< 4, sor< n, n > >, my_action >( "nnnn", 2, 1 ) ); +#endif + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< a, n > >, my_action >( "nnnn", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< f, n > >, my_action >( "nnnn", 2, 1 ) ); +#if defined( __cpp_exceptions ) + TAO_PEGTL_TEST_THROWS( parse_cstring< rep< 4, sor< s, n > >, my_action >( "nnnn", 2, 1 ) ); +#endif + + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< n, a > >, my_action >( "aaaa", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< a, a > >, my_action >( "aaaa", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< f, a > >, my_action >( "aaaa", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< s, a > >, my_action >( "aaaa", 2, 1 ) ); + +#if defined( __cpp_exceptions ) + TAO_PEGTL_TEST_THROWS( parse_cstring< rep< 4, sor< n, f > >, my_action >( "ffff", 2, 1 ) ); +#endif + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< a, f > >, my_action >( "ffff", 2, 1 ) ); +#if defined( __cpp_exceptions ) + TAO_PEGTL_TEST_THROWS( parse_cstring< rep< 4, sor< f, f > >, my_action >( "ffff", 2, 1 ) ); + TAO_PEGTL_TEST_THROWS( parse_cstring< rep< 4, sor< s, f > >, my_action >( "ffff", 2, 1 ) ); +#endif + + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< n, s > >, my_action >( "ssss", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< a, s > >, my_action >( "ssss", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< f, s > >, my_action >( "ssss", 2, 1 ) ); + TAO_PEGTL_TEST_ASSERT( parse_cstring< rep< 4, sor< s, s > >, my_action >( "ssss", 2, 1 ) ); + } + +} // namespace TAO_PEGTL_NAMESPACE + +#include "main.hpp" diff --git a/src/test/pegtl/file_file.cc b/src/test/pegtl/file_file.cc deleted file mode 100644 index b1b3b1819..000000000 --- a/src/test/pegtl/file_file.cc +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2015-2023 Dr. Colin Hirsch and Daniel Frey -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) - -#include "test.hpp" -#include "verify_file.hpp" - -namespace TAO_PEGTL_NAMESPACE -{ - void unit_test() - { - verify_file< file_input<> >(); - } - -} // namespace TAO_PEGTL_NAMESPACE - -#include "main.hpp" diff --git a/src/test/pegtl/istream_input.cpp b/src/test/pegtl/istream_input.cpp new file mode 100644 index 000000000..d29fc92d2 --- /dev/null +++ b/src/test/pegtl/istream_input.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2016-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +#include "test.hpp" + +#include + +namespace TAO_PEGTL_NAMESPACE +{ + struct file_content + : seq< TAO_PEGTL_STRING( "dummy content" ), one< '\n' >, discard > + {}; + + struct file_grammar + : seq< rep_min_max< 11, 11, file_content >, eof > + {}; + + void unit_test() + { +#if defined( __cpp_exceptions ) + { + const char* filename = "src/test/pegtl/no_such_file.txt"; + try { + std::ifstream stream( filename ); + static_input in( stream ); + parse< file_grammar >( in ); + TAO_PEGTL_TEST_UNREACHABLE; // LCOV_EXCL_LINE + } + catch( const std::system_error& e ) { + TAO_PEGTL_TEST_ASSERT( e.code().category() == std::system_category() ); + TAO_PEGTL_TEST_ASSERT( e.code().value() == ENOENT ); + } + } +#endif + const char* filename = "src/test/pegtl/file_data.txt"; + { + std::ifstream stream( filename ); + static_input in( stream ); + TAO_PEGTL_TEST_ASSERT( parse< file_grammar >( in ) ); + TAO_PEGTL_TEST_ASSERT( in.empty() ); + } { + std::ifstream stream( filename ); + dynamic_input in( 100, 90, stream ); + TAO_PEGTL_TEST_ASSERT( parse< file_grammar >( in ) ); + TAO_PEGTL_TEST_ASSERT( in.empty() ); + } + } + +} // namespace TAO_PEGTL_NAMESPACE + +#include "main.hpp" diff --git a/src/test/pegtl/pegtl.cpp b/src/test/pegtl/pegtl.cpp index 503169c7e..50e19c00a 100644 --- a/src/test/pegtl/pegtl.cpp +++ b/src/test/pegtl/pegtl.cpp @@ -3,7 +3,6 @@ // (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) #include "test.hpp" -#include #include #include @@ -11,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/pegtl/restart_input.cc b/src/test/pegtl/restart_input.cc deleted file mode 100644 index 103d049f1..000000000 --- a/src/test/pegtl/restart_input.cc +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2020-2023 Dr. Colin Hirsch and Daniel Frey -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) - -#include "test.hpp" - -namespace TAO_PEGTL_NAMESPACE -{ - using grammar = seq< string< 'a', 'b', 'c' >, eof >; - - void test_lazy() - { - const std::string data = "abc"; - memory_input< tracking_mode::lazy, ascii::lf_crlf, std::string > in( data, __FUNCTION__ ); - bool success = parse< grammar >( in ); - TAO_PEGTL_TEST_ASSERT( success ); - in.restart(); - success = parse< grammar >( in ); - TAO_PEGTL_TEST_ASSERT( success ); - } - - void test_eager() - { - const std::string data = "abc"; - memory_input< tracking_mode::eager, ascii::lf_crlf, std::string > in( std::string_view{ data }, __FUNCTION__ ); - bool success = parse< grammar >( in ); - TAO_PEGTL_TEST_ASSERT( success ); - in.restart(); - success = parse< grammar >( in ); - TAO_PEGTL_TEST_ASSERT( success ); - } - - void unit_test() - { - test_lazy(); - test_eager(); - } - -} // namespace TAO_PEGTL_NAMESPACE - -#include "main.hpp" diff --git a/src/test/pegtl/rule_discard.cpp b/src/test/pegtl/rule_discard.cpp new file mode 100644 index 000000000..0b7a6c433 --- /dev/null +++ b/src/test/pegtl/rule_discard.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2014-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#include "test.hpp" +#include "verify_meta.hpp" +#include "verify_rule.hpp" + +#include + +namespace TAO_PEGTL_NAMESPACE +{ + void unit_test() + { + verify_meta< discard, internal::discard >(); + + verify_analyze< discard >( __LINE__, __FILE__, false, false ); + + verify_rule< discard >( __LINE__, __FILE__, "", result_type::success, 0 ); + + for( char i = 1; i < 127; ++i ) { + char t[] = { i, 0 }; + verify_rule< discard >( __LINE__, __FILE__, std::string( t ), result_type::success, 1 ); + } + } + +} // namespace TAO_PEGTL_NAMESPACE + +#include "main.hpp" diff --git a/src/test/pegtl/rule_function.cpp b/src/test/pegtl/rule_function.cpp index 72bddb992..f1749e19e 100644 --- a/src/test/pegtl/rule_function.cpp +++ b/src/test/pegtl/rule_function.cpp @@ -23,6 +23,20 @@ namespace TAO_PEGTL_NAMESPACE TAO_PEGTL_TEST_ASSERT( called ); } + [[nodiscard]] bool func0( test::text_input< ascii::lf >& in ) + { + called = true; + in.consume< internal::eol_exclude_tag >( 1 ); + return true; + } + + [[nodiscard]] bool func0n( test::text_input< ascii::lf >& in ) noexcept + { + called = true; + in.consume< internal::eol_exclude_tag >( 1 ); + return true; + } + [[nodiscard]] bool func1( test::text_input< ascii::lf >& in, int /*unused*/, char*& /*unused*/, const double& /*unused*/ ) { called = true; @@ -63,6 +77,7 @@ namespace TAO_PEGTL_NAMESPACE void unit_test() { + unit_test_1< func0 >(); unit_test_1< func1 >(); unit_test_1< func2, internal::peek_char >(); unit_test_1< func2, internal::peek_ascii >(); @@ -70,6 +85,7 @@ namespace TAO_PEGTL_NAMESPACE unit_test_1< func3, internal::peek_char >(); unit_test_1< func3, internal::peek_ascii >(); unit_test_1< func3, internal::peek_current >(); + unit_test_1< func0n >(); unit_test_1< func1n >(); unit_test_1< func2n, internal::peek_char >(); unit_test_1< func2n, internal::peek_ascii >(); diff --git a/src/test/pegtl/rule_require.cpp b/src/test/pegtl/rule_require.cpp new file mode 100644 index 000000000..a6c3420b6 --- /dev/null +++ b/src/test/pegtl/rule_require.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2017-2023 Dr. Colin Hirsch and Daniel Frey +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +#include "test.hpp" +#include "verify_meta.hpp" +#include "verify_rule.hpp" + +#include + +namespace TAO_PEGTL_NAMESPACE +{ + void unit_test() + { + verify_meta< require< 0 >, internal::success >(); + verify_meta< require< 1 >, internal::require< 1 > >(); + + verify_analyze< require< 0 > >( __LINE__, __FILE__, false, false ); + verify_analyze< require< 1 > >( __LINE__, __FILE__, false, false ); + verify_analyze< require< 9 > >( __LINE__, __FILE__, false, false ); + + verify_rule< require< 0 > >( __LINE__, __FILE__, "", result_type::success, 0 ); + verify_rule< require< 0 > >( __LINE__, __FILE__, "a", result_type::success, 1 ); + verify_rule< require< 0 > >( __LINE__, __FILE__, " ", result_type::success, 2 ); + verify_rule< require< 1 > >( __LINE__, __FILE__, "", result_type::local_failure, 0 ); + verify_rule< require< 1 > >( __LINE__, __FILE__, "a", result_type::success, 1 ); + verify_rule< require< 1 > >( __LINE__, __FILE__, " ", result_type::success, 2 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "", result_type::local_failure, 0 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "1", result_type::local_failure, 1 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "12", result_type::local_failure, 2 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "123", result_type::local_failure, 3 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "1234", result_type::local_failure, 4 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "12345", result_type::local_failure, 5 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "123456", result_type::local_failure, 6 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "1234567", result_type::local_failure, 7 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "12345678", result_type::local_failure, 8 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "123456789", result_type::success, 9 ); + verify_rule< require< 9 > >( __LINE__, __FILE__, "123456789123456789", result_type::success, 18 ); + } + +} // namespace TAO_PEGTL_NAMESPACE + +#include "main.hpp"