使Boost :: Spirit ::符号解析器非贪婪

Question

我想制作一个与IE int 匹配的关键字解析器，但与 Integer 中的 int 与 eger> eger int >剩下。我使用 X3 ::符号自动获取以枚举值表示的解析关键字。

最小示例：

#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/utility/error_reporting.hpp>

namespace x3 = boost::spirit::x3;

enum class TypeKeyword { Int, Float, Bool };

struct TypeKeywordSymbolTable : x3::symbols<TypeKeyword> {
    TypeKeywordSymbolTable()
    {
        add("float", TypeKeyword::Float)
           ("int",   TypeKeyword::Int)
           ("bool",  TypeKeyword::Bool);
    }
};
const TypeKeywordSymbolTable type_keyword_symbol_table;

struct TypeKeywordRID {};
using TypeKeywordRule = x3::rule<TypeKeywordRID, TypeKeyword>;

const TypeKeywordRule type_keyword = "type_keyword";
const auto type_keyword_def        = type_keyword_symbol_table;
BOOST_SPIRIT_DEFINE(type_keyword);

using Iterator = std::string_view::const_iterator;

/* Thrown when the parser has failed to parse the whole input stream. Contains
 * the part of the input stream that has not been parsed. */
class LeftoverError : public std::runtime_error {
  public:
    LeftoverError(Iterator begin, Iterator end)
            : std::runtime_error(std::string(begin, end))
    {}

    std::string_view get_leftover_data() const noexcept { return what(); }
};

template<typename Rule>
typename Rule::attribute_type parse(std::string_view input, const Rule& rule)
{
    Iterator begin = input.begin();
    Iterator end   = input.end();

    using ExpectationFailure = boost::spirit::x3::expectation_failure<Iterator>;
    typename Rule::attribute_type result;
    try {
        bool r = x3::phrase_parse(begin, end, rule, x3::space, result);
        if (r && begin == end) {
            return result;
        } else { // Occurs when the whole input stream has not been consumed.
            throw LeftoverError(begin, end);
        }
    } catch (const ExpectationFailure& exc) {
        throw LeftoverError(exc.where(), end);
    }
}

int main()
{
    // TypeKeyword::Bool is parsed and "ean" is leftover, but failed parse with
    // "boolean" leftover is desired.
    parse("boolean", type_keyword);

    // TypeKeyword::Int is parsed and "eger" is leftover, but failed parse with
    // "integer" leftover is desired.
    parse("integer", type_keyword);

    // TypeKeyword::Int is parsed successfully and this is the desired behavior.
    parse("int", type_keyword);
}

基本上，我想要 Integer 不要被识别为关键字，而剩下的 eger 剩下的。

Answer 1

我将测试用例变成了自我描述的期望：

live on Compiler Explorer

印刷：

FAIL  "boolean"    -> TypeKeyword::Bool    (expected Leftover:"boolean")
FAIL  "integer"    -> TypeKeyword::Int     (expected Leftover:"integer")
OK    "int"        -> TypeKeyword::Int    

现在，最简单，最幼稚的方法是确保您通过简单地更改

auto actual = parse(input, Parser::type_keyword);

为

auto actual = parse(input, Parser::type_keyword >> x3::eoi);

eoi ，而实际上是测试通过：

OK    "boolean"    -> Leftover:"boolean"  
OK    "integer"    -> Leftover:"integer"  
OK    "int"        -> TypeKeyword::Int    

​让我们想象一个更涉及的语法，其中键入标识符; 将被解析：

auto identifier 
    = x3::rule<struct id_, Ast::Identifier> {"identifier"}
    = x3::lexeme[x3::char_("a-zA-Z_") >> *x3::char_("a-zA-Z_0-9")];

auto type_keyword
    = x3::rule<struct tk_, Ast::TypeKeyword> {"type_keyword"}
    = type_;

auto declaration 
    = x3::rule<struct decl_, Ast::Declaration>{"declaration"}
    = type_keyword >> identifier >> ';';

我将留下编译器资源管理器：

OK    "boolean b;" -> Leftover:"boolean b;"
OK    "integer i;" -> Leftover:"integer i;"
OK    "int j;"     -> (TypeKeyword::Int j)

看起来不错。但是，如果我们添加一些有趣的测试：

    {"flo at f;", LeftoverError("flo at f;")},
    {"boolean;", LeftoverError("boolean;")},

它打印（ live ），

OK    "boolean b;" -> Leftover:"boolean b;"
OK    "integer i;" -> Leftover:"integer i;"
OK    "int j;"     -> (TypeKeyword::Int j)
FAIL  "boolean;"   -> (TypeKeyword::Bool ean) (expected Leftover:"boolean;")

所以缺乏测试用例。您的散文描述实际上更接近：

我想制作一个与IE INT匹配的关键字解析器，但与Integer中的INT不匹配，而Eger weld over of poft

。幼稚的尝试可能会检查没有标识符字符遵循类型关键字：

auto type_keyword
    = x3::rule<struct tk_, Ast::TypeKeyword> {"type_keyword"}
    = type_ >> !identchar;

其中 Identchar 是从 didentifier 类似的情况下计入的：

auto identchar = x3::char_("a-zA-Z_0-9");

auto identifier 
    = x3::rule<struct id_, Ast::Identifier> {"identifier"}
    = x3::lexeme[x3::char_("a-zA-Z_") >> *identchar];

但是，这是行不通的。您能明白为什么（允许窥视： httpps://godbolt.org.org/z/jb4zfhfwb ）？

现在，我们最新的Devious测试用例已通过（是的），但是 int J; 现在被拒绝！如果您考虑一下，这只会有意义，因为您已经跳过了。

我刚才使用的基本词是 lexeme ：您想将某些单位视为lexemes（而Whitespace停止了Lexeme。或者更确切地说，Whitespace并未在lexeme内部自动跳过）。因此，修复程序将是：

auto type_keyword
    // ...
    = x3::lexeme[type_ >> !identchar];

（请注意我如何偷偷地将其包含在标识符之前）

lo（ live ）：

OK    "boolean b;" -> Leftover:"boolean b;"
OK    "integer i;" -> Leftover:"integer i;"
OK    "int j;"     -> (TypeKeyword::Int j)
OK    "boolean;"   -> Leftover:"boolean;" 

总结

这个主题是一个经常经常出现的话题，它首先需要对船长，Lexemes有深入的了解。这是其他一些灵感的文章：

• 
  

• 放解码标识符，关键字除外

• Boost Spirit x3：分析划分的字符串我介绍一个更普遍的帮助者，您可能会发现有用：

  >

  >

  >

    auto kw = []（auto p）{
        返回x3 :: lexeme [x3 :: as_parser（p）＆gt;＆gt; ！x3 :: char _（“ a-za-z0-9_”）];
   };
   

• 
  

• 在x3

！

完整列表

反竞争，最终列表：

#include <boost/fusion/adapted.hpp>
#include <boost/fusion/include/as_vector.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/spirit/home/x3.hpp>
#include <iomanip>
#include <iostream>

namespace x3 = boost::spirit::x3;

namespace Ast {
    enum class TypeKeyword { Int, Float, Bool };

    static std::ostream& operator<<(std::ostream& os, TypeKeyword tk) {
        switch (tk) {
            case TypeKeyword::Int:   return os << "TypeKeyword::Int";
            case TypeKeyword::Float: return os << "TypeKeyword::Float";
            case TypeKeyword::Bool:  return os << "TypeKeyword::Bool";
        };
        return os << "?";
    }

    using Identifier = std::string;

    struct Declaration {
        TypeKeyword type;
        Identifier id;

        bool operator==(Declaration const&) const = default;
    };

} // namespace Ast

BOOST_FUSION_ADAPT_STRUCT(Ast::Declaration, type, id)

namespace Ast{
    static std::ostream& operator<<(std::ostream& os, Ast::Declaration const& d) {
        return os << boost::lexical_cast<std::string>(boost::fusion::as_vector(d));
    }
} // namespace Ast

namespace Parser {
    struct Type : x3::symbols<Ast::TypeKeyword> {
        Type() {
            add("float", Ast::TypeKeyword::Float) //
                ("int", Ast::TypeKeyword::Int)    //
                ("bool", Ast::TypeKeyword::Bool); //
        }
    } const static type_;

    auto identchar = x3::char_("a-zA-Z_0-9");

    auto identifier 
        = x3::rule<struct id_, Ast::Identifier> {"identifier"}
        = x3::lexeme[x3::char_("a-zA-Z_") >> *identchar];

    auto type_keyword
        = x3::rule<struct tk_, Ast::TypeKeyword> {"type_keyword"}
        = x3::lexeme[type_ >> !identchar];

    auto declaration 
        = x3::rule<struct decl_, Ast::Declaration>{"declaration"}
        = type_keyword >> identifier >> ';';
} // namespace Parser

struct LeftoverError : std::runtime_error {
    using std::runtime_error::runtime_error;

    friend std::ostream& operator<<(std::ostream& os, LeftoverError const& e) {
        return os << (std::string("Leftover:\"") + e.what() + "\"");
    }
    bool operator==(LeftoverError const& other) const {
        return std::string_view(what()) == other.what();
    }
};

template<typename T>
using Maybe = boost::variant<T, LeftoverError>;

template <typename Rule,
          typename Attr = typename x3::traits::attribute_of<Rule, x3::unused_type>::type,
          typename R = Maybe<Attr>>
R parse(std::string_view input, Rule const& rule) {
    Attr result;
    auto f = input.begin(), l = input.end();
    return x3::phrase_parse(f, l, rule, x3::space, result)
        ? R(std::move(result))
        : LeftoverError({f, l});
}

int main()
{
    using namespace Ast;

    struct {
        std::string_view        input;
        Maybe<Declaration>      expected;
    } cases[] = {
        {"boolean b;", LeftoverError("boolean b;")},
        {"integer i;", LeftoverError("integer i;")},
        {"int j;", Declaration{TypeKeyword::Int, "j"}},
        {"boolean;", LeftoverError("boolean;")},
    };
    for (auto [input, expected] : cases) {
        auto actual = parse(input, Parser::declaration >> x3::eoi);
        bool ok     = expected == actual;

        std::cout << std::left << std::setw(6) << (ok ? "OK" : "FAIL")
                  << std::setw(12) << std::quoted(input) << " -> "
                  << std::setw(20) << actual;
        if (not ok)
            std::cout << " (expected " << expected << ")";
        std::cout << "\n";
    }
}

---

¹请参阅