Skip to content

[Pratt parser] Improve compatibility with ANTLR parser #2239

新建议题

Have a question about this project? 注册 for a free GitHub account to open an issue and contact its maintainers and the community.

注册 for GitHub

By clicking “注册 for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? 登录 to your account

Open
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[Pratt parser] Improve compatibility with ANTLR parser #2239

Show file tree
Hide file tree
Changes from all commits
提交
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 132 additions & 73 deletions parser/internal/pratt_parser_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,39 @@ std::vector<TestCase> GetParserTestCases() {
)^#1:Expr.Call#
)",
},
TestCase{
.source = "-.2.V",
.expected_ast = R"(
-0.2^#1:double#.V^#2:Expr.Select#
)",
},
TestCase{
.source = "!-.2.V",
.expected_ast = R"(
!_(
-0.2^#2:double#.V^#3:Expr.Select#
)^#1:Expr.Call#
)",
},
TestCase{
.source = "!-2.V",
.expected_ast = R"(
!_(
-2^#2:int64#.V^#3:Expr.Select#
)^#1:Expr.Call#
)",
},
TestCase{
.source = "!-.2[0]",
.expected_ast = R"(
!_(
_[_](
-0.2^#2:double#,
0^#4:int64#
)^#3:Expr.Call#
)^#1:Expr.Call#
)",
},
TestCase{
.source = "a + b",
.expected_ast = R"(
Expand Down Expand Up @@ -1156,21 +1189,24 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
ErrorTestCase{
.source = "1 + 2 * 3 4",
.expected_error =
"ERROR: <input>:1:11: unexpected token after expression\n"
"ERROR: <input>:1:11: Syntax error: unexpected token after "
"expression\n"
" | 1 + 2 * 3 4\n"
" | ..........^",
},
ErrorTestCase{
.source = "1{}",
.expected_error =
"ERROR: <input>:1:2: unexpected token after expression\n"
"ERROR: <input>:1:2: Syntax error: unexpected token after "
"expression\n"
" | 1{}\n"
" | .^",
},
ErrorTestCase{
.source = "true ? 1",
.expected_error =
"ERROR: <input>:1:9: expected ':' in conditional expression\n"
"ERROR: <input>:1:9: Syntax error: expected ':' in conditional "
"expression\n"
" | true ? 1\n"
" | ........^",
},
Expand All @@ -1182,10 +1218,10 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = "a.",
.expected_error =
"ERROR: <input>:1:3: expected identifier after '.'\n"
" | a.\n"
" | ..^",
.expected_error = "ERROR: <input>:1:3: Syntax error: expected "
"identifier after '.'\n"
" | a.\n"
" | ..^",
},
ErrorTestCase{
.source = "a[?0]",
Expand All @@ -1195,9 +1231,10 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = ". *",
.expected_error = "ERROR: <input>:1:3: expected identifier\n"
" | . *\n"
" | ..^",
.expected_error =
"ERROR: <input>:1:3: Syntax error: expected identifier\n"
" | . *\n"
" | ..^",
},
ErrorTestCase{
.source = ".as",
Expand All @@ -1208,17 +1245,19 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
ErrorTestCase{
.source = "* 2",
.expected_error =
"ERROR: <input>:1:1: unexpected token\n"
"ERROR: <input>:1:1: Syntax error: unexpected token\n"
" | * 2\n"
" | ^\n"
"ERROR: <input>:1:3: unexpected token after expression\n"
"ERROR: <input>:1:3: Syntax error: unexpected token after "
"expression\n"
" | * 2\n"
" | ..^",
},
ErrorTestCase{
.source = "(1 + 2",
.expected_error =
"ERROR: <input>:1:7: mismatched input <EOF> expecting ')'\n"
"ERROR: <input>:1:7: Syntax error: mismatched input <EOF> "
"expecting ')'\n"
" | (1 + 2\n"
" | ......^",
},
Expand All @@ -1230,7 +1269,7 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = "[1, 2",
.expected_error = "ERROR: <input>:1:6: expected ']'\n"
.expected_error = "ERROR: <input>:1:6: Syntax error: expected ']'\n"
" | [1, 2\n"
" | .....^",
},
Expand All @@ -1242,13 +1281,14 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = "{'k' 'v'}",
.expected_error = "ERROR: <input>:1:6: expected ':' in map entry\n"
" | {'k' 'v'}\n"
" | .....^",
.expected_error =
"ERROR: <input>:1:6: Syntax error: expected ':' in map entry\n"
" | {'k' 'v'}\n"
" | .....^",
},
ErrorTestCase{
.source = "{'k': 'v'",
.expected_error = "ERROR: <input>:1:10: expected '}'\n"
.expected_error = "ERROR: <input>:1:10: Syntax error: expected '}'\n"
" | {'k': 'v'\n"
" | .........^",
},
Expand All @@ -1260,26 +1300,29 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = "Msg{1: 2}",
.expected_error = "ERROR: <input>:1:5: expected struct field name\n"
" | Msg{1: 2}\n"
" | ....^",
.expected_error =
"ERROR: <input>:1:5: Syntax error: expected struct field name\n"
" | Msg{1: 2}\n"
" | ....^",
},
ErrorTestCase{
.source = "Msg{f 10}",
.expected_error = "ERROR: <input>:1:7: expected ':' in struct field\n"
" | Msg{f 10}\n"
" | ......^",
.expected_error =
"ERROR: <input>:1:7: Syntax error: expected ':' in struct field\n"
" | Msg{f 10}\n"
" | ......^",
},
ErrorTestCase{
.source = "Msg{f: 10",
.expected_error = "ERROR: <input>:1:10: expected '}'\n"
.expected_error = "ERROR: <input>:1:10: Syntax error: expected '}'\n"
" | Msg{f: 10\n"
" | .........^",
},
ErrorTestCase{
.source = "f(1, 2",
.expected_error =
"ERROR: <input>:1:7: mismatched input <EOF> expecting ')'\n"
"ERROR: <input>:1:7: Syntax error: mismatched input <EOF> "
"expecting ')'\n"
" | f(1, 2\n"
" | ......^",
},
Expand All @@ -1291,35 +1334,39 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = "999999999999999999999999999999999999999",
.expected_error = "ERROR: <input>:1:1: invalid int literal\n"
" | 999999999999999999999999999999999999999\n"
" | ^",
.expected_error =
"ERROR: <input>:1:1: Syntax error: invalid int literal\n"
" | 999999999999999999999999999999999999999\n"
" | ^",
},
ErrorTestCase{
.source = "999999999999999999999999999999999999999u",
.expected_error = "ERROR: <input>:1:1: invalid uint literal\n"
" | 999999999999999999999999999999999999999u\n"
" | ^",
.expected_error =
"ERROR: <input>:1:1: Syntax error: invalid uint literal\n"
" | 999999999999999999999999999999999999999u\n"
" | ^",
},
ErrorTestCase{
.source = "1e",
.expected_error =
"ERROR: <input>:1:1: floating point literal missing digits after "
"exponent separator\n"
"ERROR: <input>:1:1: Syntax error: floating point literal "
"missing digits after exponent separator\n"
" | 1e\n"
" | ^",
},
ErrorTestCase{
.source = "\"unterminated",
.expected_error = "ERROR: <input>:1:1: unterminated string literal\n"
" | \"unterminated\n"
" | ^",
.expected_error =
"ERROR: <input>:1:1: Syntax error: unterminated string literal\n"
" | \"unterminated\n"
" | ^",
},
ErrorTestCase{
.source = "b\"unterminated",
.expected_error = "ERROR: <input>:1:1: unterminated bytes literal\n"
" | b\"unterminated\n"
" | ^",
.expected_error =
"ERROR: <input>:1:1: Syntax error: unterminated bytes literal\n"
" | b\"unterminated\n"
" | ^",
},
ErrorTestCase{
.source = "a.?`foo`",
Expand Down Expand Up @@ -1374,81 +1421,91 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
ErrorTestCase{
.source = "`foo",
.expected_error =
"ERROR: <input>:1:1: unterminated quoted identifier\n"
"ERROR: <input>:1:1: Syntax error: unterminated quoted "
"identifier\n"
" | `foo\n"
" | ^",
.enable_quoted_identifiers = true,
},
ErrorTestCase{
.source = "f(*, 1e, {2 3})",
.expected_error =
"ERROR: <input>:1:3: unexpected token\n"
"ERROR: <input>:1:3: Syntax error: unexpected token\n"
" | f(*, 1e, {2 3})\n"
" | ..^\n"
"ERROR: <input>:1:6: floating point literal missing digits after "
"exponent separator\n"
"ERROR: <input>:1:6: Syntax error: floating point literal "
"missing digits after exponent separator\n"
" | f(*, 1e, {2 3})\n"
" | .....^\n"
"ERROR: <input>:1:13: expected ':' in map entry\n"
"ERROR: <input>:1:13: Syntax error: expected ':' in map entry\n"
" | f(*, 1e, {2 3})\n"
" | ............^",
},
ErrorTestCase{
.source = "(1 + *) + 2",
.expected_error = "ERROR: <input>:1:6: unexpected token\n"
" | (1 + *) + 2\n"
" | .....^",
.expected_error =
"ERROR: <input>:1:6: Syntax error: unexpected token\n"
" | (1 + *) + 2\n"
" | .....^",
},
ErrorTestCase{
.source = "f(1 + *, 2)",
.expected_error = "ERROR: <input>:1:7: unexpected token\n"
" | f(1 + *, 2)\n"
" | ......^",
.expected_error =
"ERROR: <input>:1:7: Syntax error: unexpected token\n"
" | f(1 + *, 2)\n"
" | ......^",
},
ErrorTestCase{
.source = "(a. + 1)",
.expected_error =
"ERROR: <input>:1:5: expected identifier after '.'\n"
"ERROR: <input>:1:5: Syntax error: expected identifier after "
"'.'\n"
" | (a. + 1)\n"
" | ....^",
},
ErrorTestCase{
.source = "f(a., 1)",
.expected_error =
"ERROR: <input>:1:5: expected identifier after '.'\n"
"ERROR: <input>:1:5: Syntax error: expected identifier after "
"'.'\n"
" | f(a., 1)\n"
" | ....^",
},
ErrorTestCase{
.source = "[a., 1]",
.expected_error =
"ERROR: <input>:1:4: expected identifier after '.'\n"
"ERROR: <input>:1:4: Syntax error: expected identifier after "
"'.'\n"
" | [a., 1]\n"
" | ...^",
},
ErrorTestCase{
.source = "-0x8000000000000001",
.expected_error = "ERROR: <input>:1:2: invalid int literal\n"
" | -0x8000000000000001\n"
" | .^",
.expected_error =
"ERROR: <input>:1:2: Syntax error: invalid int literal\n"
" | -0x8000000000000001\n"
" | .^",
},
ErrorTestCase{
.source = "-0x10000000000000000",
.expected_error = "ERROR: <input>:1:2: invalid int literal\n"
" | -0x10000000000000000\n"
" | .^",
.expected_error =
"ERROR: <input>:1:2: Syntax error: invalid int literal\n"
" | -0x10000000000000000\n"
" | .^",
},
ErrorTestCase{
.source = "-9223372036854775809",
.expected_error = "ERROR: <input>:1:2: invalid int literal\n"
" | -9223372036854775809\n"
" | .^",
.expected_error =
"ERROR: <input>:1:2: Syntax error: invalid int literal\n"
" | -9223372036854775809\n"
" | .^",
},
ErrorTestCase{
.source = "-999999999999999999999999999999999999999",
.expected_error = "ERROR: <input>:1:2: invalid int literal\n"
" | -999999999999999999999999999999999999999\n"
" | .^",
.expected_error =
"ERROR: <input>:1:2: Syntax error: invalid int literal\n"
" | -999999999999999999999999999999999999999\n"
" | .^",
},
ErrorTestCase{
.source = "-",
Expand All @@ -1460,15 +1517,17 @@ std::vector<ErrorTestCase> GetErrorTestCases() {
},
ErrorTestCase{
.source = "- *",
.expected_error = "ERROR: <input>:1:3: unexpected token\n"
" | - *\n"
" | ..^",
.expected_error =
"ERROR: <input>:1:3: Syntax error: unexpected token\n"
" | - *\n"
" | ..^",
},
ErrorTestCase{
.source = "\"😀😀😀😀😀\" ~error",
.expected_error = "ERROR: <input>:1:9: unexpected character\n"
" | \"😀😀😀😀😀\" ~error\n"
" | .．．．．．..^",
.expected_error =
"ERROR: <input>:1:9: Syntax error: unexpected character\n"
" | \"😀😀😀😀😀\" ~error\n"
" | .．．．．．..^",
},
};
}
Expand Down Expand Up @@ -1875,7 +1934,7 @@ TEST(PrattParserErrorRecoveryTest, ErrorRecoveryLimitOne) {
ASSERT_OK_AND_ASSIGN(auto source, cel::新建Source("......"));
EXPECT_EQ(Format问题(*source, issues),
"ERROR: <input>:-1:0: Error recovery limit (1) exceeded\n"
"ERROR: <input>:1:2: expected identifier\n"
"ERROR: <input>:1:2: Syntax error: expected identifier\n"
" | ......\n"
" | .^");
}
Expand Down
Loading
Loading