| // Copyright (C) 2022 Google LLC | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #include <memory> | |
| #include <string> | |
| #include <utility> | |
| #include <vector> | |
| #include "gmock/gmock.h" | |
| #include "gtest/gtest.h" | |
| #include "icing/query/advanced_query_parser/abstract-syntax-tree-test-utils.h" | |
| #include "icing/query/advanced_query_parser/abstract-syntax-tree.h" | |
| #include "icing/query/advanced_query_parser/lexer.h" | |
| #include "icing/query/advanced_query_parser/parser.h" | |
| #include "icing/testing/common-matchers.h" | |
| namespace icing { | |
| namespace lib { | |
| namespace { | |
| using ::testing::ElementsAre; | |
| using ::testing::ElementsAreArray; | |
| using ::testing::IsNull; | |
| using ::testing::SizeIs; | |
| TEST(ParserIntegrationTest, EmptyQuery) { | |
| std::string query = ""; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| EXPECT_THAT(tree_root, IsNull()); | |
| } | |
| TEST(ParserIntegrationTest, EmptyScoring) { | |
| std::string query = ""; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeScoring(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, SingleTerm) { | |
| std::string query = "foo"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // member | |
| // | | |
| // text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember))); | |
| } | |
| TEST(ParserIntegrationTest, ImplicitAnd) { | |
| std::string query = "foo bar"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // AND | |
| // / \ | |
| // member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, AND } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("AND", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, Or) { | |
| std::string query = "foo OR bar"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // OR | |
| // / \ | |
| // member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, OR } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("OR", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, And) { | |
| std::string query = "foo AND bar"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // AND | |
| // / \ | |
| // member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, AND } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("AND", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, Not) { | |
| std::string query = "NOT foo"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // NOT | |
| // | | |
| // member | |
| // | | |
| // text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, NOT } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("NOT", NodeType::kUnaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, Minus) { | |
| std::string query = "-foo"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // MINUS | |
| // | | |
| // member | |
| // | | |
| // text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, MINUS } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("MINUS", NodeType::kUnaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, Has) { | |
| std::string query = "subject:foo"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // : | |
| // / \ | |
| // member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, binaryOp } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("subject", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo(":", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, HasNested) { | |
| std::string query = "sender.name:foo"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // : | |
| // / \ | |
| // member member | |
| // / \ | | |
| // text text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, text, member, text, member, binaryOp } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("sender", NodeType::kText), | |
| EqualsNodeInfo("name", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo(":", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, EmptyFunction) { | |
| std::string query = "foo()"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // function | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { function } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kFunction))); | |
| } | |
| TEST(ParserIntegrationTest, FunctionSingleArg) { | |
| std::string query = "foo(\"bar\")"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // function | |
| // | | |
| // string | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { string, function } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("bar", NodeType::kString), | |
| EqualsNodeInfo("foo", NodeType::kFunction))); | |
| } | |
| TEST(ParserIntegrationTest, FunctionMultiArg) { | |
| std::string query = "foo(\"bar\", \"baz\")"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // function | |
| // / \ | |
| // string string | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { string, string, function } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("bar", NodeType::kString), | |
| EqualsNodeInfo("baz", NodeType::kString), | |
| EqualsNodeInfo("foo", NodeType::kFunction))); | |
| } | |
| TEST(ParserIntegrationTest, FunctionNested) { | |
| std::string query = "foo(bar())"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // function | |
| // | | |
| // function | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { function, function } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("bar", NodeType::kFunction), | |
| EqualsNodeInfo("foo", NodeType::kFunction))); | |
| } | |
| TEST(ParserIntegrationTest, FunctionWithTrailingSequence) { | |
| std::string query = "foo() OR bar"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // OR | |
| // / \ | |
| // function member | |
| // | | |
| // text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { function, text, member, OR } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kFunction), | |
| EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("OR", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, Composite) { | |
| std::string query = "foo OR (bar baz)"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // OR | |
| // / \ | |
| // member AND | |
| // | / \ | |
| // text member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, text, member, AND, OR } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("baz", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("AND", NodeType::kNaryOperator), | |
| EqualsNodeInfo("OR", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, CompositeWithTrailingSequence) { | |
| std::string query = "(bar baz) OR foo"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // OR | |
| // / \ | |
| // AND member | |
| // / \ | | |
| // member member text | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, AND, text, member, OR } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("baz", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("AND", NodeType::kNaryOperator), | |
| EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("OR", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, Complex) { | |
| std::string query = "foo bar:baz OR pal(\"bat\")"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // AND | |
| // / \ | |
| // member OR | |
| // | / \ | |
| // text : function | |
| // / \ | | |
| // member member string | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| // SimpleVisitor ordering | |
| // { text, member, text, member, text, member, :, string, function, OR, | |
| // AND } | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("foo", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("bar", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("baz", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo(":", NodeType::kNaryOperator), | |
| EqualsNodeInfo("bat", NodeType::kString), | |
| EqualsNodeInfo("pal", NodeType::kFunction), | |
| EqualsNodeInfo("OR", NodeType::kNaryOperator), | |
| EqualsNodeInfo("AND", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, InvalidHas) { | |
| std::string query = "foo:"; // No right hand operand to : | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidComposite) { | |
| std::string query = "(foo bar"; // No terminating RPAREN | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidMember) { | |
| std::string query = "foo."; // DOT must have succeeding TEXT | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidOr) { | |
| std::string query = "foo OR"; // No right hand operand to OR | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidAnd) { | |
| std::string query = "foo AND"; // No right hand operand to AND | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidNot) { | |
| std::string query = "NOT"; // No right hand operand to NOT | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidMinus) { | |
| std::string query = "-"; // No right hand operand to - | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidFunctionCallNoRparen) { | |
| std::string query = "foo("; // No terminating RPAREN | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, InvalidFunctionArgsHangingComma) { | |
| std::string query = "foo(\"bar\",)"; // no valid arg following COMMA | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserIntegrationTest, ScoringPlus) { | |
| std::string scoring = "1 + 1 + 1"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // PLUS | |
| // / | \ | |
| // member member member | |
| // | | | | |
| // text text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("PLUS", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringMinus) { | |
| std::string scoring = "1 - 1 - 1"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // MINUS | |
| // / | \ | |
| // member member member | |
| // | | | | |
| // text text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("MINUS", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringUnaryMinus) { | |
| std::string scoring = "1 + -1 + 1"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // PLUS | |
| // / | \ | |
| // member MINUS member | |
| // | | | | |
| // text member text | |
| // | | |
| // text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("MINUS", NodeType::kUnaryOperator), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("PLUS", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringPlusMinus) { | |
| std::string scoring = "11 + 12 - 13 + 14"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // PLUS | |
| // / \ | |
| // MINUS member | |
| // / \ | | |
| // PLUS member text | |
| // / \ | | |
| // member member text | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("11", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("12", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("PLUS", NodeType::kNaryOperator), | |
| EqualsNodeInfo("13", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("MINUS", NodeType::kNaryOperator), | |
| EqualsNodeInfo("14", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("PLUS", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringTimes) { | |
| std::string scoring = "1 * 1 * 1"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // TIMES | |
| // / | \ | |
| // member member member | |
| // | | | | |
| // text text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("TIMES", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringDiv) { | |
| std::string scoring = "1 / 1 / 1"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // DIV | |
| // / | \ | |
| // member member member | |
| // | | | | |
| // text text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("DIV", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringTimesDiv) { | |
| std::string scoring = "11 / 12 * 13 / 14 / 15"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // DIV | |
| // / | \ | |
| // TIMES member member | |
| // / \ | | | |
| // DIV member text text | |
| // / \ | | |
| // member member text | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("11", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("12", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("DIV", NodeType::kNaryOperator), | |
| EqualsNodeInfo("13", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("TIMES", NodeType::kNaryOperator), | |
| EqualsNodeInfo("14", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("15", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("DIV", NodeType::kNaryOperator))); | |
| } | |
| TEST(ParserIntegrationTest, ComplexScoring) { | |
| // With parentheses in function arguments. | |
| std::string scoring = "1 + pow((2 * sin(3)), 4) + -5 / 6"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| std::vector<NodeInfo> node = visitor.nodes(); | |
| EXPECT_THAT(node, | |
| ElementsAre(EqualsNodeInfo("1", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("2", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("3", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("sin", NodeType::kFunction), | |
| EqualsNodeInfo("TIMES", NodeType::kNaryOperator), | |
| EqualsNodeInfo("4", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("pow", NodeType::kFunction), | |
| EqualsNodeInfo("5", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("MINUS", NodeType::kUnaryOperator), | |
| EqualsNodeInfo("6", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("DIV", NodeType::kNaryOperator), | |
| EqualsNodeInfo("PLUS", NodeType::kNaryOperator))); | |
| // Without parentheses in function arguments. | |
| scoring = "1 + pow(2 * sin(3), 4) + -5 / 6"; | |
| lexer = Lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(lexer_tokens, std::move(lexer).ExtractTokens()); | |
| parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(tree_root, parser.ConsumeScoring()); | |
| visitor = SimpleVisitor(); | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), ElementsAreArray(node)); | |
| } | |
| TEST(ParserIntegrationTest, ScoringMemberFunction) { | |
| std::string scoring = "this.CreationTimestamp()"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // member | |
| // / \ | |
| // text function | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT( | |
| visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("this", NodeType::kText), | |
| EqualsNodeInfo("CreationTimestamp", NodeType::kFunction), | |
| EqualsNodeInfo("", NodeType::kMember))); | |
| } | |
| TEST(ParserIntegrationTest, QueryMemberFunction) { | |
| std::string query = "this.foo()"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // member | |
| // / \ | |
| // text function | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("this", NodeType::kText), | |
| EqualsNodeInfo("foo", NodeType::kFunction), | |
| EqualsNodeInfo("", NodeType::kMember))); | |
| } | |
| TEST(ParserIntegrationTest, ScoringComplexMemberFunction) { | |
| std::string scoring = "a.b.fun(c, d)"; | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeScoring()); | |
| // Expected AST: | |
| // member | |
| // / | \ | |
| // text text function | |
| // / \ | |
| // member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("a", NodeType::kText), | |
| EqualsNodeInfo("b", NodeType::kText), | |
| EqualsNodeInfo("c", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("d", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("fun", NodeType::kFunction), | |
| EqualsNodeInfo("", NodeType::kMember))); | |
| } | |
| TEST(ParserTest, QueryComplexMemberFunction) { | |
| std::string query = "this.abc.fun(def, ghi)"; | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Node> tree_root, | |
| parser.ConsumeQuery()); | |
| // Expected AST: | |
| // member | |
| // / | \ | |
| // text text function | |
| // / \ | |
| // member member | |
| // | | | |
| // text text | |
| SimpleVisitor visitor; | |
| tree_root->Accept(&visitor); | |
| EXPECT_THAT(visitor.nodes(), | |
| ElementsAre(EqualsNodeInfo("this", NodeType::kText), | |
| EqualsNodeInfo("abc", NodeType::kText), | |
| EqualsNodeInfo("def", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("ghi", NodeType::kText), | |
| EqualsNodeInfo("", NodeType::kMember), | |
| EqualsNodeInfo("fun", NodeType::kFunction), | |
| EqualsNodeInfo("", NodeType::kMember))); | |
| } | |
| TEST(ParserTest, QueryShouldNotStackOverflowAtMaxNumTokens) { | |
| // query = "(( ... (foo bar) ... ))" | |
| std::string query; | |
| for (int i = 0; i < Lexer::kMaxNumTokens / 2 - 1; ++i) { | |
| query.push_back('('); | |
| } | |
| query.append("foo bar"); | |
| for (int i = 0; i < Lexer::kMaxNumTokens / 2 - 1; ++i) { | |
| query.push_back(')'); | |
| } | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| EXPECT_THAT(lexer_tokens, SizeIs(Lexer::kMaxNumTokens)); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), IsOk()); | |
| } | |
| TEST(ParserTest, ScoringShouldNotStackOverflowAtMaxNumTokens) { | |
| // scoring = "(( ... (-1) ... ))" | |
| std::string scoring; | |
| for (int i = 0; i < Lexer::kMaxNumTokens / 2 - 1; ++i) { | |
| scoring.push_back('('); | |
| } | |
| scoring.append("-1"); | |
| for (int i = 0; i < Lexer::kMaxNumTokens / 2 - 1; ++i) { | |
| scoring.push_back(')'); | |
| } | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| EXPECT_THAT(lexer_tokens, SizeIs(Lexer::kMaxNumTokens)); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeScoring(), IsOk()); | |
| } | |
| TEST(ParserTest, InvalidQueryShouldNotStackOverflowAtMaxNumTokens) { | |
| std::string query; | |
| for (int i = 0; i < Lexer::kMaxNumTokens; ++i) { | |
| query.push_back('('); | |
| } | |
| Lexer lexer(query, Lexer::Language::QUERY); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| EXPECT_THAT(lexer_tokens, SizeIs(Lexer::kMaxNumTokens)); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeQuery(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| TEST(ParserTest, InvalidScoringShouldNotStackOverflowAtMaxNumTokens) { | |
| std::string scoring; | |
| for (int i = 0; i < Lexer::kMaxNumTokens; ++i) { | |
| scoring.push_back('('); | |
| } | |
| Lexer lexer(scoring, Lexer::Language::SCORING); | |
| ICING_ASSERT_OK_AND_ASSIGN(std::vector<Lexer::LexerToken> lexer_tokens, | |
| std::move(lexer).ExtractTokens()); | |
| EXPECT_THAT(lexer_tokens, SizeIs(Lexer::kMaxNumTokens)); | |
| Parser parser = Parser::Create(std::move(lexer_tokens)); | |
| EXPECT_THAT(parser.ConsumeScoring(), | |
| StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); | |
| } | |
| } // namespace | |
| } // namespace lib | |
| } // namespace icing |