| // Copyright (C) 2022 Google LLC | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #include "icing/tokenization/rfc822-tokenizer.h" | |
| #include <algorithm> | |
| #include <deque> | |
| #include <queue> | |
| #include <string_view> | |
| #include <utility> | |
| #include <vector> | |
| #include "icing/tokenization/token.h" | |
| #include "icing/tokenization/tokenizer.h" | |
| #include "icing/util/character-iterator.h" | |
| #include "icing/util/i18n-utils.h" | |
| #include "icing/util/status-macros.h" | |
| #include "unicode/umachine.h" | |
| namespace icing { | |
| namespace lib { | |
| namespace { | |
| bool IsDelimiter(UChar32 c) { return c == ',' || c == ';' || c == '\n'; } | |
| } // namespace | |
| class Rfc822TokenIterator : public Tokenizer::Iterator { | |
| public: | |
| // Cursor is the index into the string_view, text_end_ is the length. | |
| explicit Rfc822TokenIterator(std::string_view text) | |
| : text_(std::move(text)), | |
| iterator_(text, 0, 0, 0), | |
| text_end_(text.length()), | |
| token_index_(-1) {} | |
| // Advance will move token_index_ past the end of tokens_ | |
| bool Advance() override { | |
| // Stop the token index on a RFC822 token, or one past the end, where the | |
| // next RFC822 token will be if more are generated. | |
| do { | |
| token_index_++; | |
| } while (token_index_ < tokens_.size() && | |
| tokens_[token_index_].type != Token::Type::RFC822_TOKEN); | |
| // There is still something left, possible if we rewinded and call Advance | |
| if (token_index_ < tokens_.size()) { | |
| return true; | |
| } | |
| // Done with the entire string_view. | |
| if (iterator_.utf8_index() >= text_end_) { | |
| return false; | |
| } | |
| // Parsing a new email, update the current email marker. | |
| AdvancePastWhitespace(); | |
| // This may return false, as in the case of "<alex>,,", where after | |
| // processing <alex>, there are no more tokens. | |
| return GetNextRfc822Token(); | |
| } | |
| // Returns the current token group, an RFC822_TOKEN along with all it's | |
| // subtokens. For example, "tim@google.com" will return all tokens generated | |
| // from that text. | |
| // | |
| // Returns: | |
| // A vector of Tokens on success | |
| // An empty vector if the token list is empty | |
| // An empty vector if the index is past the end of the token list | |
| std::vector<Token> GetTokens() const override { | |
| std::vector<Token> result; | |
| if (token_index_ < tokens_.size() && token_index_ >= 0) { | |
| int index = token_index_; | |
| do { | |
| result.push_back(tokens_[index]); | |
| } while (++index < tokens_.size() && | |
| tokens_[index].type != Token::Type::RFC822_TOKEN); | |
| } | |
| return result; | |
| } | |
| bool ResetToTokenStartingAfter(int32_t utf32_offset) override { | |
| CharacterIterator tracker(text_); | |
| for (int new_index = 0; new_index < tokens_.size(); ++new_index) { | |
| const Token& t = tokens_[new_index]; | |
| if (t.type != Token::Type::RFC822_TOKEN) { | |
| continue; | |
| } | |
| tracker.AdvanceToUtf8(t.text.begin() - text_.begin()); | |
| if (tracker.utf32_index() > utf32_offset) { | |
| token_index_ = new_index; | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| // This will attempt to reset the token_index to point to the last token | |
| // ending before an offset. If it fails, due to there not being any tokens | |
| // before the offset, the token index will become -1. | |
| bool ResetToTokenEndingBefore(int32_t utf32_offset) override { | |
| // First, advance until we pass offset or Advance is false | |
| if (tokens_.empty()) { | |
| if (!Advance()) { | |
| // No tokens available, and Advancing doesn't get more, so return false. | |
| return false; | |
| } | |
| } | |
| CharacterIterator tracker(text_); | |
| // Keep advancing until we parse all the emails, or run past the offset. | |
| // Advance will always make token_index_ point to an RFC822_TOKEN, so we can | |
| // look at that tokens text end to determine if it ends before the offset. | |
| // This first loop will guarantee that we end up either past the offset or | |
| // at the end. | |
| do { | |
| tracker.AdvanceToUtf8(tokens_[token_index_].text.end() - text_.begin()); | |
| // When we Advance and have to convert names to email addresses, it's | |
| // possible that multiple RFC822 tokens are added. We need to advance | |
| // through these one at a time, we cannot skip to the top of the line. | |
| } while (tracker.utf32_index() <= utf32_offset && Advance()); | |
| // We are either past the offset or at the end. Either way, we now work | |
| // backwards and reset to the first (highest index) RFC822_TOKEN we find. | |
| while (--token_index_ >= 0) { | |
| if (tokens_[token_index_].type != Token::Type::RFC822_TOKEN) { | |
| continue; | |
| } | |
| tracker.MoveToUtf8(tokens_[token_index_].text.end() - text_.begin()); | |
| if (tracker.utf32_index() <= utf32_offset) { | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| // Returns a character iterator to the start of the token. | |
| libtextclassifier3::StatusOr<CharacterIterator> CalculateTokenStart() | |
| override { | |
| CharacterIterator token_start = iterator_; | |
| token_start.MoveToUtf8(GetTokens().at(0).text.begin() - text_.begin()); | |
| return token_start; | |
| } | |
| // Returns a character iterator to right after the end of the token. | |
| libtextclassifier3::StatusOr<CharacterIterator> CalculateTokenEndExclusive() | |
| override { | |
| CharacterIterator token_end = iterator_; | |
| token_end.MoveToUtf8(GetTokens().at(0).text.end() - text_.begin()); | |
| return token_end; | |
| } | |
| // Reset to start moves to the state we're in after the first Advance(). | |
| bool ResetToStart() override { | |
| token_index_ = -1; | |
| return Advance(); | |
| } | |
| private: | |
| // Advance until the next email delimiter, generating as many tokens as | |
| // necessary. | |
| bool GetNextRfc822Token() { | |
| if (iterator_.utf8_index() >= text_end_) { | |
| return false; | |
| } | |
| int token_start = iterator_.utf8_index(); | |
| bool address_found = false; | |
| bool name_found = false; | |
| std::vector<Token> next_tokens; | |
| Token rfc822(Token::Type::RFC822_TOKEN); | |
| // We start at unquoted and run until a ",;\n<( . | |
| while (iterator_.utf8_index() < text_end_) { | |
| UChar32 c = iterator_.GetCurrentChar(); | |
| if (IsDelimiter(c)) { | |
| // End of the token, advance cursor past all delimiters then quit. | |
| rfc822.text = | |
| text_.substr(token_start, iterator_.utf8_index() - token_start); | |
| UChar32 delimiter; | |
| do { | |
| AdvanceCursor(); | |
| delimiter = iterator_.GetCurrentChar(); | |
| // If we get current char on the end, it is not a delimiter so this | |
| // loop will end | |
| } while (IsDelimiter(delimiter)); | |
| break; | |
| } | |
| std::vector<Token> consume_result; | |
| if (c == '"') { | |
| consume_result = ConsumeQuotedSection(); | |
| name_found |= !consume_result.empty(); | |
| } else if (c == '(') { | |
| consume_result = ConsumeParenthesizedSection(); | |
| } else if (c == '<') { | |
| // Only set address_found to true if ConsumeAdress returns true. | |
| // Otherwise, keep address_found as is to prevent setting address_found | |
| // back to false if it is true. | |
| consume_result = ConsumeAddress(); | |
| address_found |= !consume_result.empty(); | |
| } else { | |
| consume_result = ConsumeUnquotedSection(); | |
| name_found |= !consume_result.empty(); | |
| } | |
| next_tokens.insert(next_tokens.end(), consume_result.begin(), | |
| consume_result.end()); | |
| } | |
| if (iterator_.utf8_index() >= text_end_) { | |
| rfc822.text = text_.substr(token_start, text_end_ - token_start); | |
| } | |
| // If an address is found, use the tokens we have. | |
| // If an address isn't found, and a name isn't found, also use the tokens | |
| // we have. | |
| // If an address isn't found but a name is, convert name Tokens to email | |
| // Tokens. | |
| if (!address_found && name_found) { | |
| // We don't add the rfc822 token, as it will be handled by | |
| // ConvertNameToEmail. | |
| std::vector<Token> converted_tokens = ConvertNameToEmail(next_tokens); | |
| tokens_.insert(tokens_.end(), converted_tokens.begin(), | |
| converted_tokens.end()); | |
| } else { | |
| if (next_tokens.empty()) { | |
| // Tokens may not be generated in the case of ",,,,,," | |
| return false; | |
| } else { | |
| // If tokens were generated, push back the RFC822 token for them | |
| tokens_.push_back(rfc822); | |
| tokens_.insert(tokens_.end(), next_tokens.begin(), next_tokens.end()); | |
| } | |
| } | |
| return true; | |
| } | |
| // We allow for the "First Last <email>" format, but if there is no email in | |
| // brackets, we won't allow for unquoted spaces. For example, the input | |
| // "alex@google.com tim@google.com" has an unquoted space, so we will split | |
| // it into two emails. We don't need to find more tokens, we just need to | |
| // find @ signs and spaces and convert name tokens to parts of the email. | |
| std::vector<Token> ConvertNameToEmail(std::vector<Token>& name_tokens) { | |
| if (name_tokens.empty()) { | |
| return name_tokens; | |
| } | |
| // There will only be names and comments, and they will be in order. | |
| std::vector<Token> converted_tokens; | |
| // Start at the beginning of the current email. | |
| CharacterIterator scanner(text_); | |
| scanner.MoveToUtf8(name_tokens[0].text.begin() - text_.begin()); | |
| int token_processed_index = 0; | |
| bool in_quote = false; | |
| // Setting at_sign_index to before the beginning, it'll only be set to | |
| // something else if we find an @ sign | |
| const char* at_sign_index = nullptr; | |
| // Run to the end | |
| while (scanner.utf8_index() < iterator_.utf8_index()) { | |
| const char* end_of_token = nullptr; | |
| UChar32 c = scanner.GetCurrentChar(); | |
| if (c == '\\') { | |
| // Skip the slash, as well as the following token. | |
| scanner.AdvanceToUtf32(scanner.utf32_index() + 1); | |
| scanner.AdvanceToUtf32(scanner.utf32_index() + 1); | |
| continue; | |
| } | |
| if (c == '"') { | |
| in_quote = !in_quote; | |
| } | |
| if (c == '@') { | |
| at_sign_index = text_.begin() + scanner.utf8_index(); | |
| } | |
| // If the next character is the end OR we hit an unquoted space. | |
| if (scanner.utf8_index() + i18n_utils::GetUtf8Length(c) == | |
| iterator_.utf8_index() || | |
| (!in_quote && c == ' ')) { | |
| if (!in_quote && c == ' ') { | |
| end_of_token = text_.begin() + scanner.utf8_index(); | |
| } else { | |
| end_of_token = text_.begin() + iterator_.utf8_index(); | |
| } | |
| std::deque<Token> more_tokens = ConvertOneNameToEmail( | |
| name_tokens, at_sign_index, end_of_token, token_processed_index); | |
| converted_tokens.insert(converted_tokens.end(), more_tokens.begin(), | |
| more_tokens.end()); | |
| // Reset the at_sign_index | |
| at_sign_index = nullptr; | |
| } | |
| scanner.AdvanceToUtf32(scanner.utf32_index() + 1); | |
| } | |
| // It's possible we left something out. | |
| if (token_processed_index < name_tokens.size()) { | |
| std::deque<Token> more_tokens = | |
| ConvertOneNameToEmail(name_tokens, at_sign_index, | |
| name_tokens[name_tokens.size() - 1].text.end(), | |
| token_processed_index); | |
| converted_tokens.insert(converted_tokens.end(), more_tokens.begin(), | |
| more_tokens.end()); | |
| } | |
| return converted_tokens; | |
| } | |
| // Once a name is determined to be an address, convert its tokens to address | |
| // tokens. | |
| std::deque<Token> ConvertOneNameToEmail(const std::vector<Token>& name_tokens, | |
| const char* at_sign_index, | |
| const char* end_of_token, | |
| int& token_processed_index) { | |
| const char* address_start = nullptr; | |
| const char* local_address_end = nullptr; | |
| const char* host_address_start = nullptr; | |
| const char* address_end = nullptr; | |
| const char* token_start = nullptr; | |
| const char* token_end = nullptr; | |
| std::deque<Token> converted_tokens; | |
| // Transform tokens up to end of token pointer. | |
| for (; token_processed_index < name_tokens.size(); | |
| ++token_processed_index) { | |
| const Token& token = name_tokens[token_processed_index]; | |
| if (token.text.end() > end_of_token) { | |
| break; | |
| } | |
| std::string_view text = token.text; | |
| // We need to do this both for comment and name tokens. Comment tokens | |
| // will get a corresponding RFC822 token, but not an address or local | |
| // address. | |
| if (token_start == nullptr) { | |
| token_start = text.begin(); | |
| } | |
| token_end = text.end(); | |
| if (token.type == Token::Type::RFC822_COMMENT) { | |
| // Comment tokens will stay as they are. | |
| converted_tokens.push_back(token); | |
| } else if (token.type == Token::Type::RFC822_NAME) { | |
| // Names need to be converted to address tokens. We keep the order of | |
| // which the name tokens appeared. Name tokens that appear before an | |
| // @ sign in the name will become RFC822_ADDRESS_COMPONENT_LOCAL, and | |
| // those after will become RFC822_ADDRESS_COMPONENT_HOST. We aren't | |
| // able to determine RFC822_ADDRESS, RFC822_LOCAL_ADDRESS, and | |
| // RFC_HOST_ADDRESS before checking the name tokens, so they will be | |
| // added after the component tokens. | |
| if (address_start == nullptr) { | |
| address_start = text.begin(); | |
| } | |
| address_end = text.end(); | |
| if (text.begin() > at_sign_index) { | |
| if (host_address_start == nullptr) { | |
| host_address_start = text.begin(); | |
| } | |
| // Once this is hit, we switch to COMPONENT_HOST and mark end of the | |
| // local address | |
| converted_tokens.push_back( | |
| Token(Token::Type::RFC822_ADDRESS_COMPONENT_HOST, token.text)); | |
| } else { | |
| local_address_end = text.end(); | |
| converted_tokens.push_back( | |
| Token(Token::Type::RFC822_ADDRESS_COMPONENT_LOCAL, token.text)); | |
| } | |
| } | |
| } | |
| if (address_start != nullptr) { | |
| converted_tokens.push_back( | |
| Token(Token::Type::RFC822_ADDRESS, | |
| std::string_view(address_start, address_end - address_start))); | |
| if (local_address_end != nullptr) { | |
| converted_tokens.push_back( | |
| Token(Token::Type::RFC822_LOCAL_ADDRESS, | |
| std::string_view(address_start, | |
| local_address_end - address_start))); | |
| } | |
| } | |
| if (host_address_start != nullptr && host_address_start < address_end) { | |
| converted_tokens.push_back( | |
| Token(Token::Type::RFC822_HOST_ADDRESS, | |
| text_.substr(host_address_start - text_.begin(), | |
| address_end - host_address_start))); | |
| } | |
| if (token_start != nullptr) { | |
| converted_tokens.push_front( | |
| Token(Token::Type::RFC822_TOKEN, | |
| std::string_view(token_start, token_end - token_start))); | |
| } | |
| return converted_tokens; | |
| } | |
| // Returns name tokens in an unquoted section. This is useful in case we do | |
| // not find an address and have to use the name. An unquoted section may look | |
| // like "Alex Sav", or "alex@google.com". In the absense of a bracketed email | |
| // address, the unquoted section will be used as the email address along with | |
| // the quoted section. | |
| std::vector<Token> ConsumeUnquotedSection() { | |
| UChar32 c; | |
| int token_start = -1; | |
| std::vector<Token> next_tokens; | |
| // Advance to another state or a character marking the end of token, one | |
| // of \n,; . | |
| while (iterator_.utf8_index() < text_end_) { | |
| c = iterator_.GetCurrentChar(); | |
| if (i18n_utils::IsAlphaNumeric(c)) { | |
| if (token_start == -1) { | |
| // Start recording | |
| token_start = iterator_.utf8_index(); | |
| } | |
| AdvanceCursor(); | |
| } else { | |
| if (token_start != -1) { | |
| // The character is non alphabetic, save a token. | |
| next_tokens.push_back(Token( | |
| Token::Type::RFC822_NAME, | |
| text_.substr(token_start, iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| if (c == '"' || c == '<' || c == '(' || IsDelimiter(c)) { | |
| // Stay on the token. | |
| break; | |
| } | |
| AdvanceCursor(); | |
| } | |
| } | |
| if (token_start != -1) { | |
| next_tokens.push_back(Token( | |
| Token::Type::RFC822_NAME, | |
| text_.substr(token_start, iterator_.utf8_index() - token_start))); | |
| } | |
| return next_tokens; | |
| } | |
| // Names that are within quotes should have all characters blindly | |
| // unescaped. When a name is made into an address, it isn't re-escaped. | |
| // Returns name tokens found in a quoted section. This is useful in case we do | |
| // not find an address and have to use the name. The quoted section may | |
| // contain whitespaces. | |
| std::vector<Token> ConsumeQuotedSection() { | |
| // Get past the first quote. | |
| AdvanceCursor(); | |
| bool end_quote_found = false; | |
| std::vector<Token> next_tokens; | |
| UChar32 c; | |
| int token_start = -1; | |
| while (!end_quote_found && (iterator_.utf8_index() < text_end_)) { | |
| c = iterator_.GetCurrentChar(); | |
| if (i18n_utils::IsAlphaNumeric(c)) { | |
| if (token_start == -1) { | |
| // Start tracking the token. | |
| token_start = iterator_.utf8_index(); | |
| } | |
| AdvanceCursor(); | |
| } else { | |
| // Non- alphabetic | |
| if (c == '\\') { | |
| // A backslash, let's look at the next character. | |
| CharacterIterator temp = iterator_; | |
| temp.AdvanceToUtf32(iterator_.utf32_index() + 1); | |
| UChar32 n = temp.GetCurrentChar(); | |
| if (i18n_utils::IsAlphaNumeric(n)) { | |
| // The next character is alphabetic, skip the slash and don't end | |
| // the last token. For quoted sections, the only things that are | |
| // escaped are double quotes and slashes. For example, in "a\lex", | |
| // an l appears after the slash. We want to treat this as if it | |
| // was just "alex". So we tokenize it as <RFC822_NAME, "a\lex">. | |
| AdvanceCursor(); | |
| } else { | |
| // Not alphabetic, so save the last token if necessary. | |
| if (token_start != -1) { | |
| next_tokens.push_back( | |
| Token(Token::Type::RFC822_NAME, | |
| text_.substr(token_start, | |
| iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| // Skip the backslash. | |
| AdvanceCursor(); | |
| if (n == '"' || n == '\\' || n == '@') { | |
| // Skip these too if they're next. | |
| AdvanceCursor(); | |
| } | |
| } | |
| } else { | |
| // Not a backslash. | |
| if (token_start != -1) { | |
| next_tokens.push_back( | |
| Token(Token::Type::RFC822_NAME, | |
| text_.substr(token_start, | |
| iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| if (c == '"') { | |
| end_quote_found = true; | |
| } | |
| // Advance one more time to get past the non-alphabetic character. | |
| AdvanceCursor(); | |
| } | |
| } | |
| } | |
| if (token_start != -1) { | |
| next_tokens.push_back(Token( | |
| Token::Type::RFC822_NAME, | |
| text_.substr(token_start, iterator_.utf8_index() - token_start))); | |
| } | |
| return next_tokens; | |
| } | |
| // '(', ')', '\\' chars should be escaped. All other escaped chars should be | |
| // unescaped. | |
| std::vector<Token> ConsumeParenthesizedSection() { | |
| // Skip the initial ( | |
| AdvanceCursor(); | |
| int paren_layer = 1; | |
| UChar32 c; | |
| std::vector<Token> next_tokens; | |
| int token_start = -1; | |
| while (paren_layer > 0 && (iterator_.utf8_index() < text_end_)) { | |
| c = iterator_.GetCurrentChar(); | |
| if (i18n_utils::IsAlphaNumeric(c)) { | |
| if (token_start == -1) { | |
| // Start tracking a token. | |
| token_start = iterator_.utf8_index(); | |
| } | |
| AdvanceCursor(); | |
| } else { | |
| // Non alphabetic. | |
| if (c == '\\') { | |
| // A backslash, let's look at the next character. | |
| UChar32 n = i18n_utils::GetUChar32At(text_.begin(), text_.length(), | |
| iterator_.utf8_index() + 1); | |
| if (i18n_utils::IsAlphaNumeric(n)) { | |
| // Alphabetic, skip the slash and don't end the last token. | |
| AdvanceCursor(); | |
| } else { | |
| // Not alphabetic, save the last token if necessary. | |
| if (token_start != -1) { | |
| next_tokens.push_back( | |
| Token(Token::Type::RFC822_COMMENT, | |
| text_.substr(token_start, | |
| iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| // Skip the backslash. | |
| AdvanceCursor(); | |
| if (n == ')' || n == '(' || n == '\\') { | |
| // Skip these too if they're next. | |
| AdvanceCursor(); | |
| } | |
| } | |
| } else { | |
| // Not a backslash. | |
| if (token_start != -1) { | |
| next_tokens.push_back( | |
| Token(Token::Type::RFC822_COMMENT, | |
| text_.substr(token_start, | |
| iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| if (c == '(') { | |
| paren_layer++; | |
| } else if (c == ')') { | |
| paren_layer--; | |
| } | |
| AdvanceCursor(); | |
| } | |
| } | |
| } | |
| if (token_start != -1) { | |
| // Ran past the end of text_ without getting the last token. | |
| // substr returns "a view of the substring [pos, pos + // rcount), where | |
| // rcount is the smaller of count and size() - pos" therefore the count | |
| // argument can be any value >= this->cursor - token_start. Therefore, | |
| // ignoring the mutation warning. | |
| next_tokens.push_back(Token( | |
| Token::Type::RFC822_COMMENT, | |
| text_.substr(token_start, iterator_.utf8_index() - token_start))); | |
| } | |
| return next_tokens; | |
| } | |
| // Returns tokens found in the address. | |
| std::vector<Token> ConsumeAddress() { | |
| // Skip the first <. | |
| AdvanceCursor(); | |
| // Save the start position. | |
| CharacterIterator address_start_iterator = iterator_; | |
| std::vector<Token> next_tokens; | |
| // Place the at sign on the '<', so that if no at_sign is found, the default | |
| // is that the entire address is the host part. | |
| int at_sign = -1; | |
| int address_end = -1; | |
| UChar32 c = iterator_.GetCurrentChar(); | |
| // Quick scan for @ and > signs. | |
| while (c != '>' && iterator_.utf8_index() < text_end_) { | |
| AdvanceCursor(); | |
| c = iterator_.GetCurrentChar(); | |
| if (c == '@') { | |
| at_sign = iterator_.utf8_index(); | |
| } | |
| } | |
| if (iterator_.utf8_index() <= address_start_iterator.utf8_index()) { | |
| // There is nothing between the brackets, either we have "<" or "<>". | |
| return next_tokens; | |
| } | |
| // Either we find a > or run to the end, either way this is the end of the | |
| // address. The ending bracket will be handled by ConsumeUnquoted. | |
| address_end = iterator_.utf8_index(); | |
| // Reset to the start. | |
| iterator_ = address_start_iterator; | |
| int address_start = address_start_iterator.utf8_index(); | |
| Token::Type type = Token::Type::RFC822_ADDRESS_COMPONENT_LOCAL; | |
| // Create a local address token. | |
| if (at_sign != -1) { | |
| next_tokens.push_back( | |
| Token(Token::Type::RFC822_LOCAL_ADDRESS, | |
| text_.substr(address_start, at_sign - address_start))); | |
| } else { | |
| // All the tokens in the address are host components. | |
| type = Token::Type::RFC822_ADDRESS_COMPONENT_HOST; | |
| // If no @ is found, treat the entire address as the host address. | |
| at_sign = address_start - 1; | |
| } | |
| // The only case where we don't have a host address part is something like | |
| // <localaddress@>. If there is no @, the at_sign is the default -1, and the | |
| // host address is [0, address_end). | |
| int host_address_start = at_sign + 1; | |
| if (host_address_start < address_end) { | |
| next_tokens.push_back(Token( | |
| Token::Type::RFC822_HOST_ADDRESS, | |
| text_.substr(host_address_start, address_end - host_address_start))); | |
| } | |
| next_tokens.push_back( | |
| Token(Token::Type::RFC822_ADDRESS, | |
| text_.substr(address_start, address_end - address_start))); | |
| int token_start = -1; | |
| while (iterator_.utf8_index() < address_end) { | |
| c = iterator_.GetCurrentChar(); | |
| if (i18n_utils::IsAlphaNumeric(c)) { | |
| if (token_start == -1) { | |
| token_start = iterator_.utf8_index(); | |
| } | |
| } else { | |
| // non alphabetic | |
| if (c == '\\') { | |
| // A backslash, let's look at the next character. | |
| CharacterIterator temp = iterator_; | |
| temp.AdvanceToUtf32(iterator_.utf32_index() + 1); | |
| UChar32 n = temp.GetCurrentChar(); | |
| if (!i18n_utils::IsAlphaNumeric(n)) { | |
| // Not alphabetic, end the last token if necessary. | |
| if (token_start != -1) { | |
| next_tokens.push_back(Token( | |
| type, text_.substr(token_start, | |
| iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| } | |
| } else { | |
| // Not backslash. | |
| if (token_start != -1) { | |
| next_tokens.push_back(Token( | |
| type, text_.substr(token_start, | |
| iterator_.utf8_index() - token_start))); | |
| token_start = -1; | |
| } | |
| // Switch to host component tokens. | |
| if (iterator_.utf8_index() == at_sign) { | |
| type = Token::Type::RFC822_ADDRESS_COMPONENT_HOST; | |
| } | |
| } | |
| } | |
| AdvanceCursor(); | |
| } | |
| if (token_start != -1) { | |
| next_tokens.push_back(Token( | |
| type, | |
| text_.substr(token_start, iterator_.utf8_index() - token_start))); | |
| } | |
| // Unquoted will handle the closing bracket > if these is one. | |
| return next_tokens; | |
| } | |
| void AdvanceCursor() { | |
| iterator_.AdvanceToUtf32(iterator_.utf32_index() + 1); | |
| } | |
| void AdvancePastWhitespace() { | |
| while (i18n_utils::IsWhitespaceAt(text_, iterator_.utf8_index())) { | |
| AdvanceCursor(); | |
| } | |
| } | |
| std::string_view text_; | |
| CharacterIterator iterator_; | |
| int text_end_; | |
| // A temporary store of Tokens. As we advance through the provided string, | |
| // we parse entire addresses at a time rather than one token at a time. | |
| // However, since we call the tokenizer with Advance() alternating with | |
| // GetToken(), we need to store tokens for subsequent GetToken calls if | |
| // Advance generates multiple tokens (it usually does). A vector is used as | |
| // we need to iterate back and forth through tokens during snippeting. It is | |
| // cleared by the destructor. | |
| std::vector<Token> tokens_; | |
| // Index to keep track of where we are in tokens_. This will always be set to | |
| // point to an RFC822_TOKEN, or one past the end of the tokens_ vector. The | |
| // only exception is before the first Advance call. | |
| int token_index_; | |
| }; | |
| libtextclassifier3::StatusOr<std::unique_ptr<Tokenizer::Iterator>> | |
| Rfc822Tokenizer::Tokenize(std::string_view text) const { | |
| return std::make_unique<Rfc822TokenIterator>(text); | |
| } | |
| libtextclassifier3::StatusOr<std::vector<Token>> Rfc822Tokenizer::TokenizeAll( | |
| std::string_view text) const { | |
| ICING_ASSIGN_OR_RETURN(std::unique_ptr<Tokenizer::Iterator> iterator, | |
| Tokenize(text)); | |
| std::vector<Token> tokens; | |
| while (iterator->Advance()) { | |
| std::vector<Token> batch_tokens = iterator->GetTokens(); | |
| tokens.insert(tokens.end(), batch_tokens.begin(), batch_tokens.end()); | |
| } | |
| return tokens; | |
| } | |
| } // namespace lib | |
| } // namespace icing |