blob: 8e78970a9b9476ab0d492cf28c7db0f03082fdaf [file] [log] [blame]
// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
option optimize_for = LITE_RUNTIME;
package libtextclassifier;
// Represents a codepoint range [start, end) with its role for tokenization.
message TokenizationCodepointRange {
optional int32 start = 1;
optional int32 end = 2;
// Role of the codepoints in the range.
enum Role {
// Concatenates the codepoint to the current run of codepoints.
DEFAULT_ROLE = 0;
// Splits a run of codepoints before the current codepoint.
SPLIT_BEFORE = 0x1;
// Splits a run of codepoints after the current codepoint.
SPLIT_AFTER = 0x2;
// Discards the codepoint.
DISCARD_CODEPOINT = 0x4;
// Common values:
// Splits on the characters and discards them. Good e.g. for the space
// character.
WHITESPACE_SEPARATOR = 0x7;
// Each codepoint will be a separate token. Good e.g. for Chinese
// characters.
TOKEN_SEPARATOR = 0x3;
}
optional Role role = 3;
}