blob: db0f83e677cf2c9eeec88b42f78041a9866d2cb9 [file] [log] [blame]
* Copyright (C) 2018 The Android Open Source Project
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
#include <mutex> // NOLINT: see comments for state_mutex_
#include <string>
#include "lang_id/common/fel/feature-extractor.h"
#include "lang_id/common/fel/task-context.h"
#include "lang_id/common/fel/workspace.h"
#include "lang_id/features/light-sentence-features.h"
#include "lang_id/light-sentence.h"
// TODO(abakalov): Add a test.
namespace libtextclassifier3 {
namespace mobile {
namespace lang_id {
// Class for computing continuous char ngram features.
// Feature function descriptor parameters:
// include_terminators(bool, false):
// If 'true', then splits the text based on spaces to get tokens, adds "^"
// to the beginning of each token, and adds "$" to the end of each token.
// NOTE: currently, we support only include_terminators=true.
// include_spaces(bool, false):
// If 'true', then includes char ngrams containing spaces.
// NOTE: currently, we support only include_spaces=false.
// use_equal_weight(bool, false):
// If 'true', then weighs each unique ngram by 1.0 / (number of unique
// ngrams in the input). Otherwise, weighs each unique ngram by (ngram
// count) / (total number of ngrams).
// NOTE: currently, we support only use_equal_weight=false.
// id_dim(int, 10000):
// The integer id of each char ngram is computed as follows:
// Hash32WithDefault(char ngram) % id_dim.
// size(int, 3):
// Only ngrams of this size will be extracted.
// NOTE: this class is not thread-safe. TODO(salcianu): make it thread-safe.
class ContinuousBagOfNgramsFunction : public LightSentenceFeature {
bool Setup(TaskContext *context) override;
bool Init(TaskContext *context) override;
// Appends the features computed from the sentence to the feature vector.
void Evaluate(const WorkspaceSet &workspaces, const LightSentence &sentence,
FeatureVector *result) const override;
// Auxiliary for Evaluate(). Fills counts_ and non_zero_count_indices_ (see
// below), and returns the total ngram count.
int ComputeNgramCounts(const LightSentence &sentence) const;
// Guards counts_ and non_zero_count_indices_. NOTE: we use std::* constructs
// (instead of absl::Mutex & co) to simplify porting to Android and to avoid
// pulling in absl (which increases our code size).
mutable std::mutex state_mutex_;
// counts_[i] is the count of all ngrams with id i. Work data for Evaluate().
// NOTE: we declare this vector as a field, such that its underlying capacity
// stays allocated in between calls to Evaluate().
mutable std::vector<int> counts_;
// Indices of non-zero elements of counts_. See comments for counts_.
mutable std::vector<int> non_zero_count_indices_;
// The integer id of each char ngram is computed as follows:
// Hash32WithDefaultSeed(char_ngram) % ngram_id_dimension_.
int ngram_id_dimension_;
// Only ngrams of size ngram_size_ will be extracted.
int ngram_size_;
} // namespace lang_id
} // namespace mobile
} // namespace nlp_saft