lang_id/lang-id.h

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef NLP_SAFT_COMPONENTS_LANG_ID_MOBILE_LANG_ID_H_
#define NLP_SAFT_COMPONENTS_LANG_ID_MOBILE_LANG_ID_H_


#include <stddef.h>

#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "lang_id/common/lite_base/macros.h"
#include "lang_id/model-provider.h"

namespace libtextclassifier3 {
namespace mobile {
namespace lang_id {

// Forward-declaration of the class that performs all underlying work.
class LangIdImpl;

struct LangIdResult {
  // An n-best list of possible language codes for a given input sorted in
  // descending order according to each code's respective probability.
  //
  // This list is guaranteed to be non-empty after calling
  // LangId::FindLanguages.  The most likely language code is always the first
  // item in this array.
  //
  // If the model cannot make a prediction, this array contains a single result:
  // a language code LangId::kUnknownLanguageCode with probability 1.
  std::vector<std::pair<string, float>> predictions;
};

// Class for detecting the language of a document.
//
// Note: this class does not handle the details of loading the actual model.
// Those details have been "outsourced" to the ModelProvider class.
//
// This class is thread safe.
class LangId {
 public:
  // Standard BCP-47 language code for Unknown/Undetermined language.
  static const char kUnknownLanguageCode[];

  // Constructs a LangId object, based on |model_provider|.
  //
  // Note: we don't crash if we detect a problem at construction time (e.g., the
  // model provider can't read an underlying file).  Instead, we mark the
  // newly-constructed object as invalid; clients can invoke FindLanguage() on
  // an invalid object: nothing crashes, but accuracy will be bad.
  explicit LangId(std::unique_ptr<ModelProvider> model_provider);

  virtual ~LangId();

  // Computes the an n-best list of language codes and probabilities
  // corresponding to the most likely languages the given input text is written
  // in. The list is sorted in descending order by language probability.
  //
  // The input text consists of the |num_bytes| bytes that starts at |data|.
  //
  // Note: If this LangId object is not valid (see is_valid()) or if this LangId
  // object can't make a prediction, this method sets the LangIdResult to
  // contain a single entry with kUnknownLanguageCode with probability 1.
  void FindLanguages(const char *data, size_t num_bytes,
                     LangIdResult *result) const;

  // Convenience version of FindLanguages(const char *, size_t, LangIdResult *).
  void FindLanguages(const string &text, LangIdResult *result) const {
    FindLanguages(text.data(), text.size(), result);
  }

  // Returns language code for the most likely language for a piece of text.
  //
  // The input text consists of the |num_bytes| bytes that start at |data|.
  //
  // Note: this method reports the most likely (1-best) language only if its
  // probability is high enough; otherwise, it returns
  // LangId::kUnknownLanguageCode.  The specific probability threshold is tuned
  // to the needs of an early client.  If you need a different threshold, you
  // can use FindLanguages (plural) to get the full LangIdResult, and apply your
  // own threshold.
  //
  // Note: if this LangId object is not valid (see is_valid()) or if this LangId
  // object can't make a prediction, then this method returns
  // LangId::kUnknownLanguageCode.
  //
  string FindLanguage(const char *data, size_t num_bytes) const;

  // Convenience version of FindLanguage(const char *, size_t).
  string FindLanguage(const string &text) const {
    return FindLanguage(text.data(), text.size());
  }

  // Returns true if this object has been correctly initialized and is ready to
  // perform predictions.  For more info, see doc for LangId
  // constructor above.
  bool is_valid() const;

  // Returns the version of the model used by this LangId object.  On success,
  // the returned version number is a strictly positive integer.  Returns 0 if
  // the model version can not be determined (e.g., for old models that do not
  // specify a version number).
  int GetModelVersion() const;

  // Returns a typed property stored in the model file.
  float GetFloatProperty(const string &property, float default_value) const;

 private:
  // Pimpl ("pointer to implementation") pattern, to hide all internals from our
  // clients.
  std::unique_ptr<LangIdImpl> pimpl_;

  SAFTM_DISALLOW_COPY_AND_ASSIGN(LangId);
};

}  // namespace lang_id
}  // namespace mobile
}  // namespace nlp_saft

#endif  // NLP_SAFT_COMPONENTS_LANG_ID_MOBILE_LANG_ID_H_