utils/resources.cc

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "utils/resources.h"
#include "utils/base/logging.h"
#include "utils/zlib/buffer_generated.h"
#include "utils/zlib/zlib.h"

namespace libtextclassifier3 {
namespace {
bool isWildcardMatch(const flatbuffers::String* left,
                     const std::string& right) {
  return (left == nullptr || right.empty());
}

bool isExactMatch(const flatbuffers::String* left, const std::string& right) {
  if (left == nullptr) {
    return right.empty();
  }
  return left->str() == right;
}

}  // namespace

int Resources::LocaleMatch(const Locale& locale,
                           const LanguageTag* entry_locale) const {
  int match = LOCALE_NO_MATCH;
  if (isExactMatch(entry_locale->language(), locale.Language())) {
    match |= LOCALE_LANGUAGE_MATCH;
  } else if (isWildcardMatch(entry_locale->language(), locale.Language())) {
    match |= LOCALE_LANGUAGE_WILDCARD_MATCH;
  }

  if (isExactMatch(entry_locale->script(), locale.Script())) {
    match |= LOCALE_SCRIPT_MATCH;
  } else if (isWildcardMatch(entry_locale->script(), locale.Script())) {
    match |= LOCALE_SCRIPT_WILDCARD_MATCH;
  }

  if (isExactMatch(entry_locale->region(), locale.Region())) {
    match |= LOCALE_REGION_MATCH;
  } else if (isWildcardMatch(entry_locale->region(), locale.Region())) {
    match |= LOCALE_REGION_WILDCARD_MATCH;
  }

  return match;
}

const ResourceEntry* Resources::FindResource(
    const StringPiece resource_name) const {
  if (resources_ == nullptr || resources_->resource_entry() == nullptr) {
    TC3_LOG(ERROR) << "No resources defined.";
    return nullptr;
  }
  const ResourceEntry* entry =
      resources_->resource_entry()->LookupByKey(resource_name.data());
  if (entry == nullptr) {
    TC3_LOG(ERROR) << "Resource " << resource_name.ToString() << " not found";
    return nullptr;
  }
  return entry;
}

int Resources::BestResourceForLocales(
    const ResourceEntry* resource, const std::vector<Locale>& locales) const {
  // Find best match based on locale.
  int resource_id = -1;
  int locale_match = LOCALE_NO_MATCH;
  const auto* resources = resource->resource();
  for (int user_locale = 0; user_locale < locales.size(); user_locale++) {
    if (!locales[user_locale].IsValid()) {
      continue;
    }
    for (int i = 0; i < resources->size(); i++) {
      for (const int locale_id : *resources->Get(i)->locale()) {
        const int candidate_match = LocaleMatch(
            locales[user_locale], resources_->locale()->Get(locale_id));

        // Only consider if at least the language matches.
        if ((candidate_match & LOCALE_LANGUAGE_MATCH) == 0 &&
            (candidate_match & LOCALE_LANGUAGE_WILDCARD_MATCH) == 0) {
          continue;
        }

        if (candidate_match > locale_match) {
          locale_match = candidate_match;
          resource_id = i;
        }
      }
    }

    // If the language matches exactly, we are already finished.
    // We found an exact language match.
    if (locale_match & LOCALE_LANGUAGE_MATCH) {
      return resource_id;
    }
  }
  return resource_id;
}

bool Resources::GetResourceContent(const std::vector<Locale>& locales,
                                   const StringPiece resource_name,
                                   std::string* result) const {
  const ResourceEntry* entry = FindResource(resource_name);
  if (entry == nullptr || entry->resource() == nullptr) {
    return false;
  }

  int resource_id = BestResourceForLocales(entry, locales);
  if (resource_id < 0) {
    return false;
  }
  const auto* resource = entry->resource()->Get(resource_id);
  if (resource->content() != nullptr) {
    *result = resource->content()->str();
    return true;
  } else if (resource->compressed_content() != nullptr) {
    std::unique_ptr<ZlibDecompressor> decompressor = ZlibDecompressor::Instance(
        resources_->compression_dictionary()->data(),
        resources_->compression_dictionary()->size());
    if (decompressor != nullptr &&
        decompressor->MaybeDecompress(resource->compressed_content(), result)) {
      return true;
    }
  }
  return false;
}

bool CompressResources(ResourcePoolT* resources,
                       const bool build_compression_dictionary,
                       const int dictionary_sample_every) {
  std::vector<unsigned char> dictionary;
  if (build_compression_dictionary) {
    {
      // Build up a compression dictionary.
      std::unique_ptr<ZlibCompressor> compressor = ZlibCompressor::Instance();
      int i = 0;
      for (auto& entry : resources->resource_entry) {
        for (auto& resource : entry->resource) {
          if (resource->content.empty()) {
            continue;
          }
          i++;

          // Use a sample of the entries to build up a custom compression
          // dictionary. Using all entries will generally not give a benefit
          // for small data sizes, so we subsample here.
          if (i % dictionary_sample_every != 0) {
            continue;
          }
          CompressedBufferT compressed_content;
          compressor->Compress(resource->content, &compressed_content);
        }
      }
      compressor->GetDictionary(&dictionary);
      resources->compression_dictionary.assign(
          dictionary.data(), dictionary.data() + dictionary.size());
    }
  }

  for (auto& entry : resources->resource_entry) {
    for (auto& resource : entry->resource) {
      if (resource->content.empty()) {
        continue;
      }
      // Try compressing the data.
      std::unique_ptr<ZlibCompressor> compressor =
          build_compression_dictionary
              ? ZlibCompressor::Instance(dictionary.data(), dictionary.size())
              : ZlibCompressor::Instance();
      if (!compressor) {
        TC3_LOG(ERROR) << "Cannot create zlib compressor.";
        return false;
      }

      CompressedBufferT compressed_content;
      compressor->Compress(resource->content, &compressed_content);

      // Only keep compressed version if smaller.
      if (compressed_content.uncompressed_size >
          compressed_content.buffer.size()) {
        resource->content.clear();
        resource->compressed_content.reset(new CompressedBufferT);
        *resource->compressed_content = compressed_content;
      }
    }
  }
  return true;
}

std::string CompressSerializedResources(const std::string& resources,
                                        const int dictionary_sample_every) {
  std::unique_ptr<ResourcePoolT> unpacked_resources(
      flatbuffers::GetRoot<ResourcePool>(resources.data())->UnPack());
  TC3_CHECK(unpacked_resources != nullptr);
  TC3_CHECK(
      CompressResources(unpacked_resources.get(), dictionary_sample_every));
  flatbuffers::FlatBufferBuilder builder;
  builder.Finish(ResourcePool::Pack(builder, unpacked_resources.get()));
  return std::string(reinterpret_cast<const char*>(builder.GetBufferPointer()),
                     builder.GetSize());
}

}  // namespace libtextclassifier3