src/objects/js-collator.cc

// Copyright 2018 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef V8_INTL_SUPPORT
#error Internationalization is expected to be enabled.
#endif  // V8_INTL_SUPPORT

#include "src/objects/js-collator.h"

#include "src/isolate.h"
#include "src/objects-inl.h"
#include "src/objects/js-collator-inl.h"
#include "unicode/coll.h"
#include "unicode/locid.h"
#include "unicode/strenum.h"
#include "unicode/ucol.h"
#include "unicode/uloc.h"

namespace v8 {
namespace internal {

namespace {

// TODO(gsathya): Consider internalizing the value strings.
void CreateDataPropertyForOptions(Isolate* isolate, Handle<JSObject> options,
                                  Handle<String> key, const char* value) {
  CHECK_NOT_NULL(value);
  Handle<String> value_str =
      isolate->factory()->NewStringFromAsciiChecked(value);

  // This is a brand new JSObject that shouldn't already have the same
  // key so this shouldn't fail.
  CHECK(JSReceiver::CreateDataProperty(isolate, options, key, value_str,
                                       kDontThrow)
            .FromJust());
}

void CreateDataPropertyForOptions(Isolate* isolate, Handle<JSObject> options,
                                  Handle<String> key, bool value) {
  Handle<Object> value_obj = isolate->factory()->ToBoolean(value);

  // This is a brand new JSObject that shouldn't already have the same
  // key so this shouldn't fail.
  CHECK(JSReceiver::CreateDataProperty(isolate, options, key, value_obj,
                                       kDontThrow)
            .FromJust());
}

}  // anonymous namespace

// static
Handle<JSObject> JSCollator::ResolvedOptions(Isolate* isolate,
                                             Handle<JSCollator> collator) {
  Handle<JSObject> options =
      isolate->factory()->NewJSObject(isolate->object_function());

  JSCollator::Usage usage = collator->usage();
  CreateDataPropertyForOptions(isolate, options,
                               isolate->factory()->usage_string(),
                               JSCollator::UsageToString(usage));

  icu::Collator* icu_collator = collator->icu_collator()->raw();
  CHECK_NOT_NULL(icu_collator);

  UErrorCode status = U_ZERO_ERROR;
  bool numeric =
      icu_collator->getAttribute(UCOL_NUMERIC_COLLATION, status) == UCOL_ON;
  CHECK(U_SUCCESS(status));
  CreateDataPropertyForOptions(isolate, options,
                               isolate->factory()->numeric_string(), numeric);

  const char* case_first = nullptr;
  status = U_ZERO_ERROR;
  switch (icu_collator->getAttribute(UCOL_CASE_FIRST, status)) {
    case UCOL_LOWER_FIRST:
      case_first = "lower";
      break;
    case UCOL_UPPER_FIRST:
      case_first = "upper";
      break;
    default:
      case_first = "false";
  }
  CHECK(U_SUCCESS(status));
  CreateDataPropertyForOptions(
      isolate, options, isolate->factory()->caseFirst_string(), case_first);

  const char* sensitivity = nullptr;
  status = U_ZERO_ERROR;
  switch (icu_collator->getAttribute(UCOL_STRENGTH, status)) {
    case UCOL_PRIMARY: {
      CHECK(U_SUCCESS(status));
      status = U_ZERO_ERROR;
      // case level: true + s1 -> case, s1 -> base.
      if (UCOL_ON == icu_collator->getAttribute(UCOL_CASE_LEVEL, status)) {
        sensitivity = "case";
      } else {
        sensitivity = "base";
      }
      CHECK(U_SUCCESS(status));
      break;
    }
    case UCOL_SECONDARY:
      sensitivity = "accent";
      break;
    case UCOL_TERTIARY:
      sensitivity = "variant";
      break;
    case UCOL_QUATERNARY:
      // We shouldn't get quaternary and identical from ICU, but if we do
      // put them into variant.
      sensitivity = "variant";
      break;
    default:
      sensitivity = "variant";
  }
  CHECK(U_SUCCESS(status));
  CreateDataPropertyForOptions(
      isolate, options, isolate->factory()->sensitivity_string(), sensitivity);

  status = U_ZERO_ERROR;
  bool ignore_punctuation = icu_collator->getAttribute(UCOL_ALTERNATE_HANDLING,
                                                       status) == UCOL_SHIFTED;
  CHECK(U_SUCCESS(status));
  CreateDataPropertyForOptions(isolate, options,
                               isolate->factory()->ignorePunctuation_string(),
                               ignore_punctuation);

  status = U_ZERO_ERROR;
  const char* collation;
  std::unique_ptr<icu::StringEnumeration> collation_values(
      icu_collator->getKeywordValues("co", status));
  // Collation wasn't provided as a keyword to icu, use default.
  if (status == U_ILLEGAL_ARGUMENT_ERROR) {
    CreateDataPropertyForOptions(
        isolate, options, isolate->factory()->collation_string(), "default");
  } else {
    CHECK(U_SUCCESS(status));
    CHECK_NOT_NULL(collation_values.get());

    int32_t length;
    status = U_ZERO_ERROR;
    collation = collation_values->next(&length, status);
    CHECK(U_SUCCESS(status));

    // There has to be at least one value.
    CHECK_NOT_NULL(collation);
    CreateDataPropertyForOptions(
        isolate, options, isolate->factory()->collation_string(), collation);

    status = U_ZERO_ERROR;
    collation_values->reset(status);
    CHECK(U_SUCCESS(status));
  }

  status = U_ZERO_ERROR;
  icu::Locale icu_locale = icu_collator->getLocale(ULOC_VALID_LOCALE, status);
  CHECK(U_SUCCESS(status));

  char result[ULOC_FULLNAME_CAPACITY];
  status = U_ZERO_ERROR;
  uloc_toLanguageTag(icu_locale.getName(), result, ULOC_FULLNAME_CAPACITY,
                     FALSE, &status);
  CHECK(U_SUCCESS(status));

  CreateDataPropertyForOptions(isolate, options,
                               isolate->factory()->locale_string(), result);

  return options;
}

namespace {

std::map<std::string, std::string> LookupUnicodeExtensions(
    const icu::Locale& icu_locale, const std::set<std::string>& relevant_keys) {
  std::map<std::string, std::string> extensions;

  UErrorCode status = U_ZERO_ERROR;
  std::unique_ptr<icu::StringEnumeration> keywords(
      icu_locale.createKeywords(status));
  if (U_FAILURE(status)) return extensions;

  if (!keywords) return extensions;
  char value[ULOC_FULLNAME_CAPACITY];

  int32_t length;
  status = U_ZERO_ERROR;
  for (const char* keyword = keywords->next(&length, status);
       keyword != nullptr; keyword = keywords->next(&length, status)) {
    // Ignore failures in ICU and skip to the next keyword.
    //
    // This is fine.™
    if (U_FAILURE(status)) {
      status = U_ZERO_ERROR;
      continue;
    }

    icu_locale.getKeywordValue(keyword, value, ULOC_FULLNAME_CAPACITY, status);

    // Ignore failures in ICU and skip to the next keyword.
    //
    // This is fine.™
    if (U_FAILURE(status)) {
      status = U_ZERO_ERROR;
      continue;
    }

    const char* bcp47_key = uloc_toUnicodeLocaleKey(keyword);

    // Ignore keywords that we don't recognize - spec allows that.
    if (bcp47_key && (relevant_keys.find(bcp47_key) != relevant_keys.end())) {
      const char* bcp47_value = uloc_toUnicodeLocaleType(bcp47_key, value);
      extensions.insert(
          std::pair<std::string, std::string>(bcp47_key, bcp47_value));
    }
  }

  return extensions;
}

void SetCaseFirstOption(icu::Collator* icu_collator, const char* value) {
  CHECK_NOT_NULL(icu_collator);
  CHECK_NOT_NULL(value);
  UErrorCode status = U_ZERO_ERROR;
  if (strcmp(value, "upper") == 0) {
    icu_collator->setAttribute(UCOL_CASE_FIRST, UCOL_UPPER_FIRST, status);
  } else if (strcmp(value, "lower") == 0) {
    icu_collator->setAttribute(UCOL_CASE_FIRST, UCOL_LOWER_FIRST, status);
  } else {
    icu_collator->setAttribute(UCOL_CASE_FIRST, UCOL_OFF, status);
  }
  CHECK(U_SUCCESS(status));
}

}  // anonymous namespace

// static
MaybeHandle<JSCollator> JSCollator::InitializeCollator(
    Isolate* isolate, Handle<JSCollator> collator, Handle<Object> locales,
    Handle<Object> options_obj) {
  // 1. Let requestedLocales be ? CanonicalizeLocaleList(locales).
  Handle<JSObject> requested_locales;
  ASSIGN_RETURN_ON_EXCEPTION(isolate, requested_locales,
                             Intl::CanonicalizeLocaleListJS(isolate, locales),
                             JSCollator);

  // 2. If options is undefined, then
  if (options_obj->IsUndefined(isolate)) {
    // 2. a. Let options be ObjectCreate(null).
    options_obj = isolate->factory()->NewJSObjectWithNullProto();
  } else {
    // 3. Else
    // 3. a. Let options be ? ToObject(options).
    ASSIGN_RETURN_ON_EXCEPTION(
        isolate, options_obj,
        Object::ToObject(isolate, options_obj, "Intl.Collator"), JSCollator);
  }

  // At this point, options_obj can either be a JSObject or a JSProxy only.
  Handle<JSReceiver> options = Handle<JSReceiver>::cast(options_obj);

  // 4. Let usage be ? GetOption(options, "usage", "string", « "sort",
  // "search" », "sort").
  std::vector<const char*> values = {"sort", "search"};
  std::unique_ptr<char[]> usage_str = nullptr;
  JSCollator::Usage usage = JSCollator::Usage::SORT;
  Maybe<bool> found_usage = Intl::GetStringOption(
      isolate, options, "usage", values, "Intl.Collator", &usage_str);
  MAYBE_RETURN(found_usage, MaybeHandle<JSCollator>());

  if (found_usage.FromJust()) {
    DCHECK_NOT_NULL(usage_str.get());
    if (strcmp(usage_str.get(), "search") == 0) {
      usage = JSCollator::Usage::SEARCH;
    }
  }

  // 5. Set collator.[[Usage]] to usage.
  collator->set_usage(usage);

  // 6. If usage is "sort", then
  //    a. Let localeData be %Collator%.[[SortLocaleData]].
  // 7. Else,
  //    a. Let localeData be %Collator%.[[SearchLocaleData]].
  //
  // The above two spec operations aren't required, the Intl spec is
  // crazy. See https://github.com/tc39/ecma402/issues/256

  // TODO(gsathya): This is currently done as part of the
  // Intl::ResolveLocale call below. Fix this once resolveLocale is
  // changed to not do the lookup.
  //
  // 9. Let matcher be ? GetOption(options, "localeMatcher", "string",
  // « "lookup", "best fit" », "best fit").
  // 10. Set opt.[[localeMatcher]] to matcher.

  // 11. Let numeric be ? GetOption(options, "numeric", "boolean",
  // undefined, undefined).
  // 12. If numeric is not undefined, then
  //    a. Let numeric be ! ToString(numeric).
  //
  // Note: We omit the ToString(numeric) operation as it's not
  // observable. Intl::GetBoolOption returns a Boolean and
  // ToString(Boolean) is not side-effecting.
  //
  // 13. Set opt.[[kn]] to numeric.
  bool numeric;
  Maybe<bool> found_numeric = Intl::GetBoolOption(isolate, options, "numeric",
                                                  "Intl.Collator", &numeric);
  MAYBE_RETURN(found_numeric, MaybeHandle<JSCollator>());

  // 14. Let caseFirst be ? GetOption(options, "caseFirst", "string",
  //     « "upper", "lower", "false" », undefined).
  // 15. Set opt.[[kf]] to caseFirst.
  values = {"upper", "lower", "false"};
  std::unique_ptr<char[]> case_first_str = nullptr;
  Maybe<bool> found_case_first = Intl::GetStringOption(
      isolate, options, "caseFirst", values, "Intl.Collator", &case_first_str);
  MAYBE_RETURN(found_case_first, MaybeHandle<JSCollator>());

  // The relevant unicode extensions accepted by Collator as specified here:
  // https://tc39.github.io/ecma402/#sec-intl-collator-internal-slots
  //
  // 16. Let relevantExtensionKeys be %Collator%.[[RelevantExtensionKeys]].
  std::set<std::string> relevant_extension_keys{"co", "kn", "kf"};

  // We don't pass the relevant_extension_keys to ResolveLocale here
  // as per the spec.
  //
  // In ResolveLocale, the spec makes sure we only pick and use the
  // relevant extension keys and ignore any other keys. Also, in
  // ResolveLocale, the spec makes sure that if a given key has both a
  // value in the options object and an unicode extension value, then
  // we pick the value provided in the options object.
  // For example: in the case of `new Intl.Collator('en-u-kn-true', {
  // numeric: false })` the value `false` is used for the `numeric`
  // key.
  //
  // Instead of performing all this validation in ResolveLocale, we
  // just perform it inline below. In the future when we port
  // ResolveLocale to C++, we can make all these validations generic
  // and move it ResolveLocale.
  //
  // 17. Let r be ResolveLocale(%Collator%.[[AvailableLocales]],
  // requestedLocales, opt, %Collator%.[[RelevantExtensionKeys]],
  // localeData).
  // 18. Set collator.[[Locale]] to r.[[locale]].
  Handle<JSObject> r;
  ASSIGN_RETURN_ON_EXCEPTION(
      isolate, r,
      Intl::ResolveLocale(isolate, "collator", requested_locales, options),
      JSCollator);

  Handle<String> locale_with_extension_str =
      isolate->factory()->NewStringFromStaticChars("localeWithExtension");
  Handle<Object> locale_with_extension_obj =
      JSObject::GetDataProperty(r, locale_with_extension_str);

  // The locale_with_extension has to be a string. Either a user
  // provided canonicalized string or the default locale.
  CHECK(locale_with_extension_obj->IsString());
  Handle<String> locale_with_extension =
      Handle<String>::cast(locale_with_extension_obj);

  icu::Locale icu_locale =
      Intl::CreateICULocale(isolate, locale_with_extension);
  DCHECK(!icu_locale.isBogus());

  std::map<std::string, std::string> extensions =
      LookupUnicodeExtensions(icu_locale, relevant_extension_keys);

  // 19. Let collation be r.[[co]].
  //
  // r.[[co]] is already set as part of the icu::Locale creation as
  // icu parses unicode extensions and sets the keywords.
  //
  // We need to sanitize the keywords based on certain ECMAScript rules.
  //
  // As per https://tc39.github.io/ecma402/#sec-intl-collator-internal-slots:
  // The values "standard" and "search" must not be used as elements
  // in any [[SortLocaleData]][locale].co and
  // [[SearchLocaleData]][locale].co list.
  auto co_extension_it = extensions.find("co");
  if (co_extension_it != extensions.end()) {
    const std::string& value = co_extension_it->second;
    if ((value == "search") || (value == "standard")) {
      UErrorCode status = U_ZERO_ERROR;
      icu_locale.setKeywordValue("co", NULL, status);
      CHECK(U_SUCCESS(status));
    }
  }

  // 20. If collation is null, let collation be "default".
  // 21. Set collator.[[Collation]] to collation.
  //
  // We don't store the collation value as per the above two steps
  // here. The collation value can be looked up from icu::Collator on
  // demand, as part of Intl.Collator.prototype.resolvedOptions.

  UErrorCode status = U_ZERO_ERROR;
  std::unique_ptr<icu::Collator> icu_collator(
      icu::Collator::createInstance(icu_locale, status));
  if (U_FAILURE(status) || icu_collator.get() == nullptr) {
    status = U_ZERO_ERROR;
    // Remove extensions and try again.
    icu::Locale no_extension_locale(icu_locale.getBaseName());
    icu_collator.reset(
        icu::Collator::createInstance(no_extension_locale, status));

    if (U_FAILURE(status) || icu_collator.get() == nullptr) {
      FATAL("Failed to create ICU collator, are ICU data files missing?");
    }
  }
  DCHECK(U_SUCCESS(status));
  CHECK_NOT_NULL(icu_collator.get());

  // 22. If relevantExtensionKeys contains "kn", then
  //     a. Set collator.[[Numeric]] to ! SameValue(r.[[kn]], "true").
  //
  // If the numeric value is passed in through the options object,
  // then we use it. Otherwise, we check if the numeric value is
  // passed in through the unicode extensions.
  status = U_ZERO_ERROR;
  if (found_numeric.FromJust()) {
    icu_collator->setAttribute(UCOL_NUMERIC_COLLATION,
                               numeric ? UCOL_ON : UCOL_OFF, status);
    CHECK(U_SUCCESS(status));
  } else {
    auto kn_extension_it = extensions.find("kn");
    if (kn_extension_it != extensions.end()) {
      const std::string& value = kn_extension_it->second;

      numeric = (value == "true");

      icu_collator->setAttribute(UCOL_NUMERIC_COLLATION,
                                 numeric ? UCOL_ON : UCOL_OFF, status);
      CHECK(U_SUCCESS(status));
    }
  }

  // 23. If relevantExtensionKeys contains "kf", then
  //     a. Set collator.[[CaseFirst]] to r.[[kf]].
  //
  // If the caseFirst value is passed in through the options object,
  // then we use it. Otherwise, we check if the caseFirst value is
  // passed in through the unicode extensions.
  if (found_case_first.FromJust()) {
    const char* case_first_cstr = case_first_str.get();
    SetCaseFirstOption(icu_collator.get(), case_first_cstr);
  } else {
    auto kf_extension_it = extensions.find("kf");
    if (kf_extension_it != extensions.end()) {
      const std::string& value = kf_extension_it->second;
      SetCaseFirstOption(icu_collator.get(), value.c_str());
    }
  }

  // Normalization is always on, by the spec. We are free to optimize
  // if the strings are already normalized (but we don't have a way to tell
  // that right now).
  status = U_ZERO_ERROR;
  icu_collator->setAttribute(UCOL_NORMALIZATION_MODE, UCOL_ON, status);
  CHECK(U_SUCCESS(status));

  // 24. Let sensitivity be ? GetOption(options, "sensitivity",
  // "string", « "base", "accent", "case", "variant" », undefined).
  values = {"base", "accent", "case", "variant"};
  std::unique_ptr<char[]> sensitivity_str = nullptr;
  Maybe<bool> found_sensitivity =
      Intl::GetStringOption(isolate, options, "sensitivity", values,
                            "Intl.Collator", &sensitivity_str);
  MAYBE_RETURN(found_sensitivity, MaybeHandle<JSCollator>());

  // 25. If sensitivity is undefined, then
  if (!found_sensitivity.FromJust()) {
    // 25. a. If usage is "sort", then
    if (usage == Usage::SORT) {
      // 25. a. i. Let sensitivity be "variant".
      // 26. Set collator.[[Sensitivity]] to sensitivity.
      icu_collator->setStrength(icu::Collator::TERTIARY);
    }
  } else {
    DCHECK(found_sensitivity.FromJust());
    const char* sensitivity_cstr = sensitivity_str.get();
    DCHECK_NOT_NULL(sensitivity_cstr);

    // 26. Set collator.[[Sensitivity]] to sensitivity.
    if (strcmp(sensitivity_cstr, "base") == 0) {
      icu_collator->setStrength(icu::Collator::PRIMARY);
    } else if (strcmp(sensitivity_cstr, "accent") == 0) {
      icu_collator->setStrength(icu::Collator::SECONDARY);
    } else if (strcmp(sensitivity_cstr, "case") == 0) {
      icu_collator->setStrength(icu::Collator::PRIMARY);
      status = U_ZERO_ERROR;
      icu_collator->setAttribute(UCOL_CASE_LEVEL, UCOL_ON, status);
      CHECK(U_SUCCESS(status));
    } else {
      DCHECK_EQ(0, strcmp(sensitivity_cstr, "variant"));
      icu_collator->setStrength(icu::Collator::TERTIARY);
    }
  }

  // 27.Let ignorePunctuation be ? GetOption(options,
  // "ignorePunctuation", "boolean", undefined, false).
  bool ignore_punctuation;
  Maybe<bool> found_ignore_punctuation =
      Intl::GetBoolOption(isolate, options, "ignorePunctuation",
                          "Intl.Collator", &ignore_punctuation);
  MAYBE_RETURN(found_ignore_punctuation, MaybeHandle<JSCollator>());

  // 28. Set collator.[[IgnorePunctuation]] to ignorePunctuation.
  if (found_ignore_punctuation.FromJust() && ignore_punctuation) {
    status = U_ZERO_ERROR;
    icu_collator->setAttribute(UCOL_ALTERNATE_HANDLING, UCOL_SHIFTED, status);
    CHECK(U_SUCCESS(status));
  }

  Handle<Managed<icu::Collator>> managed_collator =
      Managed<icu::Collator>::FromUniquePtr(isolate, 0,
                                            std::move(icu_collator));
  collator->set_icu_collator(*managed_collator);

  // 29. Return collator.
  return collator;
}

// static
const char* JSCollator::UsageToString(Usage usage) {
  switch (usage) {
    case Usage::SORT:
      return "sort";
    case Usage::SEARCH:
      return "search";
    case Usage::COUNT:
      UNREACHABLE();
  }
}

}  // namespace internal
}  // namespace v8