common/fml-parser.cc

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "common/fml-parser.h"

#include <ctype.h>
#include <string>

#include "util/base/logging.h"
#include "util/strings/numbers.h"

namespace libtextclassifier {
namespace nlp_core {

namespace {
inline bool IsValidCharAtStartOfIdentifier(char c) {
  return isalpha(c) || (c == '_') || (c == '/');
}

// Returns true iff character c can appear inside an identifier.
inline bool IsValidCharInsideIdentifier(char c) {
  return isalnum(c) || (c == '_') || (c == '-') || (c == '/');
}

// Returns true iff character c can appear at the beginning of a number.
inline bool IsValidCharAtStartOfNumber(char c) {
  return isdigit(c) || (c == '+') || (c == '-');
}

// Returns true iff character c can appear inside a number.
inline bool IsValidCharInsideNumber(char c) {
  return isdigit(c) || (c == '.');
}
}  // namespace

bool FMLParser::Initialize(const std::string &source) {
  // Initialize parser state.
  source_ = source;
  current_ = source_.begin();
  item_start_ = line_start_ = current_;
  line_number_ = item_line_number_ = 1;

  // Read first input item.
  return NextItem();
}

void FMLParser::ReportError(const std::string &error_message) {
  const int position = item_start_ - line_start_ + 1;
  const std::string line(line_start_, current_);

  TC_LOG(ERROR) << "Error in feature model, line " << item_line_number_
                << ", position " << position << ": " << error_message
                << "\n    " << line << " <--HERE";
}

void FMLParser::Next() {
  // Move to the next input character. If we are at a line break update line
  // number and line start position.
  if (CurrentChar() == '\n') {
    ++line_number_;
    ++current_;
    line_start_ = current_;
  } else {
    ++current_;
  }
}

bool FMLParser::NextItem() {
  // Skip white space and comments.
  while (!eos()) {
    if (CurrentChar() == '#') {
      // Skip comment.
      while (!eos() && CurrentChar() != '\n') Next();
    } else if (isspace(CurrentChar())) {
      // Skip whitespace.
      while (!eos() && isspace(CurrentChar())) Next();
    } else {
      break;
    }
  }

  // Record start position for next item.
  item_start_ = current_;
  item_line_number_ = line_number_;

  // Check for end of input.
  if (eos()) {
    item_type_ = END;
    return true;
  }

  // Parse number.
  if (IsValidCharAtStartOfNumber(CurrentChar())) {
    std::string::iterator start = current_;
    Next();
    while (!eos() && IsValidCharInsideNumber(CurrentChar())) Next();
    item_text_.assign(start, current_);
    item_type_ = NUMBER;
    return true;
  }

  // Parse std::string.
  if (CurrentChar() == '"') {
    Next();
    std::string::iterator start = current_;
    while (CurrentChar() != '"') {
      if (eos()) {
        ReportError("Unterminated string");
        return false;
      }
      Next();
    }
    item_text_.assign(start, current_);
    item_type_ = STRING;
    Next();
    return true;
  }

  // Parse identifier name.
  if (IsValidCharAtStartOfIdentifier(CurrentChar())) {
    std::string::iterator start = current_;
    while (!eos() && IsValidCharInsideIdentifier(CurrentChar())) {
      Next();
    }
    item_text_.assign(start, current_);
    item_type_ = NAME;
    return true;
  }

  // Single character item.
  item_type_ = CurrentChar();
  Next();
  return true;
}

bool FMLParser::Parse(const std::string &source,
                      FeatureExtractorDescriptor *result) {
  // Initialize parser.
  if (!Initialize(source)) {
    return false;
  }

  while (item_type_ != END) {
    // Current item should be a feature name.
    if (item_type_ != NAME) {
      ReportError("Feature type name expected");
      return false;
    }
    std::string name = item_text_;
    if (!NextItem()) {
      return false;
    }

    // Parse feature.
    FeatureFunctionDescriptor *descriptor = result->add_feature();
    descriptor->set_type(name);
    if (!ParseFeature(descriptor)) {
      return false;
    }
  }

  return true;
}

bool FMLParser::ParseFeature(FeatureFunctionDescriptor *result) {
  // Parse argument and parameters.
  if (item_type_ == '(') {
    if (!NextItem()) return false;
    if (!ParseParameter(result)) return false;
    while (item_type_ == ',') {
      if (!NextItem()) return false;
      if (!ParseParameter(result)) return false;
    }

    if (item_type_ != ')') {
      ReportError(") expected");
      return false;
    }
    if (!NextItem()) return false;
  }

  // Parse feature name.
  if (item_type_ == ':') {
    if (!NextItem()) return false;
    if (item_type_ != NAME && item_type_ != STRING) {
      ReportError("Feature name expected");
      return false;
    }
    std::string name = item_text_;
    if (!NextItem()) return false;

    // Set feature name.
    result->set_name(name);
  }

  // Parse sub-features.
  if (item_type_ == '.') {
    // Parse dotted sub-feature.
    if (!NextItem()) return false;
    if (item_type_ != NAME) {
      ReportError("Feature type name expected");
      return false;
    }
    std::string type = item_text_;
    if (!NextItem()) return false;

    // Parse sub-feature.
    FeatureFunctionDescriptor *subfeature = result->add_feature();
    subfeature->set_type(type);
    if (!ParseFeature(subfeature)) return false;
  } else if (item_type_ == '{') {
    // Parse sub-feature block.
    if (!NextItem()) return false;
    while (item_type_ != '}') {
      if (item_type_ != NAME) {
        ReportError("Feature type name expected");
        return false;
      }
      std::string type = item_text_;
      if (!NextItem()) return false;

      // Parse sub-feature.
      FeatureFunctionDescriptor *subfeature = result->add_feature();
      subfeature->set_type(type);
      if (!ParseFeature(subfeature)) return false;
    }
    if (!NextItem()) return false;
  }
  return true;
}

bool FMLParser::ParseParameter(FeatureFunctionDescriptor *result) {
  if (item_type_ == NUMBER) {
    int32 argument;
    if (!ParseInt32(item_text_.c_str(), &argument)) {
      ReportError("Unable to parse number");
      return false;
    }
    if (!NextItem()) return false;

    // Set default argument for feature.
    result->set_argument(argument);
  } else if (item_type_ == NAME) {
    std::string name = item_text_;
    if (!NextItem()) return false;
    if (item_type_ != '=') {
      ReportError("= expected");
      return false;
    }
    if (!NextItem()) return false;
    if (item_type_ >= END) {
      ReportError("Parameter value expected");
      return false;
    }
    std::string value = item_text_;
    if (!NextItem()) return false;

    // Add parameter to feature.
    Parameter *parameter;
    parameter = result->add_parameter();
    parameter->set_name(name);
    parameter->set_value(value);
  } else {
    ReportError("Syntax error in parameter list");
    return false;
  }
  return true;
}

void ToFMLFunction(const FeatureFunctionDescriptor &function,
                   std::string *output) {
  output->append(function.type());
  if (function.argument() != 0 || function.parameter_size() > 0) {
    output->append("(");
    bool first = true;
    if (function.argument() != 0) {
      output->append(IntToString(function.argument()));
      first = false;
    }
    for (int i = 0; i < function.parameter_size(); ++i) {
      if (!first) output->append(",");
      output->append(function.parameter(i).name());
      output->append("=");
      output->append("\"");
      output->append(function.parameter(i).value());
      output->append("\"");
      first = false;
    }
    output->append(")");
  }
}

void ToFML(const FeatureFunctionDescriptor &function, std::string *output) {
  ToFMLFunction(function, output);
  if (function.feature_size() == 1) {
    output->append(".");
    ToFML(function.feature(0), output);
  } else if (function.feature_size() > 1) {
    output->append(" { ");
    for (int i = 0; i < function.feature_size(); ++i) {
      if (i > 0) output->append(" ");
      ToFML(function.feature(i), output);
    }
    output->append(" } ");
  }
}

void ToFML(const FeatureExtractorDescriptor &extractor, std::string *output) {
  for (int i = 0; i < extractor.feature_size(); ++i) {
    ToFML(extractor.feature(i), output);
    output->append("\n");
  }
}

}  // namespace nlp_core
}  // namespace libtextclassifier