src/lib/symbol-table-ops.cc

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright 2005-2010 Google, Inc.
// Author: sorenj@google.com (Jeffrey Sorensen)

#include <fst/symbol-table-ops.h>

namespace fst {

SymbolTable *MergeSymbolTable(const SymbolTable &left, const SymbolTable &right,
                              bool *right_relabel_output) {
  // MergeSymbolTable detects several special cases.  It will return a reference
  // copied version of SymbolTable of left or right if either symbol table is
  // a superset of the other.
  SymbolTable *merged = new SymbolTable("merge_" + left.Name() + "_" +
                                        right.Name());
  // copy everything from the left symbol table
  bool left_has_all = true, right_has_all = true, relabel = false;
  SymbolTableIterator liter(left);
  for (; !liter.Done(); liter.Next()) {
    merged->AddSymbol(liter.Symbol(), liter.Value());
    if (right_has_all) {
      int64 key = right.Find(liter.Symbol());
      if (key == -1) {
        right_has_all = false;
      } else if (!relabel && key != liter.Value()) {
        relabel = true;
      }
    }
  }
  if (right_has_all) {
    delete merged;
    if (right_relabel_output != NULL) {
      *right_relabel_output = relabel;
    }
    return right.Copy();
  }
  // add all symbols we can from right symbol table
  vector<string> conflicts;
  SymbolTableIterator riter(right);
  for (; !riter.Done(); riter.Next()) {
    int64 key = merged->Find(riter.Symbol());
    if (key != -1) {
      // Symbol already exists, maybe with different value
      if (key != riter.Value()) {
        relabel = true;
      }
      continue;
    }
    // Symbol doesn't exist from left
    left_has_all = false;
    if (!merged->Find(riter.Value()).empty()) {
      // we can't add this where we want to, add it later, in order
      conflicts.push_back(riter.Symbol());
      continue;
    }
    // there is a hole and we can add this symbol with its id
    merged->AddSymbol(riter.Symbol(), riter.Value());
  }
  if (right_relabel_output != NULL) {
    *right_relabel_output = relabel;
  }
  if (left_has_all) {
    delete merged;
    return left.Copy();
  }
  // Add all symbols that conflicted, in order
  for (int i= 0; i < conflicts.size(); ++i) {
    merged->AddSymbol(conflicts[i]);
  }
  return merged;
}

SymbolTable *CompactSymbolTable(const SymbolTable &syms) {
  map<int, string> sorted;
  SymbolTableIterator stiter(syms);
  for (; !stiter.Done(); stiter.Next()) {
    sorted[stiter.Value()] = stiter.Symbol();
  }
  SymbolTable *compact = new SymbolTable(syms.Name() + "_compact");
  uint64 newkey = 0;
  for (map<int, string>::const_iterator si = sorted.begin();
       si != sorted.end(); ++si) {
    compact->AddSymbol(si->second, newkey++);
  }
  return compact;
}

SymbolTable *FstReadSymbols(const string &filename, bool input_symbols) {
  ifstream in(filename.c_str(), ifstream::in | ifstream::binary);
  if (!in) {
    LOG(ERROR) << "FstReadSymbols: Can't open file " << filename;
    return NULL;
  }
  FstHeader hdr;
  if (!hdr.Read(in, filename)) {
    LOG(ERROR) << "FstReadSymbols: Couldn't read header from " << filename;
    return NULL;
  }
  if (hdr.GetFlags() & FstHeader::HAS_ISYMBOLS) {
    SymbolTable *isymbols = SymbolTable::Read(in, filename);
    if (isymbols == NULL) {
      LOG(ERROR) << "FstReadSymbols: Could not read input symbols from "
                 << filename;
      return NULL;
    }
    if (input_symbols) {
      return isymbols;
    }
    delete isymbols;
  }
  if (hdr.GetFlags() & FstHeader::HAS_OSYMBOLS) {
    SymbolTable *osymbols = SymbolTable::Read(in, filename);
    if (osymbols == NULL) {
      LOG(ERROR) << "FstReadSymbols: Could not read output symbols from "
                 << filename;
      return NULL;
    }
    if (!input_symbols) {
      return osymbols;
    }
    delete osymbols;
  }
  LOG(ERROR) << "FstReadSymbols: The file " << filename
             << " doesn't contain the requested symbols";
  return NULL;
}

}  // namespace fst