Merge pull request apertium#22 from Techievena/issue2

Implementation of weights in lttoolbox

lttoolbox/alphabet.cc

@@ -174,11 +174,11 @@ Alphabet::read(FILE *input)
  int first = Compression::multibyte_read(input);
  int second = Compression::multibyte_read(input);
  pair<int, int> tmp(first - bias, second - bias);
- int spair_size = a_new.spair.size();
+ int spair_size = a_new.spair.size();
  a_new.spair[tmp] = spair_size;
  a_new.spairinv.push_back(tmp);
  }
- 
+
  *this = a_new;
 }
 
@@ -303,7 +303,7 @@ Alphabet::createLoopbackSymbols(set<int> &symbols, Alphabet &basis, Side s, bool
  it++)
  {
  // Only include tags that were actually seen on the correct side
- if(tags.find(it->second) != tags.end()) 
+ if(tags.find(it->second) != tags.end())
  {
  includeSymbol(it->first);
  symbols.insert(operator()(operator()(it->first),

lttoolbox/alphabet.h

@@ -38,7 +38,7 @@ class Alphabet
  * Symbol-identifier relationship. Only contains <tags>.
  * @see slexicinv
  */
- map<wstring, int, Ltstr> slexic; 
+ map<wstring, int, Ltstr> slexic;
 
  /**
  * Identifier-symbol relationship. Only contains <tags>.

lttoolbox/att_compiler.cc

@@ -24,7 +24,8 @@
 using namespace std;
 
 AttCompiler::AttCompiler() :
-starting_state(0)
+starting_state(0),
+default_weight(0.0000)
 {
 }
 
@@ -33,7 +34,7 @@ AttCompiler::~AttCompiler()
 }
 
 void 
-AttCompiler::clear() 
+AttCompiler::clear()
 {
  for (map<int, AttNode*>::const_iterator it = graph.begin(); it != graph.end();
  ++it) 
@@ -49,14 +50,14 @@ AttCompiler::clear()
  * @todo Are there other special symbols? If so, add them, and maybe use a map
  * for conversion?
  */
-void 
-AttCompiler::convert_hfst(wstring& symbol) 
+void
+AttCompiler::convert_hfst(wstring& symbol)
 {
- if (symbol == L"@0@" || symbol == L"ε") 
+ if (symbol == L"@0@" || symbol == L"ε")
  {
  symbol = L"";
  } 
- else if (symbol == L"@_SPACE_@") 
+ else if (symbol == L"@_SPACE_@")
  {
  symbol = L" ";
  }
@@ -75,7 +76,7 @@ AttCompiler::is_word_punct(wchar_t symbol)
  return true;
  }
 
- return false; 
+ return false;
 }
 
 /**
@@ -88,7 +89,7 @@ AttCompiler::is_word_punct(wchar_t symbol)
  * only) character otherwise.
  */
 int 
-AttCompiler::symbol_code(const wstring& symbol) 
+AttCompiler::symbol_code(const wstring& symbol)
 {
  if (symbol.length() > 1) {
  alphabet.includeSymbol(symbol);
@@ -111,8 +112,8 @@ AttCompiler::symbol_code(const wstring& symbol)
  }
 }
 
-void 
-AttCompiler::parse(string const &file_name, wstring const &dir) 
+void
+AttCompiler::parse(string const &file_name, wstring const &dir)
 {
  clear();
 
@@ -121,11 +122,12 @@ AttCompiler::parse(string const &file_name, wstring const &dir)
  wstring line;
  bool first_line = true; // First line -- see below
  bool seen_input_symbol = false;
- while (getline(infile, line)) 
+ while (getline(infile, line))
  {
  tokens.clear();
  int from, to;
  wstring upper, lower;
+ double weight;
 
  if (line.length() == 0 && first_line) 
  {
@@ -139,30 +141,38 @@ AttCompiler::parse(string const &file_name, wstring const &dir)
  }
 
  /* Empty line. */
- if (line.length() == 0) 
+ if (line.length() == 0)
  {
  continue;
  }
  split(line, L'\t', tokens);
 
- from = convert(tokens[0]);
+ from = static_cast<int>(convert(tokens[0]));
 
  AttNode* source = get_node(from);
  /* First line: the initial state is of both types. */
- if (first_line) 
+ if (first_line)
  {
  starting_state = from;
  first_line = false;
  }
 
  /* Final state. */
- if (tokens.size() <= 2) 
+ if (tokens.size() <= 2)
  {
- finals.insert(from);
+ if (tokens.size() > 1)
+ {
+ weight = static_cast<double>(convert(tokens[1]));
+ }
+ else
+ {
+ weight = default_weight;
+ }
+ finals.insert(pair <int, double>(from, weight));
  } 
- else 
+ else
  {
- to = convert(tokens[1]);
+ to = static_cast<int>(convert(tokens[1]));
  if(dir == L"RL")
  {
  upper = tokens[3];
@@ -177,17 +187,24 @@ AttCompiler::parse(string const &file_name, wstring const &dir)
  convert_hfst(lower);
  if(upper != L"")
  {
- seen_input_symbol = true; 
+ seen_input_symbol = true;
  }
- /* skip lines that have an empty left side and output 
+ /* skip lines that have an empty left side and output
  if we haven't seen an input symbol */
- if(upper == L"" && lower != L"" && !seen_input_symbol) 
+ if(upper == L"" && lower != L"" && !seen_input_symbol)
  {
  continue;
  }
  int tag = alphabet(symbol_code(upper), symbol_code(lower));
- /* We don't read the weights, even if they are defined. */
- source->transductions.push_back(Transduction(to, upper, lower, tag));
+ if(tokens.size() > 4)
+ {
+ weight = static_cast<double>(convert(tokens[4]));
+ }
+ else
+ {
+ weight = default_weight;
+ }
+ source->transductions.push_back(Transduction(to, upper, lower, tag, weight));
 
  get_node(to);
  }
@@ -202,7 +219,7 @@ AttCompiler::parse(string const &file_name, wstring const &dir)
 
 /** Extracts the sub-transducer made of states of type @p type. */
 Transducer 
-AttCompiler::extract_transducer(TransducerType type) 
+AttCompiler::extract_transducer(TransducerType type)
 {
  Transducer transducer;
  /* Correlation between the graph's state ids and those in the transducer. */
@@ -214,11 +231,11 @@ AttCompiler::extract_transducer(TransducerType type)
 
  /* The final states. */
  bool noFinals = true;
- for (set<int>::const_iterator f = finals.begin(); f != finals.end(); ++f) 
+ for (map<int, double>::const_iterator f = finals.begin(); f != finals.end(); ++f)
  {
- if (corr.find(*f) != corr.end()) 
+ if (corr.find(f->first) != corr.end())
  {
- transducer.setFinal(corr[*f]);
+ transducer.setFinal(corr[f->first], f->second);
  noFinals = false;
  }
  }
@@ -229,7 +246,7 @@ AttCompiler::extract_transducer(TransducerType type)
  wcerr << L"No final states (" << type << ")" << endl;
  wcerr << L" were:" << endl;
  wcerr << L"\t" ;
- for (set<int>::const_iterator f = finals.begin(); f != finals.end(); ++f) 
+ for (set<int>::const_iterator f = finals.begin(); f != finals.end(); ++f)
  {
  wcerr << *f << L" ";
  }
@@ -245,14 +262,14 @@ AttCompiler::extract_transducer(TransducerType type)
  */
 void 
 AttCompiler::_extract_transducer(TransducerType type, int from,
- Transducer& transducer, map<int, int>& corr,
- set<int>& visited) 
+  Transducer& transducer, map<int, int>& corr,
+  set<int>& visited)
 {
- if (visited.find(from) != visited.end()) 
+ if (visited.find(from) != visited.end())
  {
  return;
  } 
- else 
+ else
  {
  visited.insert(from);
  }
@@ -266,30 +283,30 @@ AttCompiler::_extract_transducer(TransducerType type, int from,
  for (vector<Transduction>::const_iterator it = source->transductions.begin();
  it != source->transductions.end(); ++it) 
  {
- if ((it->type & type) != type) 
+ if ((it->type & type) != type)
  {
  continue; // Not the right type
  }
  /* Is the target state new? */
  bool new_to = corr.find(it->to) == corr.end();
 
- if (new_from) 
+ if (new_from)
  {
  corr[from] = transducer.size() + (new_to ? 1 : 0);
  }
  from_t = corr[from];
 
  /* Now with the target state: */
- if (!new_to) 
+ if (!new_to)
  {
  /* We already know it, possibly by a different name: link them! */
  to_t = corr[it->to];
- transducer.linkStates(from_t, to_t, it->tag);
- } 
- else 
+ transducer.linkStates(from_t, to_t, it->tag, it->weight);
+ }
+ else
  {
  /* We haven't seen it yet: add a new state! */
- to_t = transducer.insertNewSingleTransduction(it->tag, from_t);
+ to_t = transducer.insertNewSingleTransduction(it->tag, from_t, it->weight);
  corr[it->to] = to_t;
  }
  _extract_transducer(type, it->to, transducer, corr, visited);
@@ -310,31 +327,31 @@ AttCompiler::_extract_transducer(TransducerType type, int from,
  * @param visited the ids of states visited by this path.
  * @param path are we in a path?
  */
-void 
+void
 AttCompiler::classify(int from, map<int, TransducerType>& visited, bool path,
- TransducerType type) 
+  TransducerType type)
 {
  AttNode* source = get_node(from);
- if (visited.find(from) != visited.end()) 
+ if (visited.find(from) != visited.end())
  {
- if (path && ( (visited[from] & type) == type) ) 
+ if (path && ( (visited[from] & type) == type) )
  {
  return;
  }
  }
 
- if (path) 
+ if (path)
  {
  visited[from] |= type;
  }
 
  for (vector<Transduction>::iterator it = source->transductions.begin();
- it != source->transductions.end(); ++it) 
+ it != source->transductions.end(); ++it)
  {
  bool next_path = path;
  int next_type = type;
  bool first_transition = !path && it->upper != L"";
- if (first_transition) 
+ if (first_transition)
  {
  /* First transition: we now know the type of the path! */
  bool upper_word = (it->upper.length() == 1 &&
@@ -344,8 +361,8 @@ AttCompiler::classify(int from, map<int, TransducerType>& visited, bool path,
  if (upper_word) next_type |= WORD;
  if (upper_punct) next_type |= PUNCT;
  next_path = true;
- } 
- else 
+ }
+ else
  {
  /* Otherwise (not yet, already): target's type is the same as ours. */
  next_type = type;
@@ -357,7 +374,7 @@ AttCompiler::classify(int from, map<int, TransducerType>& visited, bool path,
 
 /** Writes the transducer to @p file_name in lt binary format. */
 void 
-AttCompiler::write(FILE *output) 
+AttCompiler::write(FILE *output)
 {
 // FILE* output = fopen(file_name, "w");
  Transducer punct_fst = extract_transducer(PUNCT);
@@ -377,13 +394,13 @@ AttCompiler::write(FILE *output)
  }
  Compression::wstring_write(L"main@standard", output);
  Transducer word_fst = extract_transducer(WORD);
- word_fst.write(output);
+ word_fst.write(output, 0, true);
  wcout << L"main@standard" << " " << word_fst.size();
  wcout << " " << word_fst.numberOfTransitions() << endl;
  Compression::wstring_write(L"final@inconditional", output);
- if(punct_fst.numberOfTransitions() != 0) 
+ if(punct_fst.numberOfTransitions() != 0)
  {
- punct_fst.write(output);
+ punct_fst.write(output, 0, true);
  wcout << L"final@inconditional" << " " << punct_fst.size();
  wcout << " " << punct_fst.numberOfTransitions() << endl;
  }