// file: $isip/class/pr/HiddenMarkovModel/hmm_07.cc
// version: $Id: hmm_07.cc 9449 2004-04-13 19:27:00Z gao $
//

// isip include files
//
#include "HiddenMarkovModel.h"
#include <CommandLine.h>
#include <LanguageModel.h>

// method: networkDecoder
//
// arguments:
//  Sdb& sdb: (input) signal data base to run on
//
// return: a bool8 value indicating status
//
// this is the run method for a network decoder
//
bool8 HiddenMarkovModel::networkDecoder(Sdb& sdb_a) {
  
  // branch on the DECODING using VITERBI
  //
  if ((algorithm_d == DECODE && implementation_d == VITERBI) ||
      (algorithm_d == FORCED_ALIGNMENT && implementation_d == VITERBI)) {
    
    // declare local variables
    //
    int32 num_valid_files = 0;
    int32 current_file_num = 0;
    int32 total_num_frames = 0;

    String identifier;
    Filename input_file_name;
    
    Filename input_ID;        
    String output;
    Filename output_file_name;
    Sdb output_sdb;
    Vector<VectorFloat> data;

    TranscriptionDatabase trans_db;
    
    // load the hmm models from the model file
    //
    if (!load()) {
      return Error::handle(name(), L"run: load hmm models",
			   Error::ARG, __FILE__, __LINE__);
    }
    
    // set the search engine mode
    //
    search_engine_d.setSearchMode(HierarchicalSearch::DECODE);    

    // when the output mode is FILE
    //
    File total_output_file;      
    if (output_mode_d == FILE) {

      // open the output file for the utterance hypotheses
      //
      if (output_file_d.length() != 0) {
      
	if (verbosity_d >= Integral::BRIEF) {
	  Console::increaseIndention();
	  output.assign(L"\nopening the output file: ");
	  output.concat(output_file_d);
	  Console::put(output);
	  Console::decreaseIndention();
	}

	if (!total_output_file.open(output_file_d, File::WRITE_ONLY)) {
	  return Error::handle(name(), L"networkDecoder - error opening output file", Error::ARG, __FILE__, __LINE__);
	}
	else {
	  total_output_file.close();
	}
      }
    }

    // when the output mode is DATABASE
    //
    Sof output_db_sof;
    if (output_mode_d == DATABASE) {

      if (output_file_d.length() > 0) {

	if (verbosity_d >= Integral::BRIEF) {
	  Console::increaseIndention();
	  output.assign(L"\nopening the output file: ");
	  output.concat(output_file_d);
	  Console::put(output);
	  Console::decreaseIndention();
	}

	// open the output file for the utterance hypotheses (TEXT)
	//
	if (output_type_d == TEXT) {	

	  if (!output_db_sof.open(output_file_d, File::WRITE_ONLY)) {
	    return Error::handle(name(), L"run: opening output file",
				 Error::ARG, __FILE__, __LINE__);
	  }
	  else {
	    
	    // read all identifiers form the sdb object
	    //
	    Vector<String> identifier_keys;
	    identifier_keys.setCapacity(sdb_a.length());
	    
	    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID);
		 sdb_a.gotoNext()) {
	      identifier.assign(input_ID);
	      identifier_keys.concat(identifier);
	    }

	    // write the frame-duration to the output file
	    //
	    Float frame_dur =
	      vector_fe_d(vector_fe_d.length() - 1).getFrameDuration();
	    frame_dur.write(output_db_sof, (int32)0);
	    
	    // write the identifiers to the transcription database	    
	    //
	    trans_db.storePartial(output_db_sof, 0, identifier_keys);
	  }
	} // end of output type TEXT

	// open the output file for the utterance hypotheses (BINARY)
	//
	if (output_type_d == BINARY) {	

	  if (!output_db_sof.open(output_file_d,
				  File::WRITE_ONLY, File::BINARY)) {
	    return Error::handle(name(), L"run: opening output file",
				 Error::ARG, __FILE__, __LINE__);
	  }
	  else {
	    
	    // read all identifiers form the sdb object
	    //
	    Vector<String> identifier_keys;
	    identifier_keys.setCapacity(sdb_a.length());
	    
	    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID);
		 sdb_a.gotoNext()) {
	      identifier.assign(input_ID);
	      identifier_keys.concat(identifier);
	    }

	    // write the frame-duration to the output file
	    //
	    Float frame_dur = vector_fe_d((int32)0).getFrameDuration();
	    frame_dur.write(output_db_sof, (int32)0);

	    // write the identifiers to the transcription database
	    //
	    trans_db.storePartial(output_db_sof, 0, identifier_keys);
	  }
	} // end of output type BINARY
      }
    }    
    
    // loop through the input utterances
    //
    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID); sdb_a.gotoNext()) {

      current_file_num++;

      // if input is streaming, set input_file_name to STREAM_FILE
      //
      if (stream_d) {
	input_file_name.assign(File::STREAM_FILE);
      }
      
      // otherwise, get the path
      //
      else {
	// get audio file path
	//
	identifier.assign(input_ID);
	if (!audio_db_d.getRecord(identifier, input_file_name)) {
	  input_ID.debug(L"input_ID");
	  return Error::handle(name(), L"initialize - unable to find audio file for identifier", Error::ARG, __FILE__, __LINE__);	    
	}
      }
 
      // print utterance processing information
      //      
      if (verbosity_d >= Integral::BRIEF) {
	Console::increaseIndention();
	output.assign(L"\nprocessing file ");
	output.concat(current_file_num);
	output.concat(L" (");
	output.concat(identifier);
	output.concat(L")");	
	output.concat(L": ");
	output.concat(input_file_name);
	Console::put(output);
	Console::decreaseIndention();
      }

      // initialize the decoder
      //
      if (!search_engine_d.initializeNetworkDecoder()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
      
      // initialize the top search level with the corresponding transcription
      //  we don't need a transcription database for segmented utterances
      //
      if (transcription_db_file_d.length() > 0) {
	initTranscription(identifier, current_file_num - 1);
      }
                  
      // loop over all the front ends
      //
      for (int32 i = 0; i < vector_fe_d.length(); i++) {
	
	// process the utterance file by the front end
	//
	vector_fe_d(i).open(input_file_name);

	// retrieve the all frames of data in advance
	//
	extractFeatures(i, data);
      }
      
      // decode the utterance
      //	
      if (!verify_d) {
	search_engine_d.networkDecoder(vector_fe_d);
      }
      
      // close the front ends
      //
      for (int32 i = 0; i < vector_fe_d.length(); i++) {
	vector_fe_d(i).close();
      }

      // pick up the best hypothesis and its parameters
      //
      String hypotheses;
      float32 score = 0;      
      int32 num_frames = 0;
      DoubleLinkedList<Instance> instance_path;
      
      if (!search_engine_d.getLexHypotheses(hypotheses, alignment_level_d, score,
					 num_frames, instance_path)) {
	  
	// if no hypothesis found
	//
	hypotheses.clear();
	instance_path.clear();

	if ((!verify_d) && (verbosity_d >= Integral::BRIEF)) {
	  
	  // print the warning message
	  //
	  Console::increaseIndention();
	  Console::put(L"\nno hypothesis found");
	  Console::decreaseIndention();
	}
      }
      else {
	
	num_valid_files++;
	total_num_frames += num_frames;
      }

      // print the detailed info about the hypothesis
      //
      if (verbosity_d >= Integral::BRIEF) {
	Console::increaseIndention();
	Console::increaseIndention();
	output.assign(L"\nhyp:    ");
	output.concat(hypotheses);
	output.concat(L"\nscore:  ");
	output.concat(score);
	output.concat(L"   frames: ");
	output.concat(num_frames);
	Console::put(output);
	Console::decreaseIndention();
	Console::decreaseIndention();
      }
     
      if (output_mode_d == TRANSFORM) {

	// transform the input file and its path to the output file
	//
	sdb_a.transformName(output_file_name, input_file_name);

	// open the output file and write best hypothesis
	//
	File output_file;
	if (!output_file.open(output_file_name, File::WRITE_ONLY)) {
	  return Error::handle(name(), L"networkDecoder - error opening output file", Error::ARG, __FILE__, __LINE__);
	}
	output_file.put(hypotheses);
	      
	// close the output file
	//
	output_file.close();
      }

      if (output_mode_d == DATABASE) {

	String name_00(identifier);
	String gtype_00(identifier);	
	AnnotationGraph output_graph(name_00, gtype_00);

	// convert the best search path output format
	//
	search_engine_d.convertLexInstances(instance_path);
	

	// convert the best search path to an annotation graph
	//
	if (!createAnnotationGraph(output_graph, instance_path)) {
	  return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
	}

	// prune the annotation graph according to output levels
	//
	pruneAnnotationGraph(output_graph);
	
	// write the annotation graph to the database
	//
	trans_db.storePartial(output_db_sof, current_file_num - 1, output_graph);
      }
      
      if (output_mode_d == FILE) {
	
	// output the best hypothesis and sent ID to the output file
	//                                           or to stdout
	//
	output.assign(hypotheses);
	if ((int32)alignment_level_d < 0) {
	  output.concat(L" (");
	  output.concat(input_ID);
	  output.concat(L")\n");
	}

	if (output_file_d.length() != 0) {
	  total_output_file.open(output_file_d, File::APPEND_PLUS);
	  total_output_file.put(output);
	  total_output_file.close();
	}
	else {
	  if (verbosity_d < Integral::BRIEF) {
	    Console::increaseIndention();
	    Console::increaseIndention();
	    Console::put(output);
	    Console::decreaseIndention();
	    Console::decreaseIndention();
	  }
	}
      }

      if (output_mode_d == LIST) {
	
	// output the hypothesis to the corresponding file
	// from the output list
	//
	bool8 more_files;
	
	if (verbosity_d >= Integral::BRIEF) {
	  Console::increaseIndention();
	  output.assign(L"\nopening the output list: ");
	  output.concat(output_file_d);
	  Console::put(output);
	  Console::decreaseIndention();
	}
	
	if (current_file_num == 1) {
	  
	  // read output files list into signal database
	  //
	  Sof output_list_file;
	  if (!output_list_file.open(output_list_d)) {
	    return Error::handle(name(), L"networkDecoder - error opening output list", Error::ARG, __FILE__, __LINE__);
	  }
	  
	  output_sdb.read(output_list_file, 0);
	  output_list_file.close();
	  
	  more_files = output_sdb.gotoFirst();
	}
	else {

	  // move to the next output file
	  //
	  more_files = output_sdb.gotoNext();
	}
	
	if (!more_files) {
	  return Error::handle(name(), L"networkDecoder - insufficient output files in the output list", Error::ARG, __FILE__, __LINE__);
	}
	
	// open the next output file and write the best hypothesis
	//
	output_sdb.getName(output_file_name);
	
	File output_file;
	if (!output_file.open(output_file_name, File::WRITE_ONLY)) {
	  Console::increaseIndention();
	  output.assign(L"\ncannot open output file: ");
	  output.concat(output_file_name);
	  Console::put(output);
	  Console::decreaseIndention();
	}
	else {
	  output_file.put(hypotheses);
	  output_file.put(L"\n");
	  output_file.close();
	}
      }
    } // end of looping through the input utterances

    // clean up all memory
    //
    search_engine_d.clear();
    
    // close database file
    //
    if (output_mode_d == DATABASE) {
      output_db_sof.close();
    }

    // close the audio database (optional)
    //
    if (audio_db_file_d.length() > 0) {
      if (!audio_db_d.close()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
    }

    // close the transcription database (optional)
    //
    if (transcription_db_file_d.length() > 0) {
      if (!transcription_db_d.close()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
    }
    
    // print the number of successfully processed files
    //
    output.assign(L"\nprocessed ");
    output.concat(num_valid_files);
    output.concat(L" file(s) successfully, attempted ");
    output.concat(current_file_num);
    output.concat(L" file(s), ");
    output.concat(total_num_frames);
    output.concat(L" frame(s)\n");    
    Console::put(output);
    
  } // end of DECODE algorithm VITERBI implementation

  // branch on the SYMBOL_GRAPH_RESCORING using RESCORE_LANGUAGE
  //
  else if (algorithm_d == SYMBOL_GRAPH_RESCORING &&
	   implementation_d == RESCORE_LANGUAGE) {

    return Error::handle(name(), L"algorithm and implementation is not implemented", Error::ARG, __FILE__, __LINE__);
    
  } // end of SYMBOL_GRAPH_RESCORING using RESCORE_LANGUAGE
    
  // branch on the SYMBOL_GRAPH_RESCORING using RESCORE_ACOUSTIC
  // or RESCORE_BOTH
  //
  else if (algorithm_d == SYMBOL_GRAPH_RESCORING &&
	   (implementation_d == RESCORE_ACOUSTIC ||
	    implementation_d == RESCORE_BOTH)) {
    
    // declare local variables
    //
    int32 num_valid_files = 0;
    int32 current_file_num = 0;
    int32 total_num_frames = 0;

    String identifier;
    Filename input_file_name;
    Filename symbol_graph_file_name;
    
    File buffer_file;
    Filename buffer_filename;
    Filename temp_buffer_filename;

    Filename input_ID;    
    String output;
    Filename output_file_name;
    Sdb output_sdb;
    Vector<VectorFloat> data;

    TranscriptionDatabase trans_db;
    
    // load the hmm models from the language model file
    //
    if (!load()) {
      return Error::handle(name(), L"run: load hmm models",
			   Error::ARG, __FILE__, __LINE__);
    }

    // set the search engine mode
    //
    search_engine_d.setInitialLevel(initial_level_d);          
    search_engine_d.setSearchMode(HierarchicalSearch::DECODE);    

    // set the rescoring mode to read the language model score from file
    //
    if (implementation_d == RESCORE_ACOUSTIC) {
      search_engine_d.getSearchLevel(initial_level_d).setRescore(true);
    }

    // when the output mode is FILE
    //
    File total_output_file;      
    if (output_mode_d == FILE) {

      // open the output file for the utterance hypotheses
      //
      if (output_file_d.length() != 0) {
      
	if (verbosity_d >= Integral::BRIEF) {
	  Console::increaseIndention();
	  output.assign(L"\nopening the output file: ");
	  output.concat(output_file_d);
	  Console::put(output);
	  Console::decreaseIndention();
	}

	if (!total_output_file.open(output_file_d, File::WRITE_ONLY)) {
	  return Error::handle(name(), L"networkDecoder - error opening output file", Error::ARG, __FILE__, __LINE__);
	}
	else {
	  total_output_file.close();
	}
      }
    }

    // when the output mode is DATABASE
    //
    Sof output_db_sof;
    if (output_mode_d == DATABASE) {

      if (output_file_d.length() > 0) {

	if (verbosity_d >= Integral::BRIEF) {
	  Console::increaseIndention();
	  output.assign(L"\nopening the output file: ");
	  output.concat(output_file_d);
	  Console::put(output);
	  Console::decreaseIndention();
	}

	// open the output file for the utterance hypotheses (TEXT)
	//
	if (output_type_d == TEXT) {	

	  if (!output_db_sof.open(output_file_d, File::WRITE_ONLY)) {
	    return Error::handle(name(), L"run: opening output file",
				 Error::ARG, __FILE__, __LINE__);
	  }
	  else {
	    
	    // read all identifiers form the sdb object
	    //
	    Vector<String> identifier_keys;
	    identifier_keys.setCapacity(sdb_a.length());
	    
	    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID);
		 sdb_a.gotoNext()) {
	      identifier.assign(input_ID);
	      identifier_keys.concat(identifier);
	    }

	    // write the frame-duration to the output file
	    //
	    Float frame_dur =
	      vector_fe_d(vector_fe_d.length() - 1).getFrameDuration();
	    frame_dur.write(output_db_sof, (int32)0);
	    
	    // write the identifiers to the transcription database	    
	    //
	    trans_db.storePartial(output_db_sof, 0, identifier_keys);
	  }
	} // end of output type TEXT

	// open the output file for the utterance hypotheses (BINARY)
	//
	if (output_type_d == BINARY) {	

	  if (!output_db_sof.open(output_file_d,
				  File::WRITE_ONLY, File::BINARY)) {
	    return Error::handle(name(), L"run: opening output file",
				 Error::ARG, __FILE__, __LINE__);
	  }
	  else {
	    
	    // read all identifiers form the sdb object
	    //
	    Vector<String> identifier_keys;
	    identifier_keys.setCapacity(sdb_a.length());
	    
	    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID);
		 sdb_a.gotoNext()) {
	      identifier.assign(input_ID);
	      identifier_keys.concat(identifier);
	    }

	    // write the frame-duration to the output file
	    //
	    Float frame_dur = vector_fe_d((int32)0).getFrameDuration();
	    frame_dur.write(output_db_sof, (int32)0);

	    // write the identifiers to the transcription database
	    //
	    trans_db.storePartial(output_db_sof, 0, identifier_keys);
	  }
	} // end of output type BINARY
      }
    }    
    
    // loop through the input utterances
    //
    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID); sdb_a.gotoNext()) {

      current_file_num++;

      // if input is streaming
      //
      if (stream_d) {

	// buffer the input so that it can be accesses multiple times
	//
	buffer_filename.assign(File::STREAM_FILE);
	buffer_file.open(buffer_filename);
	temp_buffer_filename = (String)(*(buffer_file.getTempFilename())[0]);
	input_file_name.assign(temp_buffer_filename);
      }

      // otherwise get the file path
      //
      else {
	
	// get audio file path
	//
	identifier.assign(input_ID);
	if (!audio_db_d.getRecord(identifier, input_file_name)) {
	  return Error::handle(name(), L"initialize - unable to find audio file for identifier", Error::ARG, __FILE__, __LINE__);	    
	}

	// get the symbol graph path
	//
	if (!symbol_graph_db_d.getRecord(identifier, symbol_graph_file_name)) {
	  return Error::handle(name(), L"initialize - unable to find audio file for identifier", Error::ARG, __FILE__, __LINE__);	    
	}	
      }
	
      // print utterance processing information
      //            
      if (verbosity_d >= Integral::BRIEF) {
	Console::increaseIndention();
	output.assign(L"\nprocessing file ");
	output.concat(current_file_num);
	output.concat(L" (");
	output.concat(identifier);
	output.concat(L")");
	output.concat(L": ");
	output.concat(input_file_name);
	Console::put(output);
	Console::decreaseIndention();
      }

      // initialize the decoder
      //
      if (!search_engine_d.initializeNetworkDecoder()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
      
      // initialize the top search level with the corresponding transcription
      //  we don't need a transcription database for segmented utterances
      //
      if (transcription_db_file_d.length() > 0) {
	initTranscription(identifier, current_file_num - 1);
      }

      // read the symbol graph and initialize the lattice
      //
      if (symbol_graph_db_file_d.length() > 0) {
	initSymbolGraph(symbol_graph_file_name);
      }
      
      // loop over all the front end files
      //
      for (int32 i = 0; i < vector_fe_d.length(); i++) {
	
	// process the utterance file by the front end
	//
	vector_fe_d(i).open(input_file_name);
	
	// retrieve the all frames of data in advance
	//
	extractFeatures(i, data);
      }
      
      // decode the utterance
      //	
      if (!verify_d) {
	search_engine_d.networkDecoder(vector_fe_d);
      }

      // close the FrontEnds
      //
      for (int32 i = 0; i < vector_fe_d.length(); i++) {
	vector_fe_d(i).close();
      }

      // close the input file
      //
      if (stream_d) {
	buffer_file.close();
      }
      
      // pick up the best hypothesis and its parameters
      //
      String hypotheses;
      float32 score = 0;
      int32 num_frames = 0;
      DoubleLinkedList<Instance> instance_path;
      
      if (!search_engine_d.getLexHypotheses(hypotheses, alignment_level_d, score,
					    num_frames, instance_path)) {

	// if no hypothesis found
	//
	hypotheses.clear();
	instance_path.clear();
	  
	if ((!verify_d) && (verbosity_d >= Integral::BRIEF)) {

	  // print the warning message
	  //
	  Console::increaseIndention();
	  Console::put(L"\nno hypothesis found");
	  Console::decreaseIndention();
	}
      }
      else {

	num_valid_files++;
	total_num_frames += num_frames;
      }

      // print the detailed info about the hypothesis
      //
      if (verbosity_d >= Integral::BRIEF) {
	Console::increaseIndention();
	Console::increaseIndention();
	output.assign(L"\nhyp:    ");
	output.concat(hypotheses);
	output.concat(L"\nscore:  ");
	output.concat(score);
	output.concat(L"   frames: ");
	output.concat(num_frames);
	Console::put(output);
	Console::decreaseIndention();
	Console::decreaseIndention();
      }

     
      if (output_mode_d == TRANSFORM) {

	// transform the input file and its path to the output file
	//
	sdb_a.transformName(output_file_name, input_file_name);

	// open the output file and write best hypothesis
	//
	File output_file;
	if (!output_file.open(output_file_name, File::WRITE_ONLY)) {
	  return Error::handle(name(), L"networkDecoder - error opening output file", Error::ARG, __FILE__, __LINE__);
	}
	output_file.put(hypotheses);
	      
	// close the output file
	//
	output_file.close();
      }

      if (output_mode_d == DATABASE) {

	String name_00(identifier);
	String gtype_00(identifier);	
	AnnotationGraph output_graph(name_00, gtype_00);

	// convert the best search path output format
	//
	search_engine_d.convertLexInstances(instance_path);
	

	// convert the best search path to an annotation graph
	//
	if (!createAnnotationGraph(output_graph, instance_path)) {
	  return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
	}

	// prune the annotation graph according to output levels
	//
	pruneAnnotationGraph(output_graph);
	
	// write the annotation graph to the database
	//
	trans_db.storePartial(output_db_sof, current_file_num - 1, output_graph);
      }
      
      if (output_mode_d == FILE) {
	
	// output the best hypothesis and sent ID to the output file
	//                                           or to stdout
	//
	output.assign(hypotheses);
	if ((int32)alignment_level_d < 0) {
	  output.concat(L" (");
	  output.concat(input_ID);
	  output.concat(L")\n");
	}

	if (output_file_d.length() != 0) {
	  total_output_file.open(output_file_d, File::APPEND_PLUS);
	  total_output_file.put(output);
	  total_output_file.close();
	}
	else {
	  if (verbosity_d < Integral::BRIEF) {
	    Console::increaseIndention();
	    Console::increaseIndention();
	    Console::put(output);
	    Console::decreaseIndention();
	    Console::decreaseIndention();
	  }
	}
      }

      if (output_mode_d == LIST) {
	
	// output the hypothesis to the corresponding file
	// from the output list
	//
	bool8 more_files;
	
	if (verbosity_d >= Integral::BRIEF) {
	  Console::increaseIndention();
	  output.assign(L"\nopening the output list: ");
	  output.concat(output_file_d);
	  Console::put(output);
	  Console::decreaseIndention();
	}
	
	if (current_file_num == 1) {
	  
	  // read output files list into signal database
	  //
	  Sof output_list_file;
	  if (!output_list_file.open(output_list_d)) {
	    return Error::handle(name(), L"networkDecoder - error opening output list", Error::ARG, __FILE__, __LINE__);
	  }
	  
	  output_sdb.read(output_list_file, 0);
	  output_list_file.close();
	  
	  more_files = output_sdb.gotoFirst();
	}
	else {

	  // move to the next output file
	  //
	  more_files = output_sdb.gotoNext();
	}
	
	if (!more_files) {
	  return Error::handle(name(), L"networkDecoder - insufficient output files in the output list", Error::ARG, __FILE__, __LINE__);
	}
	
	// open the next output file and write the best hypothesis
	//
	output_sdb.getName(output_file_name);
	
	File output_file;
	if (!output_file.open(output_file_name, File::WRITE_ONLY)) {
	  Console::increaseIndention();
	  output.assign(L"\ncannot open output file: ");
	  output.concat(output_file_name);
	  Console::put(output);
	  Console::decreaseIndention();
	}
	else {
	  output_file.put(hypotheses);
	  output_file.put(L"\n");
	  output_file.close();
	}
      }
    } // end of looping through the input utterances
      
    // clean up all memory
    //
    search_engine_d.clear();

    // close database files
    //
    if (output_mode_d == DATABASE) {
      output_db_sof.close();
    }

    // close the audio database (optional)
    //
    if (audio_db_file_d.length() > 0) {
      if (!audio_db_d.close()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
    }

    // close the audio database (optional)
    //
    if (symbol_graph_db_file_d.length() > 0) {
      if (!symbol_graph_db_d.close()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
    }
    
    // close the transcription database (optional)
    //
    if (transcription_db_file_d.length() > 0) {
      if (!transcription_db_d.close()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
    }
    
    // print the number of successfully processed files
    //
    if (verbosity_d >= Integral::BRIEF) {    
      output.assign(L"\nprocessed ");
      output.concat(num_valid_files);
      output.concat(L" file(s) successfully, attempted ");
      output.concat(current_file_num);
      output.concat(L" file(s), ");
      output.concat(total_num_frames);
      output.concat(L" frame(s)\n");    
      Console::put(output);
    }

  } // end of SYMBOL_GRAPH_RESCORING using RESCORE_ACOUSTIC or RESCORE_BOTH
  
  // branch on the SYMBOL_GRAPH_GENERATION using WORD_DEPENDENT_NBEST
  //
  else if (algorithm_d == SYMBOL_GRAPH_GENERATION &&
	   implementation_d == WORD_DEPENDENT_NBEST) {


    // declare local variables
    //
    int32 num_valid_files = 0;
    int32 current_file_num = 0;
    int32 total_num_frames = 0;

    String identifier;
    Filename input_file_name;
    Filename input_ID;
    
    String output;
    Filename output_file_name;
    Sdb output_sdb;

    // load the hmm models from the language model file
    //
    if (!load()) {
      return Error::handle(name(), L"networkDecoder - error loading models", Error::ARG, __FILE__, __LINE__);
    }

    // we only support the LIST and TRANSFORM mode for symbol graph generation
    //
    if ((output_mode_d == FILE) || (output_mode_d == DATABASE)) {
      return Error::handle(name(), L"networkDecoder - output mode is not supported for this algorithm", Error::ARG, __FILE__, __LINE__);
    }
    
    // set the search engine mode
    //
    search_engine_d.setInitialLevel(initial_level_d);          
    search_engine_d.setSearchMode(HierarchicalSearch::DECODE);    

    // loop through the input utterances
    //
    for (sdb_a.gotoFirst(); sdb_a.getName(input_ID); sdb_a.gotoNext()) {

      current_file_num++;

      // if input is streaming, set input_file_name to STREAM_FILE
      //
      if (stream_d) {
	input_file_name.assign(File::STREAM_FILE);
      }
      
      // otherwise, get the path
      //
      else {      

	// get audio file path
	//
	identifier.assign(input_ID);
	if (!audio_db_d.getRecord(identifier, input_file_name)) {
	  return Error::handle(name(), L"networkDecoder - unable to find audio file for identifier", Error::ARG, __FILE__, __LINE__);
	}
      }
      
      // print utterance processing information
      //
      if (verbosity_d >= Integral::BRIEF) {
	Console::increaseIndention();
	output.assign(L"\nprocessing file ");
	output.concat(current_file_num);
	output.concat(L" (");
	output.concat(identifier);
	output.concat(L")");
	output.concat(L": ");
	output.concat(input_file_name);
	Console::put(output);
	Console::decreaseIndention();
      }

      // initialize the decoder
      //
      if (!search_engine_d.initializeNetworkDecoder()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }      

      // initialize the top search level with the corresponding transcription
      //  we don't need a transcription database for segmented utterances
      //
      if (transcription_db_file_d.length() > 0) {
	initTranscription(identifier, current_file_num - 1);
      }
      
      // loop over all the front end files
      //
      for (int32 i = 0; i < vector_fe_d.length(); i++) {
	
	// process the utterance file by the front end
	//
	vector_fe_d(i).open(input_file_name);
      }
      
      // decode the utterance
      //      
      if (!verify_d) {
	search_engine_d.networkDecoder(vector_fe_d);
      }

      // close the FrontEnds
      //
      for (int32 i = 0; i < vector_fe_d.length(); i++) {
	vector_fe_d(i).close();
      }
      
      // initialize the symbol graph
      //
      float64 score = 0;
      int32 num_frames = 0;
      
      if (search_engine_d.initializeSymbolGraphInstance(score, num_frames)) {
	num_valid_files++;
	total_num_frames += num_frames;
      }

      else {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }

      // generate the symbol graph
      //
      if (!search_engine_d.generateSymbolGraph()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }

      // print the detailed info about the hypothesis
      //
      if (verbosity_d >= Integral::BRIEF) {
	Console::increaseIndention();
	output.assign(L"score:  ");
	output.concat(score);
	output.concat(L"   frames: ");
	output.concat(num_frames);
	Console::put(output);
	Console::decreaseIndention();
      }

      // when the output mode is a TRANSFORM
      //     
      if (output_mode_d == TRANSFORM) {

	// transform the input file and its path to the output file
	//
	sdb_a.transformName(output_file_name, input_file_name);

	// retrieve the symbol graph
	//
	SymbolGraph& symbol_graph =
	  search_engine_d.getSymbolGraph();

	Sof tmp_sof;
	if (!tmp_sof.open(output_file_name, File::WRITE_ONLY, File::TEXT)) {
	  Console::increaseIndention();
	  output.assign(L"\ncannot open output file: ");
	  output.concat(output_file_name);
	  Console::put(output);
	  Console::decreaseIndention();
	}
	
	// write the symbol graph to file
	//
	symbol_graph.write(tmp_sof, 0);
	tmp_sof.close();
      }
      
      // when the output mode is a LIST
      //
      if (output_mode_d == LIST) {

	// output the hypothesis to the corresponding file
	// from the output list
	//
	bool8 more_files;
	  
	if (current_file_num == 1) {
	    
	  // read output files list into signal database
	  //
	  Sof output_list_file;
	  if (!output_list_file.open(output_list_d)) {
	    return Error::handle(name(), L"networkDecoder - error opening output list", Error::ARG, __FILE__, __LINE__);
	  }
	  
	  output_sdb.read(output_list_file, 0);
	  output_list_file.close();
	  
	  more_files = output_sdb.gotoFirst();
	}
	else {
	  
	  // move to the next output file
	  //
	  more_files = output_sdb.gotoNext();
	}

	if (!more_files) {
	  return Error::handle(name(), L"linearDecoder - insufficient output files in the output list", Error::ARG, __FILE__, __LINE__);
	}
	  
	// open the next output file and write the best hypothesis
	//
	output_sdb.getName(output_file_name);

	// retrieve the symbol graph
	//
	SymbolGraph& symbol_graph =
	  search_engine_d.getSymbolGraph();

	Sof tmp_sof;
	if (!tmp_sof.open(output_file_name, File::WRITE_ONLY, File::TEXT)) {
	  Console::increaseIndention();
	  output.assign(L"\ncannot open output file: ");
	  output.concat(output_file_name);
	  Console::put(output);
	  Console::decreaseIndention();
	}
	
	// write the symbol graph to file
	//
	symbol_graph.write(tmp_sof, 0);
	tmp_sof.close();
      }
	
    } // end of looping through the input utterances

    // clean up all memory
    //
    search_engine_d.clear();

    // close the audio database (optional)
    //
    if (audio_db_file_d.length() > 0) {
      if (!audio_db_d.close()) {
	return Error::handle(name(), L"networkDecoder", Error::ARG, __FILE__, __LINE__);
      }
    }
    
    // print the number of successfully processed files
    //
    if (verbosity_d >= Integral::BRIEF) {    
      output.assign(L"\nprocessed ");
      output.concat(num_valid_files);
      output.concat(L" file(s) successfully, attempted ");
      output.concat(current_file_num);
      output.concat(L" file(s), ");
      output.concat(total_num_frames);
      output.concat(L" frame(s)\n");    
      Console::put(output);
    }    
    
  } // end of SYMBOL_GRAPH_GENERATION using WORD_DEPENDENT_NBEST
  
  // branch on the UNKNOWN options
  //
  else {    
    return Error::handle(name(), L"invalid algorithm and/or implementation", Error::ARG, __FILE__, __LINE__);
  }
  
  // gracefully exit
  //
  return true;    
}