// file: $isip/class/search/HierarchicalSearch/hsrch_09.cc
// version: $Id: hsrch_09.cc 9742 2004-08-20 20:43:15Z may $
//

// isip include files
//
#include "HierarchicalSearch.h"

// method: grammarDecoder
//
// arguments:
//  Vector<FrontEnd>& fe: (input) front-ends for extracting features
//  int32 num_frames: (input) the number of steps to move forward
//
// return: logical error status
//
// decode part (or all) of the search space - one pass grammar decoder
//
bool8 HierarchicalSearch::grammarDecoder(Vector<FrontEnd>& vector_fe_a,
					   int32 num_frames_a) {
  
  // if we haven't started the search then we need to initialize the search
  //
  if ((int32)current_frame_d == DEF_START_FRAME) {
    initializeGrammarDecoder();
  }

  // determine the number of frames to loop through. If the user specifies
  // to loop over all frames then the frame_num is set to a default value
  // so that the decoder loops indefinitely
  //
  int32 frame_num = 0;
  if (num_frames_a == DEF_NUM_FRAMES) {
    frame_num = num_frames_a - 1;
  }
  
  // loop over all the front ends except the last one that generates
  // the feature data
  //
  for (int32 i = 0; i < vector_fe_a.length() - 1; i++) {    
    vector_fe_a(i).getVector(features_d, 0, current_frame_d);
  }

  // loop over the specified number of frames
  //
  while (vector_fe_a(vector_fe_a.length() - 1).getVector(features_d, 0,
							 current_frame_d) &&
	 frame_num < num_frames_a) {

    // decode one frame of data
    //
    if (!traverseInstanceLevels()) {
      break;
    }

    // increment the frame_num if necessary - otherwise we loop indefinitely
    //
    if (num_frames_a != DEF_NUM_FRAMES) {
      frame_num++;
    }
  }

  // exit gracefully
  //
  return true;  
}

// method: grammarDecoder
//
// arguments:
//  Vector<VectorFloat>& fe: (input) vector of inputs
//  int32 num_frames: (input) the number of steps to move forward
//
// return: logical error status
//
// decode part (or all) of the search space - one pass grammar decoder
//
bool8 HierarchicalSearch::grammarDecoder(Vector<VectorFloat>& fe_a,
					int32 num_frames_a) {

  Error::handle(name(), L"not implemented", Error::ARG, __FILE__, __LINE__);
    
  // exit gracefully
  //
  return true;  
}

// method: linearDecoder
//
// arguments:
//  Vector<FrontEnd>& fe: (input) front-ends for extracting features
//  int32 num_frames: (input) the number of steps to move forward
//
// return: logical error status
//
// decode part (or all) of the search space
//
bool8 HierarchicalSearch::linearDecoder(Vector<FrontEnd>& vector_fe_a,
					  int32 num_frames_a) {

  // if we haven't started the search then we need to initialize the
  // search
  //
  if ((int32)current_frame_d == DEF_START_FRAME) {
    initializeLinearDecoder();
  }

  // determine the number of frames to loop through. If the user specifies
  // to loop over all frames then the frame_num is set to a default value
  // so that the decoder loops indefinitely
  //
  int32 frame_num = 0;
  if (num_frames_a == DEF_NUM_FRAMES) {
    frame_num = num_frames_a - 1;
  }
  
  // loop over all the front ends except the last one that generates
  // the feature data
  //
  for (int32 i = 0; i < vector_fe_a.length() - 1; i++) {    
    vector_fe_a(i).getVector(features_d, 0, current_frame_d);
  }
    
  // loop over the specified number of frames
  // ISIP_BUG: (hamaker) the channel is set to zero here which won't be
  //           right for multi-channel data
  //
  while (vector_fe_a(vector_fe_a.length() - 1).getVector(features_d, 0,
							 current_frame_d) &&
	 frame_num < num_frames_a) {
    
    // ISIP_BUG: check the feature length here
    // decode one frame of data
    //
    if (!traverseTraceLevels()) {
      break;
    }

    // increment the frame_num if necessary - otherwise we loop indefinitely
    //
    if (num_frames_a != DEF_NUM_FRAMES) {
      frame_num++;
    }
  }

  // exit gracefully
  //
  return true;
}

// method: linearDecoder
//
// arguments:
//  Vector<VectorFloat>& fe: (input) vector of inputs
//  int32 num_frames: (input) the number of steps to move forward
//
// return: logical error status
//
// decode part (or all) of the search space
//
bool8 HierarchicalSearch::linearDecoder(Vector<VectorFloat>& fe_a,
				   int32 num_frames_a) {

  // if we haven't started the search then we need to initialize the
  // search
  //
  if ((int32)current_frame_d == DEF_START_FRAME) {
    initializeLinearDecoder();
  }

  // determine the number of frames to loop through. If the user specifies
  // to loop over all frames then the frame_num is set to a default value
  // so that the decoder loops indefinitely
  //
  int32 frame_num = 0;
  if (num_frames_a == DEF_NUM_FRAMES) {
    frame_num = num_frames_a - 1;
  }
  
  // loop over the specified number of frames
  //
  int32 size = fe_a.length();
  
  while (current_frame_d < size &&
	 features_d.assign(fe_a(current_frame_d)) &&
	 frame_num < num_frames_a) {

    // decode one frame of data
    //
    if (!traverseTraceLevels()) {
      break;
    }
    
    // increment the frame_num if necessary - otherwise we loop indefinitely
    //
    if (num_frames_a != DEF_NUM_FRAMES) {
      frame_num++;
    }
  }
  
  // exit gracefully
  //
  return true;
}

// method: initializeContextGeneration
//
// arguments: none
//
// return: logical error status
//
// initialize the context generation setup
//
bool8 HierarchicalSearch::initializeContextGeneration() {

  // make sure that context does not exist for the transcription level
  //
  if ((*h_digraph_d)((int32)initial_level_d).useContext()) {
    return Error::handle(name(), L"initializeContextGeneration - context can not be defined for the transcription level", Error::ARG, __FILE__, __LINE__);
  }

  // set the context mode flag
  //
  context_generation_mode_d = true;
  
  // set the capacity of the hash tables and symbol tables
  //
  int32 num_symbols =
    (*h_digraph_d)((int32)context_level_d).getSymbolTable().length();

  int32 num_left_context =
    (*h_digraph_d)((int32)context_level_d).getLeftContext();

  int32 num_right_context =
    (*h_digraph_d)((int32)context_level_d).getRightContext();
  
  int32 capacity = (int32)Integral::pow((float64)num_symbols, (num_left_context + num_right_context + 1));
  
  context_list_d.setCapacity(capacity);
  context_hash_d.setCapacity(capacity);
  
  // exit gracefully
  //
  return true;
}

// method: initializeGrammarPartial
//
// arguments: none
//
// return: logical error status
//
// initialize the hypothesis path containers and forward-backward structures
//
bool8 HierarchicalSearch::initializeGrammarPartial() {

  // make sure that we have at least one level defined
  //
  int32 num_levels = getNumLevels();
  
  if (num_levels < 1) {
    return Error::handle(name(), L"initializeGrammarPartial", Error::ARG, __FILE__, __LINE__);
  }

  // make sure that context does not exist for the acoustic level
  //
  if ((*h_digraph_d)(num_levels - 1).useContext()) {
    return Error::handle(name(), L"initializeGrammarPartial - context can not be defined for the acoustic level", Error::ARG, __FILE__, __LINE__);
  }
  
  // clear the instance hypothesis list
  //
  clearInstanceStorage();
  
  // clear the instance lists of all the search nodes
  //
  clearSearchNodesInstanceLists();
         
  // clear the history pool
  //
  if (history_pool_d.length() > 0) {
    history_pool_d.clear(Integral::RETAIN);
  }

  // clear the context pool
  //
  if (context_pool_d.length() > 0) {
    context_pool_d.clear(Integral::RETAIN);
  }

  // clear the trellis before starting
  //  
  if (search_mode_d == TRAIN) {
    trellis_d.clear();
  }

  // set the current frame
  //
  current_frame_d = 0;
  
  // exit gracefully
  //
  return true;
}

// method: initializeGrammarDecoder
//
// arguments: none
//
// return: logical error status
//
// initialize the lists, etc. needed to begin searching
//
bool8 HierarchicalSearch::initializeGrammarDecoder() {

  // initialize the hypothesis path containers and forward-backward structures
  //
  initializeGrammarPartial();

  // error checking
  //
  if (getSearchLevel(initial_level_d).getRightContext() !=
      getSearchLevel(initial_level_d).getLeftContext()) {
    return Error::handle(name(), L"initializeGrammarDecoder - left and right context lengths must be the same", Error::ARG, __FILE__, __LINE__);
  }
  
  // get the position of the central item in the context
  //
  int32 central_pos =
    getSearchLevel(initial_level_d).getRightContext() + 1;
  int32 left_context_length =
    getSearchLevel(initial_level_d).getLeftContext();
  
  int32 total_context_length = left_context_length + central_pos;
  
  // initialize the context length
  //
  Context left_context(total_context_length, central_pos);  

  for (int32 i = 0; i < central_pos; i++) {
    left_context.assignAndAdvance((ulong)(*h_digraph_d)((int32)initial_level_d).getSubGraph(0).getStart());
  }
  
  // output the debugging information
  //
  if (debug_level_d >= Integral::ALL) {
    left_context.print();
  }
  
  // generate list of all possible right contexts
  //
  Context* symbol = context_pool_d.get(left_context);
  int32 depth = (*h_digraph_d)((int32)initial_level_d).getRightContext();
  if (!initializeRightContexts((Instance*)NULL, *symbol, 0, 0.0, depth)) {
    return Error::handle(name(), L"propagateInstancesDown", Error::ARG, __FILE__, __LINE__);
  }
  
  // exit gracefully
  //
  return true;
}

// method: initializeLinearPartial
//
// arguments: none
//
// return: logical error status
//
// initialize the hypothesis path containers and forward-backward structures
//
bool8 HierarchicalSearch::initializeLinearPartial() {

  // make sure that we have at least one level defined
  //
  if ( getNumLevels() < 1) {
    return Error::handle(name(), L"initializeLinearPartial", Error::ARG, __FILE__, __LINE__);
  }
  
  // clear the trace hypothesis list
  //
  clearTraceStorage();
  
  // clear the trace lists of all the search nodes
  //
  clearSearchNodesTraceLists();

  // clear the history pool
  //
  if (history_pool_d.length() > 0) {
    history_pool_d.clear(Integral::RETAIN);
  }
  
  // clear the context pool
  //
  if (context_pool_d.length() > 0) {
    context_pool_d.clear(Integral::RETAIN);
  }
  
  // clear the trellis before starting
  //
  if (search_mode_d == TRAIN) {
    trellis_d.clear();
  }

  // clear the symbol graph
  //
  symbol_graph_d.clear(Integral::RESET);
  
  // set the current frame
  //
  current_frame_d = 0;
  
  // exit gracefully
  //
  return true;  
}

// method: initializeLinearDecoder
//
// arguments: none
//
// return: logical error status
//
// initialize the lists, etc. needed to begin searching
//
bool8 HierarchicalSearch::initializeLinearDecoder() {

  // declare local variables
  //
  DoubleLinkedList<Float> score_list(DstrBase::USER);  
  DoubleLinkedList<Context> right_context_list(DstrBase::USER);  
  
  // initialize the hypothesis path containers and forward-backward structures
  //
  initializeLinearPartial();

  // error checking
  //
  if (getSearchLevel(initial_level_d).getRightContext() !=
      getSearchLevel(initial_level_d).getLeftContext()) {
    return Error::handle(name(), L"initializeLinearDecoder - left and right context lengths must be the same", Error::ARG, __FILE__, __LINE__);
  }
  
  // get the position of the central item in the context
  //
  int32 central_pos =
    getSearchLevel(initial_level_d).getRightContext() + 1;
  int32 left_context_length =
    getSearchLevel(initial_level_d).getLeftContext();

  int32 total_context_length = left_context_length + central_pos;
  
  // initialize the context length
  //
  Context left_context(total_context_length, central_pos);

  for (int32 i = 0; i < central_pos; i++) {
    left_context.assignAndAdvance((ulong)(*h_digraph_d)((int32)initial_level_d).getSubGraph(0).getStart());
  }

  // output the debugging information
  //
  if (debug_level_d >= Integral::ALL) {
    left_context.print();
  }
  
  // generate all possible right contexts by extending left context to
  // the depth of the right context
  //
  Context* symbol = context_pool_d.get(left_context);
  generateRightContexts(right_context_list, score_list, symbol,
		      (*h_digraph_d)((int32)initial_level_d).getRightContext(),
		      (int32)initial_level_d);
  
  // generate trace for each right conext
  //
  for (bool8 more_items =
	 right_context_list.gotoFirst() && score_list.gotoFirst();
       more_items;
       more_items = right_context_list.gotoNext() && score_list.gotoNext()) {
    
    // seed the top trace list with the start node of the search graph
    //
    Trace* tmp_trace = new Trace();
    tmp_trace->setFrame((ulong)current_frame_d);
    
    if (search_mode_d == TRAIN) {
      tmp_trace->setTraceMode(Trace::TRAIN);
    }

    tmp_trace->setBackPointer((Trace*)NULL);    
    tmp_trace->setSymbol(right_context_list.getCurr());

    // initialize the structure that holds the n-symbols
    //
    initializeNsymbolTrace(tmp_trace);
    
    // update the score
    //
    tmp_trace->setScore(tmp_trace->getScore() + (float32)(*score_list.getCurr()));
 
    // has the history been previously generated? if so reuse the history,
    // if not generate a new history and add it to the history pool
    //
    tmp_trace->setHistory(history_pool_d.initAndAllocate());
    
    // add the start trace to the list
    //
    trace_lists_d((int32)initial_level_d).insertLast(tmp_trace);
    
    // add the start trace to the trellis
    //
    if (search_mode_d == TRAIN) {
      String output;
      Long val;
      BiGraphVertex<TrainNode>* src = trellis_d.getStart();
      BiGraphVertex<TrainNode>* dst = insertTrace(tmp_trace);
      trellis_d.insertArc(src, dst, false, 0);
    }

    // output the debugging information
    //
    if (debug_level_d >= Integral::DETAILED) {
      printNewPath(tmp_trace, (Trace*)NULL);
    }
  }
    
  // clear up memory
  //
  right_context_list.clear();
  
  score_list.setAllocationMode(DstrBase::SYSTEM);
  score_list.clear();
  
  // exit gracefully
  //
  return true;
}

// method: initializeNsymbolTrace
//
// arguments:
//  Trace* trace_a: (input) current trace
//
// return: logical error status
//
// initialze the first n-symbols for a trace
//
bool8 HierarchicalSearch::initializeNsymbolTrace(Trace*& curr_trace_a){

    // initialize the structure that holds the n-symbol contexts
    //
    int32 num_levels = getNumLevels();
    nsymbol_indices_d.setLength(num_levels);

    // error checking
    //
    if (num_levels == 0) {
      return Error::handle(name(), L"initializeNsymbolTrace", Error::ARG, __FILE__, __LINE__);
    }
    
    // loop over each search level in the hierarchy
    //
    int32 num_nsymbol_levels = 0;
    for (int32 i = 0; i < num_levels; i++) {
      
      nsymbol_indices_d(i) = -1;
      
      if (getSearchLevel(i).useNSymbol()) {	
	nsymbol_indices_d(i) = num_nsymbol_levels++;	
      }
    }
  
    // set the actual n-symbol levels
    //
    Context**& nsymbol = curr_trace_a->getNSymbol();
    
    if (num_nsymbol_levels > 0) {
      nsymbol = new Context*[num_nsymbol_levels];
      curr_trace_a->setNSymbolLength(num_nsymbol_levels);
    }
    
    // loop over each search level in the hierarchy
    //
    for (int32 i = 0; i < num_levels; i++) {
      
      if (getSearchLevel(i).useNSymbol()) {
	
	// get the n-symbol order for this search level
	//
	int32 nsymbol_order = getSearchLevel(i).getNSymbolOrder();
	
	// initialize the n-symbol context for this level
	//
	Context tmp_context(nsymbol_order);	  
	for (int32 j = 0; j < nsymbol_order; j++) {
	    tmp_context.assignAndAdvance((ulong)(*h_digraph_d)(i).getSubGraph(0).getStart());
	}

	// error checking
	//
	if (((int32)nsymbol_indices_d(i) < 0) ||
	    ((int32)nsymbol_indices_d(i) >= num_levels)) {
	  return Error::handle(name(), L"initializeNsymbolTrace", Error::ARG, __FILE__, __LINE__);
	}
	
	nsymbol[(int32)nsymbol_indices_d(i)] = context_pool_d.get(tmp_context);
      }
    }

    // exit gracefully
    //
    return true;  
}

// method: initializeNsymbolInstance
//
// arguments:
//  Instance* instance_a: (input) current instance
//
// return: logical error status
//
// initialze the first n-symbols for a instance
//
bool8 HierarchicalSearch::initializeNsymbolInstance(Instance*& curr_instance_a){

    // initialize the structure that holds the n-symbol contexts
    //
    int32 num_levels = getNumLevels();
    nsymbol_indices_d.setLength(num_levels);
    
    // loop over each search level in the hierarchy
    //
    int32 num_nsymbol_levels = 0;
    for (int32 i = 0; i < num_levels; i++) {
      
      // determine is n-symbol probabilities are used at this level
      //
      nsymbol_indices_d(i) = -1;
      
      if (getSearchLevel(i).useNSymbol()) {	
	nsymbol_indices_d(i) = num_nsymbol_levels++;	
      }
    }
  
    // set the actual n-symbol levels
    //
    Context**& nsymbol = curr_instance_a->getNSymbol();
    
    if (num_nsymbol_levels > 0) {
      nsymbol = new Context*[num_nsymbol_levels];
      curr_instance_a->setNSymbolLength(num_nsymbol_levels);      
    }
    
    // loop over each search level in the hierarchy
    //
    for (int32 i = 0; i < num_levels; i++) {
      
      // determine is n-symbol probabilities are used at this level
      //
      if (getSearchLevel(i).useNSymbol()) {
	
	// get the n-symbol order for this search level
	//
	int32 nsymbol_order = getSearchLevel(i).getNSymbolOrder();
	
	// initialize the n-symbol context for this level
	//
	Context tmp_context(nsymbol_order);	  
	for (int32 j = 0; j < nsymbol_order; j++) {
	    tmp_context.assignAndAdvance((ulong)(*h_digraph_d)(i).getSubGraph(0).getStart());
	}

	// error checking
	//
	if (((int32)nsymbol_indices_d(i) < 0) ||
	    ((int32)nsymbol_indices_d(i) >= num_levels)) {
	  return Error::handle(name(), L"initializeNsymbolInstance", Error::ARG, __FILE__, __LINE__);
	}
	
	nsymbol[(int32)nsymbol_indices_d(i)] = context_pool_d.get(tmp_context);
      }
    }

    // exit gracefully
    //
    return true;  
}

// method: shiftNSymbol
//
// arguments:
//   Trace*& trace: (input) input hypothesis path
//   int32 level: (input) search level we are at
//   GraphVertex<SearchNode>* vertex: (input) graph vertex wer are at
//
// return: n-symbol probability
//
// method computes the n-symbol probability corresponding to the instance
//
bool8 HierarchicalSearch::shiftNSymbol(Trace*& trace_a, int32 level_a,
					 GraphVertex<SearchNode>* vertex_a) {

  // error checking
  //
  if (trace_a == (Trace*)NULL) {
    return Error::handle(name(), L"shiftNSymbol", Error::ARG, __FILE__, __LINE__);
  }

  if ((level_a < 0) || (level_a >= nsymbol_indices_d.length())) {
    return Error::handle(name(), L"shiftNSymbol", Error::ARG, __FILE__, __LINE__);
  }
  
  // shift the vertex into the n-symbol list at this level
  //
  Context**& nsymbol = trace_a->getNSymbol();

  if (nsymbol == (Context**)NULL) {
    return Error::handle(name(), L"shiftNSymbol", Error::ARG, __FILE__, __LINE__);
  }

  Context* symbol = nsymbol[(int32)nsymbol_indices_d(level_a)];
  
  nsymbol[(int32)nsymbol_indices_d(level_a)] =
    context_pool_d.shiftAndAllocate(symbol, vertex_a);
  
  // exit gracefully
  //
  return true;  
}

// method: shiftNSymbol
//
// arguments:
//   Instance*& instance: (input) input hypothesis path
//   int32 level: (input) search level we are at
//   GraphVertex<SearchNode>* vertex: (input) graph vertex wer are at
//
// return: n-symbol probability
//
// method computes the n-symbol probability corresponding to the instance
//
bool8 HierarchicalSearch::shiftNSymbol(Instance*& instance_a, int32 level_a,
					 GraphVertex<SearchNode>* vertex_a) {

  // error checking
  //
  if (instance_a == (Instance*)NULL) {
    return Error::handle(name(), L"shiftNSymbol", Error::ARG, __FILE__, __LINE__);
  }

  if ((level_a < 0) || (level_a >= nsymbol_indices_d.length())) {
    return Error::handle(name(), L"shiftNSymbol", Error::ARG, __FILE__, __LINE__);
  }

  // shift the vertex into the n-symbol list at this level
  //
  Context**& nsymbol = instance_a->getNSymbol();

  if (nsymbol == (Context**)NULL) {
    return Error::handle(name(), L"shiftNSymbol", Error::ARG, __FILE__, __LINE__);
  }

  Context* symbol = nsymbol[(int32)nsymbol_indices_d(level_a)];
  
  nsymbol[(int32)nsymbol_indices_d(level_a)] =
    context_pool_d.shiftAndAllocate(symbol, vertex_a);
  
  // exit gracefully
  //
  return true;  
}

// method: getPosteriorScore
//
// arguments:
//   Context** context: (input) n-symbol history
//   int32 level: (input) level of the context 
//
// return: n-symbol probability
//
// method computes the n-symbol probability corresponding to the instance
//
float32 HierarchicalSearch::getPosteriorScore(Context** context_a, int32 level_a) {

  // declare local variables
  //
  int32 lm_order = 0;
  int32 nsymbol_index = (int32)nsymbol_indices_d(level_a);
    
  float32 lm_scale = 1.0;
  float32 lm_score = 0.0;
  
  // error checking
  //
  if (context_a == (Context**)NULL) {
    return Error::handle(name(), L"getPosteriorScore", Error::ARG, __FILE__, __LINE__);
  }

  // print debugging information
  //
  if (debug_level_d >= Integral::ALL) {
    context_a[nsymbol_index]->print();
  }
  
  // get the language model scale factor
  //
  lm_scale = getSearchLevel(level_a).getLmScale();

  // get the order of the language model
  //
  lm_order = getSearchLevel(level_a).getNSymbolOrder();

  // get the n-gram language model
  //
  NGramModel& ngram_model = getSearchLevel(level_a).getNSymbolModel();

  // lookup the ngram score for the context
  //
  lm_score = nsymbol_cache_d.getScore(ngram_model, *(context_a[nsymbol_index]));

  // return the scaled n-symbol probability
  //
  return (lm_scale * lm_score);
}