// file: $isip/class/search/StackSearch/ssrch_11.cc
// version: $Id: ssrch_11.cc 9173 2003-05-30 02:02:08Z jelinek $
//

// isip include files
//
#include "StackSearch.h"

// method: traverseTraceLevels
//
// arguments: none
//
// return: logical error status
//
// do a single-step traversal of the hypotheses at each level
//
bool8 StackSearch::traverseTraceLevels() {
  
  // define local variables
  //
  bool8 status = false;
  
  // make sure we have some paths left to traverse
  //
  if (!pathsRemainTrace()) {
    return false;
  }

  // propagate traces forward until they are all ready to be evaluated
  //
  status = (propagateTraces() || status);

  // now we evaluate all traces that are in the lowest level and move 
  // them forward one arc. Viterbi pruning takes place at this level 
  // as does beam pruning.
  //
  status = (evaluateTraceModels() || status);

  // exit gracefully
  //
  return status;
}

// method: propagateTraces
//
// arguments:
//  DoubleLinkedList<Trace>* pickup_list: (output) traces from pickup level
//  int32 pickup_level: (input) level to pick up the traces from
//
// return: logical error status
//
// move traces through the hierarchy until all active traces are ready to be
// evaluated
//
bool8 StackSearch::propagateTraces(DoubleLinkedList<Trace>* pickup_list_a,
				     int32 pickup_level_a) {
  
  // define local variables
  //
  int32 level_num = 0;
  bool8 status = true;
  int32 num_levels = getNumLevels();
  String out;
  
  // in viterbi mode, clear the hypothesis list since we are generating new
  // hypotheses for this frame
  // in stack mode, (picking up some level) do not clear valid traces list 
  //
  if (pickup_level_a == -1) {
    clearValidHypsTrace();
  }
  // clear the list of terminated traces - those who made it to the
  // end of grammar sooner than the end frame
  //
  else {
    clearTermHypsTrace();
  }
  
  // move all traces forward in the search space. we do this by
  // pushing traces up and down in the hierarchy until they all rest
  // in a state that is ready to be evaluated. if there is an infinite
  // "skip" loop in the search graph then this process will go into an
  // infinite loop. an infinite skip loop is of the form:
  // A -> B && B -> A.
  // this infinite loop structure may be masked by the fact that it 
  // extends over multiple search levels.
  // 
  // when we exit the while loop below, all traces should be in the lowest
  // level and should be ready for evaluation.
  //
  while ((getActiveTraces() != getActiveTraces(num_levels - 1)) ||
	 status) {
    
    // status indicates whether any movement in the traces happened
    //
    status = false;
    
    // work from the bottom up until we reach a point where all traces
    // are ready to descend again. this segment takes care of
    // posterior scores such as nsymbol probabilities and applies
    // pruning (beam and instance) at each level.
    //
    for (level_num = num_levels - 1; level_num >= 0; level_num--) {
      
      // propagate up
      //
      status = (propagateTracesUp(level_num, pickup_list_a, pickup_level_a) ||
		status);
    }
    
    for (level_num = 0;  level_num < (num_levels - 1); level_num++) {
      if (getSearchLevel(level_num).useBeam()) {
	if (debug_level_d >= Integral::DETAILED) {
	  Console::put(L"\npruning after propagation traces up");
	}
	beamPruneTrace(level_num);
      }
    }
    
    // propagate traces down the hierarchy, carrying out Viterbi pruning
    // as we go.
    //
    for (level_num = 0;  level_num < num_levels - 1; level_num++) {
      
      // propagate down
      //
      status = (propagateTracesDown(level_num, pickup_list_a, pickup_level_a) ||
		status);
    }
  }
  
  // exit gracefully
  //
  return true;
}

// method: evaluateTraceModels
//
// arguments:
//  DoubleLinkedList<Trace>* pickup_list: (output) traces from pickup level
//  int32 pickup_level: (input) level to pick up the traces from
//
// return: logical error status.
//
// evaluate all statistical models
//
bool8 StackSearch::evaluateTraceModels(
                                DoubleLinkedList<Trace>* pickup_list_a,
		                int32 pickup_level_a) {

  // define local variables
  //
  String out;
  String val;
  int32 level_num = getNumLevels() - 1;
  bool8 end_loop = false;
  Trace* curr_trace = (Trace*)NULL;
  Trace* next_trace = (Trace*)NULL;    
  History* curr_history = (History*)NULL;
  GraphVertex<SearchNode>* curr_vert = (GraphVertex<SearchNode>*)NULL;
  SearchSymbol item;
 
  // output the frame information
  //
  if (debug_level_d >= Integral::DETAILED) {
    out.assign(L"\nevaluating data in frame "); out.concat(current_frame_d);
    Console::put(out);
  }

  // in FRAME_EVAL mode, evaluate frame and do a beam pruning
  //
  if (evaluation_mode_d == FRAME_EVAL) {
    
    // loop over all active traces and score them
    //
    for (bool8 more_traces = trace_lists_d(level_num).gotoFirst();
	 more_traces; more_traces = trace_lists_d(level_num).gotoNext()) {
    
      // get the current vertex in the search graph
      //
      curr_trace = trace_lists_d(level_num).getCurr();
      curr_history = curr_trace->getHistory();
      curr_vert = curr_trace->getSymbol()->getCentralVertex();

      // evaluate the statistical model
      //
      SearchNode* p_sn = curr_vert->getItem();
      if (p_sn == NULL) {
	return Error::handle(name(),
			     L"evaluateTraceModels - no SearchNode in Vertex",
			     Error::ARG, __FILE__, __LINE__);
      }

      // find the statistical model index
      //
      // get the symbol id and find its statistical model index
      //
      int32 symbol_id = p_sn->getSymbolId();
      int32 stat_model_id = (int32)stat_model_mapping_d(symbol_id);

      // having the statistical model index, we can check whether the
      // statistical model has been evaluated
      //
      float32 stat_score = stat_model_scores_d(current_frame_d, stat_model_id);
      if (stat_score == Trace::INACTIVE_SCORE) {
	num_eval_d(current_frame_d)++;
	stat_score =
	  p_sn->evaluateData(features_d(current_frame_d), current_frame_d);
	stat_model_scores_d.setValue(current_frame_d, stat_model_id, stat_score);
      }
      else {
	num_avoided_eval_d(current_frame_d)++;
	Float stat_score2 = stat_score;
	if (stat_score2.almostEqual(Trace::INACTIVE_SCORE)) {
	  return Error::handle(name(), L"evaluateTraceModels - float32 precision",
			       Error::ARG, __FILE__, __LINE__);
	}
      }
    
      float32 new_score = curr_trace->getScore() + stat_score;
      curr_trace->setScore(new_score);
    
      // update the maximal trace score for the beam pruning
      //
      if (getSearchLevel(level_num).useBeam()
	  && (new_score > max_frame_scores_d(current_frame_d, level_num))) {

	if (debug_level_d >= Integral::DETAILED) {
		
	  // print the score update information
	  //
	  out.assign(L"fr:");
	  out.concat(current_frame_d);
	  out.concat(L" lv:");
	  out.concat(level_num);
	  out.concat(L" max_sc updt (after the model EVALUATION) ");
	  out.concat(max_frame_scores_d(current_frame_d, level_num));
	  out.concat(L"->");
	  out.concat(new_score);
	  Console::put(out);
	}

	// update the maximal trace score
	//
	max_frame_scores_d.setValue(current_frame_d, level_num, new_score);
      }
    }
    
    // do the beam pruning at this level
    //
    if (getSearchLevel(getNumLevels() - 1).useBeam()) {
      if (debug_level_d >= Integral::DETAILED) {
	Console::put(L"\npruning after the model EVALUATION");
      }
      beamPruneTrace(getNumLevels() - 1);
    }

  } // end of if FRAME_EVAL

  // update current frame
  //        - traces generated later will correspond to the next frame
  //
  current_frame_d += 1;
  
  if (debug_level_d >= Integral::DETAILED) {
      out.assign(L"\nTIME CHANGED to: ");
      out.concat(current_frame_d);
      out.concat(L"\n");
      Console::put(out);
  }

  // we mark the current end of the list so we only propagate those traces
  // forward that were just evaluated
  //
  trace_lists_d(level_num).gotoLast();
  trace_lists_d(level_num).setMark();
  trace_lists_d(level_num).gotoFirst();

  if (debug_level_d >= Integral::DETAILED) {
    Console::put(L"  propagating traces forward");
  }
  
  // loop until we have propagated all traces forward
  //
  end_loop = (trace_lists_d(level_num).length() < 1);
  while (!end_loop) {
    
    end_loop = trace_lists_d(level_num).isMarkedElement();
    
    // get the current vertex in the search graph
    //
    curr_trace = trace_lists_d(level_num).getCurr();
    curr_history = curr_trace->getHistory();
    curr_vert = curr_trace->getSymbol()->getLastVertex();
    
    // if this trace is inactive, then delete it
    //
    if (!curr_trace->isActive()) {
      
      // bump the current trace off of the trace list
      //
      if (debug_level_d >= Integral::ALL) {
	Console::put(L" not active");
      }      
      trace_lists_d(level_num).removeFirst(curr_trace);
      num_traces_vit_prun_d(curr_trace->getFrame())++;
      Trace::deleteTrace(curr_trace, true);
    }
    
    // else, extend paths and apply viterbi pruning
    //
    else {
      
      // extend all paths from that vertex and apply viterbi pruning
      //
      for (bool8 more_paths = curr_vert->gotoFirst(); more_paths;
	   more_paths = curr_vert->gotoNext()) {
	
	// create a new trace
	//
	next_trace = new Trace(*curr_trace);
	num_traces_gen_d(current_frame_d)++;
	next_trace->setFrame((ulong)current_frame_d);
	
	// update the score with the arc score
	//
	GraphArc<SearchNode>* tmp_arc = curr_vert->getCurr();
	next_trace->setScore(next_trace->getScore() +
			     tmp_arc->getWeight());
	
	// has the context been previously generated? if so reuse the
	// context, if not generate a new context and add it to the
	// context pool
	//
	next_trace->setSymbol(context_pool_d.shiftAndAllocate(next_trace->getSymbol(), tmp_arc->getVertex()));
	
	// check for segment evaluation
	//
	if (evaluation_mode_d == SEGMENT_EVAL) {
    
	  GVSnode* central_vert = curr_trace->getSymbol()->getCentralVertex();
	  GVSnode* next_central_vert = next_trace->getSymbol()
	    ->getCentralVertex();

	  // if there is no sefl-loop in SEGMENT_EVAL mode, segment is
	  // evaluated, score is updated and trace is added to node
	  //
	  if (central_vert != next_central_vert) {
	  
	    // find the start frame
	    //
	    Trace* prev = curr_trace->getBackPointer();
	    GVSnode* prev_vert = prev
	      ->getSymbol()->getCentralVertex();
	    
	    int32 start_frame = current_frame_d - (int32)1;
	    while (central_vert == prev_vert) {
	      prev = prev->getBackPointer();
	      prev_vert = prev->getSymbol()->getCentralVertex();
	      start_frame = prev->getFrame();
	    }
	    
	    // evaluate segment feature
	    //
	    SearchNode* p_sn = curr_vert->getItem();
	    if (p_sn == NULL) {
	      return Error::handle(name(),
			      L"evaluateTraceModels - no SearchNode in Vertex",
			      Error::ARG, __FILE__, __LINE__);
	    }
	    if ((prev_vert->getItem()->getSearchLevel()->getLevelIndex()
		== level_num) && (start_frame != (current_frame_d - (int32)1))
		) {
	      ++start_frame;
	    }

	    if (debug_level_d >= Integral::ALL) {
	      printf("stf:%d endf: %d", start_frame,
		     current_frame_d - (int32)1);
	    }
	    
	    int32 shift = current_frame_d - start_frame - 1;
	    StatisticalModel* sm = p_sn->getStatisticalModel();
	    float32 score = sm->getLogLikelihood(
				    svm_features_d(start_frame)(shift)
				    );
	    num_eval_d(current_frame_d - (int32)1)++;

	    SearchSymbol ss;
	    p_sn->getSymbol(ss);
	    if (ss.eq(L"sil")) {
	      if (debug_level_d >= Integral::ALL) {
		String out(L" sc without penalty: ");
		out.concat(score);
		Console::put(out);
	      }
	      score -= sil_dur_penalty_d;
	    }
	    
	    if (score_normalization_d == DURATION) {
	      score = score * (shift + 1);
	    }
	    else if (score_normalization_d == NUM_OF_PHONES) {
	      return Error::handle(name(), L"evaluateTraceModels - num of phones normalization is not implemented yet",
				   Error::ARG, __FILE__, __LINE__);
	    }

	    // output score and symbol
	    //
	    if (debug_level_d >= Integral::ALL) {
	      String out(L" ev_sc: ");
	      out.concat(score);
	      out.concat(L" ");
	      out.concat(ss);
	      Console::put(out);
	    }

	    // update trace score
	    //
	    float32 new_score = curr_trace->getScore() + score;
	    next_trace->setScore(new_score);
	    
	    // add the trace to the search node's trace list
	    //
	    SearchNode* next_node = next_central_vert->getItem();
	    Trace* existing_trace = NULL;
	    if (next_node->addTrace(next_trace, current_frame_d,
				    existing_trace)) {
	      
	      // update the maximal trace score for the beam pruning.
	      // not evaluated (looping) traces may have better score,
	      // thus only evaluated traces are used to update max score
	      //
	      if (getSearchLevel(level_num).useBeam() &&
		  (new_score > max_frame_scores_d(current_frame_d,
						  level_num))) {

		// print the score update information
		//
		if (debug_level_d >= Integral::DETAILED) {
		  out.assign(L"fr:");
		  out.concat(current_frame_d);
		  out.concat(L" lv:");
		  out.concat(level_num);
		  out.concat(L" max_sc updt (after the SEGMENT EVALUATION) ");
		  out.concat(max_frame_scores_d(current_frame_d, level_num));
		  out.concat(L"->");
		  out.concat(new_score);
		  Console::put(out);
		}

		// update the maximal trace score
		//
		max_frame_scores_d.setValue(current_frame_d,
					    level_num, new_score);
	      } // end of if max score updated

	      // set the backpointer
	      //
	      next_trace->setBackPointer(curr_trace);
	      
	      // debug output
	      //
	      if (debug_level_d >= Integral::ALL) {
		printNewPath(next_trace, curr_trace);
		if (existing_trace != NULL) {
		  printDeletedPath(existing_trace,
				   existing_trace->getBackPointer());
		}
	      } // end of debug
	      
	      // add the trace to this level's tarce list
	      //
	      if ((pickup_level_a != level_num)
		  || next_central_vert->isTerm()) {
		trace_lists_d(level_num).insertLast(next_trace);
	      } // end of if adding to trace list

	      // add the trace to the pickup list
	      //
	      else {
		pickup_list_a->insertLast(next_trace);
		if (debug_level_d >= Integral::ALL) {
		  printf("catch\n");
		}
	      } // end of adding to pickup list
	    } // end of if trace added succesffully
	  } // end of no self-loop

	  // else there is a self-loop in SEGMENT_EVAL mode, we will
	  // neither evaluate trace, nor add it to SearchNode, but
	  // propagate anyway
	  //
	  else {
	    
	    // set the backpointer
	    //
	    next_trace->setBackPointer(curr_trace);

	    // add the trace to theis level's tarce list
	    //
	    trace_lists_d(level_num).insertLast(next_trace);

	    if (debug_level_d >= Integral::ALL) { 
	      printNewPath(next_trace, curr_trace);
	    } // end of debug
	    
	  } // end of else there is a self-loop in SEGMENT_EVAL mode
	} // end of SEGMENT_EVAL mode
	
	else if (evaluation_mode_d == FRAME_EVAL) {
	  // add the trace to the search node's trace list
	  //
	  Trace* existing_trace = (Trace*)NULL;
	  GVSnode* next_vert = next_trace->getSymbol()->
	    getCentralVertex(); 
	  SearchNode* next_node = next_vert->getItem();
 
	  if (next_node->addTrace(next_trace, current_frame_d,
				  existing_trace)) {
	  
	    // set the backpointer
	    //
	    next_trace->setBackPointer(curr_trace);
	
	    if (debug_level_d >= Integral::ALL) {
	      printNewPath(next_trace, curr_trace);
	    }
	    
	    // add the trace to this level's trace list
	    //
	    if ((pickup_level_a != level_num) || (next_vert->isTerm())) {
	      trace_lists_d(level_num).insertLast(next_trace);
	    }
	    else {
	      pickup_list_a->insertLast(next_trace);
	      if (debug_level_d >= Integral::ALL) {
		printf("catch\n");
	      } 
	    } // end of else trace was added to pickup list
	  } // end of if trace was added to search node succesfully
	  
	  // else trace was not added to search node
	  //
	  else {

	    // delete the trace since there is an existing trace -
	    // same history
	    //
	    if (debug_level_d >= Integral::ALL) { 
	      printDeletedPath(next_trace, curr_trace);
	    }	  
	    delete next_trace;
	  } 
	} // end of FRAME_EVAL
      } // end of loop over all extensions

      // bump the current trace off of the trace list
      //
      if (curr_trace->getRefCount() < 1) {
	num_traces_vit_prun_d(curr_trace->getFrame())++;
	Trace::deleteTrace(curr_trace, true);
      }
      trace_lists_d(level_num).removeFirst(curr_trace);
      trace_lists_d(level_num).gotoFirst();
      
    } // end of if the trace is active
  } // end of loop over all state level traces - forward propagation
  
  // exit gracefully
  //
  return true;
}


// method: propagateTracesUp
//
// arguments:
//  int32 level_num: (input) the level for which we will propagate traces
//  DoubleLinkedList<Trace>* pickup_list: (output) traces from pickup level
//  int32 pickup_level: (input) level to pick up the traces from
//
// return: logical error status. if no propagations were made then return false
//
// propagate all traces up. only traces at the end of a subgraph are considered
//
bool8 StackSearch::propagateTracesUp(int32 level_num_a,
		       DoubleLinkedList<Trace>* pickup_list_a,
				       int32 pickup_level_a) {

  // define local variables
  //
  bool8 status = false;
  bool8 end_loop = false;
  Trace* curr_trace = (Trace*)NULL;
  Trace* tmp_trace = (Trace*)NULL;
  Trace* next_trace = (Trace*)NULL;
  GraphVertex<SearchNode>* curr_vert = (GraphVertex<SearchNode>*)NULL;
  GraphVertex<SearchNode>* tmp_vert = (GraphVertex<SearchNode>*)NULL;
  Context useless;

  // output the debugging information
  //
  if (debug_level_d >= Integral::ALL) {
    String output(L"propagating traces up from level: ");
    output.concat(level_num_a); output.concat(L" in frame: ");
    output.concat(current_frame_d);
    Console::put(output);
  }
  
  // make sure there is at least one trace to propagate
  //
  if (trace_lists_d(level_num_a).length() < 1) {
    return false;
  }
  
  // loop over all current traces and create the next level's traces. all new
  // traces will go at the end of the list so we set the mark to tell us when
  // to stop propagating
  //
  trace_lists_d(level_num_a).gotoLast();
  trace_lists_d(level_num_a).setMark();
  trace_lists_d(level_num_a).gotoFirst();
  
  while (!end_loop) {
    
    // loop until we reached the marked element
    //
    end_loop = trace_lists_d(level_num_a).isMarkedElement();
    
    // get the current first trace from the list
    //
    trace_lists_d(level_num_a).removeFirst(curr_trace);
    
    // if the trace is inactive then delete it and its backpath if necessary
    //
    if (!curr_trace->isActive()) {
      num_traces_vit_prun_d(curr_trace->getFrame())++;
      Trace::deleteTrace(curr_trace, true);
    }
    
    // else, extend paths and update scores
    //
    else {
      
      curr_vert = curr_trace->getSymbol()->getCentralVertex();
      
      // if this vertex is a terminal vertex then ascend and create the traces
      // at the higher level
      //
      if (curr_vert->isTerm()) {
	
	// set the status to true since we are propagating a trace
	//
	status = true;

	// if we are at the top level then put the trace into the valid
	// hypothesis list
	//
	if (level_num_a == 0) {
	  if (pickup_level_a == (int32)-1) {
	    valid_hyps_d.push(curr_trace);
	    if (valid_hyps_d.top()->getScore() > best_valid_score_d) {
	      best_valid_score_d = valid_hyps_d.top()->getScore();
	    }
	  }
	  else {
	    if (pickup_level_a == 0) {
	      if (current_frame_d == num_frames_d) {
		if (debug_level_d >= Integral::ALL) {
		  printf("to valid: %p\n", curr_trace);
		}
		valid_hyps_d.push(curr_trace);
		if (valid_hyps_d.top()->getScore() > best_valid_score_d) {
		  // update best valid score
		  //
		  best_valid_score_d = valid_hyps_d.top()->getScore();
		}
	      }
	      else {
		pickup_list_a->insertLast(curr_trace);
	      
		// output the debugging information
		//
		if (debug_level_d >= Integral::ALL) {
		  printf("catch\n");
		}
	      }
	    }
	    else {
	      if (current_frame_d == num_frames_d) {
		if (debug_level_d >= Integral::ALL) {
		  printf("to valid: %p\n", curr_trace);
		}
		valid_hyps_d.push(curr_trace);
		if (valid_hyps_d.top()->getScore() > best_valid_score_d) {
		  // update best valid score
		  //
		  best_valid_score_d = valid_hyps_d.top()->getScore();
		}
	      }
	      else {
		if (debug_level_d >= Integral::ALL) {
		  printf("to term: %p\n", curr_trace);
		}
		term_hyps_d.insert(curr_trace);
	      }
	    }
	  }
	}
	
	// else move up one level and create the next set of traces
	//
	else {
	  
	  // create a new trace, 
	  //
	  next_trace = new Trace(*curr_trace);
	  num_traces_gen_d(current_frame_d)++;
	  next_trace->setFrame((ulong)current_frame_d);
	  next_trace->setBackPointer(curr_trace);
	
	  // decrement the history of the new trace
	  //
	  Context* tmp_context = NULL;
	  next_trace->
	    setHistory(history_pool_d.
		       popAndAllocate(next_trace->getHistory(),
				      tmp_context));

	  // the context must exist in the context pool in this case
	  //
	  next_trace->setSymbol(context_pool_d.get(*tmp_context));
	  
	  if (debug_level_d >= Integral::ALL) {
	    printNewPath(next_trace, curr_trace);
	  }
		
	  // propagate all paths we can move to from this higher level trace
	  //
	  tmp_vert = next_trace->getSymbol()->getLastVertex();

	  // if the vertex to propagate from is terminal, generate one trace,
	  // history will be shifted
	  //
	  if (tmp_vert->isTerm()) {

	    // create a temporary trace
	    //
	    tmp_trace = new Trace(*next_trace);
	    num_traces_gen_d(current_frame_d)++;
	    tmp_trace->setFrame((ulong)current_frame_d);

	    // shift the history and set the backpointer
	    //
	    tmp_trace->setSymbol(context_pool_d.
				 shiftAndAllocate(tmp_trace->getSymbol(),
						  tmp_vert));
	    tmp_trace->setBackPointer(next_trace);

	    // add the trace to this level's trace list (or pickup list)
	    //
	    if (pickup_level_a == (int32)-1) {
	      trace_lists_d(level_num_a - 1).insertLast(tmp_trace);
	    }
	    else {
	      GVSnode* tmp_central = tmp_trace->
		getSymbol()->getCentralVertex();
	      if ((level_num_a - 1 != pickup_level_a) || tmp_central->isTerm()
		  ) {
		pickup_list_a->insertLast(tmp_trace);
		if (debug_level_d >= Integral::ALL) {
		  printf("catch\n");
		}
	      }
	      else {
		trace_lists_d(level_num_a - 1).insertLast(tmp_trace);
	      }
	    }

	    // print the debugging information
	    //
	    if (debug_level_d >= Integral::ALL) {
	      printNewPath(tmp_trace, next_trace);
	    }
	  }

	  // vertex to propagate from is not terminal
	  //
	  else {

	    // generate traces to all next vertices
	    //	  
	    for (bool8 more_paths = tmp_vert->gotoFirst(); more_paths;
		 more_paths = tmp_vert->gotoNext()) {
	      
	      // create a new trace, we store the extra trace so we
	      // can later view the scores as they are added. this can
	      // be changed later for efficiency
	      //
	      tmp_trace = new Trace(*next_trace);
	      num_traces_gen_d(current_frame_d)++;
	      tmp_trace->setFrame((ulong)current_frame_d);
	      
	      // update the score with the arc score
	      //
	      GraphArc<SearchNode>* tmp_arc = tmp_vert->getCurr();
	      float32 tmp_score = tmp_trace->getScore() + tmp_arc->getWeight();
	      tmp_trace->setScore(tmp_score);
	    
	      // has the context been previously generated?
	      // if so reuse the context,
	      // if not generate a new context and add it to the context pool
	      //
	      tmp_trace->setSymbol(context_pool_d.
				   shiftAndAllocate(tmp_trace->getSymbol(),
						    tmp_arc->getVertex()));
	      
	      // add the trace to the search node's trace list
	      //
	      Trace* existing_trace = (Trace*)NULL;
	      GVSnode* tmp_vert2 = tmp_trace->getSymbol()->
		getCentralVertex();
	      SearchNode* tmp_node = tmp_vert2->getItem();

	      if (tmp_node->isDummyNode() ||
		  tmp_node->addTrace(tmp_trace, current_frame_d,
				      existing_trace)) {
		
		// update the maximal trace score for the beam pruning
		//
		if (getSearchLevel(level_num_a - 1).useBeam()
		    && (tmp_score > max_frame_scores_d(current_frame_d, 
						       level_num_a - 1)) ) {
		  
		  if (debug_level_d >= Integral::DETAILED) {
		    
		    // print the debugging information
		    //
		    String output(L"fr:");
		    output.concat(current_frame_d);
		    output.concat(L" lv:");
		    output.concat(level_num_a - 1);
		    output.concat(L" max_sc updt (going to higher level YES) ");
		    output.concat(max_frame_scores_d(current_frame_d, 
						     level_num_a - 1));
		    output.concat(L"->");
		    output.concat(tmp_score);
		    Console::put(output);
		  }
		  
		  // update the maximal trace score
		  //
		  max_frame_scores_d.setValue(current_frame_d, level_num_a - 1,
					      tmp_score);
		}
	      
		if (debug_level_d >= Integral::ALL) {
		  printNewPath(tmp_trace, next_trace);
		}
		
		// set the backpointer
		//
		tmp_trace->setBackPointer(next_trace);
		
		// add the trace to the higher level's trace list (or top list)
		//
		if ((pickup_level_a == -1) || tmp_vert->isTerm()) {
		  trace_lists_d(level_num_a - 1).insertLast(tmp_trace);
		}
		else {
		  GVSnode* tmp_central = tmp_trace->getSymbol()->
		    getCentralVertex();
		  if ((level_num_a - 1 != pickup_level_a)
		      || tmp_central->isTerm()) {
		    trace_lists_d(level_num_a - 1).insertLast(tmp_trace);
		  } // end of adding trace to trace list
		  else {
		    pickup_list_a->insertLast(tmp_trace);
		    if (debug_level_d >= Integral::ALL) {
		      printf("catch\n");
		    } // end of debug
		  } // end of adding trace to pickup list
		} // end of adding trace to some list
	      }
	      else {
	      
		// delete the trace since there is existing trace -
		// same history
		//
		if (debug_level_d >= Integral::ALL) { 
		  printDeletedPath(tmp_trace, next_trace);
		}
		num_traces_vit_prun_d(tmp_trace->getFrame())++;
		delete tmp_trace;
	      }
	    }
	  }
	  
	  // delete next_trace if no traces propagated from it have survived
	  //
	  if (next_trace->getRefCount() < 1) {
	    Trace::deleteTrace(next_trace, false);
	    num_traces_vit_prun_d(next_trace->getFrame())++;
	  }
	}
	
	// bump the current trace off of the trace list
	//
	if (curr_trace->getRefCount() < 1) {
	  
	  // if this is at top level, we can not delete them since they
	  // are also referenced by the valid_hyps
	  //
	  if (level_num_a != 0) {
	    num_traces_vit_prun_d(curr_trace->getFrame())++;
	    Trace::deleteTrace(curr_trace, true);
	  }
	}
	trace_lists_d(level_num_a).gotoFirst();
      }
      
      // if this trace is not at the end of a subgraph then keep it on this
      // level
      //
      else {
	
	float32 curr_score = curr_trace->getScore();

	// update the maximal trace score for the beam pruning
	//
	if (getSearchLevel(level_num_a).useBeam()
	    && (curr_score > max_frame_scores_d(current_frame_d, level_num_a))
	    ) {

	  if (debug_level_d >= Integral::DETAILED) {

	    // print the debugging information
	    //
	    String output(L"fr:");
	    output.concat(current_frame_d);
	    output.concat(L" lv:");
	    output.concat(level_num_a);
	    output.concat(L" max_sc updt (going to higher level NO ) ");
	    output.concat(max_frame_scores_d(current_frame_d, level_num_a));
	    output.concat(L"->");
	    output.concat(curr_score);
	    Console::put(output);
	  }

	  // update the maximal trace score
	  //
	  max_frame_scores_d.setValue(current_frame_d, level_num_a,
				      curr_score);
	}
      
	// keep the trace on this level
	//
	trace_lists_d(level_num_a).insertLast(curr_trace);
	trace_lists_d(level_num_a).gotoFirst();
      }
    }
  }
  
  // exit gracefully
  //
  return status;
}


// method: propagateTracesDown
//
// arguments:
//  int32 level_num: (input) the level for which we will propagate traces
//  DoubleLinkedList<Trace>* pickup_list: (output) traces from pickup level
//  int32 pickup_level: (input) level to pick up the traces from
//
// return: logical error status. if no propagations were made then return false
//
// propagate all traces down. any traces that are at the end of a subgraph are
// discarded.
//
bool8 StackSearch::propagateTracesDown(int32 level_num_a,
		      DoubleLinkedList<Trace>* pickup_list_a,
		      int32 pickup_level_a) {

  // define local variables
  //
  Trace* curr_trace = (Trace*)NULL;
  Trace* next_trace = (Trace*)NULL;    
  GraphVertex<SearchNode>* curr_vert = (GraphVertex<SearchNode>*)NULL;
  GraphVertex<SearchNode>* tmp_vert = (GraphVertex<SearchNode>*)NULL;
  float32 score = 0.0;
  SearchSymbol item;
  bool8 status = false;

  // output the debugging information
  //
  if (debug_level_d >= Integral::ALL) {
    String output;        
    output.assign(L"propagating traces down from level: ");
    output.concat(level_num_a);
    output.concat(L" in frame: ");    
    output.concat(current_frame_d);
    Console::put(output);
  }

  // make sure there is at least one trace to propagate
  //
  if (trace_lists_d(level_num_a).length() < 1) {
    return false;
  }

  // switch on whether we are at the lowest level or not
  //
  // if we are not at the lowest level, push traces down the hierarchy -
  // there should be no traces left at this level when we are done
  //
  trace_lists_d(level_num_a).gotoFirst();
  
  if (level_num_a != getNumLevels() - 1) {
    
    // loop over all current traces and create the next level's traces
    //
    while (!trace_lists_d(level_num_a).isEmpty()) {
      
      // get the current vertex in the search graph
      //
      trace_lists_d(level_num_a).removeFirst(curr_trace);

      // delete this trace if it is now inactive
      //
      if (!curr_trace->isActive()) {

	// bump the current trace off of the trace list and delete it
	// and its backpath if necessary
	//
	num_traces_vit_prun_d(curr_trace->getFrame())++;
	Trace::deleteTrace(curr_trace, true);
      }

      // if it is active then propagate it
      //
      else {

	// get the current history and its central vertex
	//
	Context* curr_context = curr_trace->getSymbol();
	curr_vert = curr_context->getCentralVertex();
	
	// if this vertex has a subgraph then descend and create the traces
	// at the next level
	//
	int32 symbol_id = curr_vert->getItem()->getSymbolId();

	if (!curr_vert->isStart() && !curr_vert->isTerm() &&
	    !getSearchLevel(level_num_a).isDummySymbol(symbol_id)) {
	  
	  // we are propagating paths so set status to true
	  //
	  status = true;

	  // get the subgraph's start vertex
	  //
	  SearchLevel& sl_curr = getSearchLevel(level_num_a);
	  SearchLevel& sl_lower = getSearchLevel(level_num_a + 1);

	  // convert context vertices to search symbol indices
	  // we need to do it before hashing in getSubGraphIndex
	  //
	  Context* tmp_context = NULL;
	  curr_context->convert(tmp_context);
	  Ulong* subgr_ind = sl_curr.getSubGraphIndex(*tmp_context);
	  if (tmp_context != NULL) {
	    delete tmp_context;
	  }
	  
	  if (subgr_ind == (Ulong*)NULL) {

	    // throw an error message and exit
	    //
	    String out(L"propagateTracesDown - no subgraph for context: ");
	    curr_context->debug(L"undefined context");
	    curr_trace->getSymbol()->print();
	    Console::put(out);
	    return Error::handle(name(), L"propagateTracesDown", Error::ARG,
				 __FILE__, __LINE__);	    
	  }

	  int32 central_pos =
	    getSearchLevel(level_num_a + 1).getRightContext() + 1;
	  int32 left_context_length =
	    getSearchLevel(level_num_a + 1).getLeftContext();
	  int32 total_context_length = 
	    left_context_length + central_pos;
	  
	  // get the subgraph start vertex
	  // 
	  tmp_vert = (sl_lower.getSubGraph((int32)*subgr_ind)).getStart();
	  
	  // initialize left vertices of the context
	  //
	  Context left_context(total_context_length, central_pos);
	  for (int32 i = 0; i < left_context_length; i++) {
	    left_context.assignAndAdvance((ulong)tmp_vert);
	  }

	  // loop over all paths in the subgraph and create new traces in the
	  // next level
	  //
	  Context* symbol = context_pool_d.get(left_context);
	  for (bool8 more_paths = tmp_vert->gotoFirst(); more_paths;
	       more_paths = tmp_vert->gotoNext()) {

	    // initialize central vertex of the context
	    //
	    GraphArc<SearchNode>* tmp_arc = tmp_vert->getCurr();
	    GraphVertex<SearchNode>* target_vertex = tmp_arc->getVertex();
	    Context* left_center_context =
	      context_pool_d.shiftAndAllocate(symbol, target_vertex);
	    
	    // generate list of all possible right contexts
	    //
	    DoubleLinkedList<Context> right_context_list;
	    right_context_list.setAllocationMode(DstrBase::USER);
	    int32 depth = getSearchLevel(level_num_a + 1).getRightContext();
	    generateRightContexts(right_context_list, left_center_context,
				  depth);

	    // output the debugging information
	    //
	    if (debug_level_d > Integral::ALL) {
	      String output(L"Generated contexts for depth");
	      output.concat(depth);
	      Console::put(output);
	      int32 num = 0;
	      for (bool8 more_items = right_context_list.gotoFirst();
		  more_items;
		  more_items = right_context_list.gotoNext()) {
		num++;
		String out(L" ");
		out.concat(num);
		out.concat(L":");
		String cont;
		right_context_list.getCurr()->print();
		out.concat(cont);
		Console::put(out);
	      }
	    }

	    // generate a new trace for each right context
	    //
	    for (bool8 more_items = right_context_list.gotoFirst();
		 more_items;
		 more_items = right_context_list.gotoNext()) {
	      
	      // create a new trace
	      //
	      next_trace = new Trace(*curr_trace);
	      num_traces_gen_d(current_frame_d)++;
	      next_trace->setFrame((ulong)current_frame_d);
	      
	      // update the score with the arc score
	      //
	      next_trace->setScore(next_trace->getScore() +
				   tmp_arc->getWeight());
	    
	      next_trace->setSymbol(right_context_list.getCurr());

	      // has the history been previously generated?
	      // if so reuse the history,
	      // if not generate a new history and add it to the
	      // history pool
	      //
	      next_trace->setHistory(history_pool_d.pushAndAllocate(next_trace->getHistory(), curr_trace->getSymbol()));
	      
	      // add the trace to the search node's trace list.
	      //
	      Trace* existing_trace = (Trace*)NULL;
	      SearchNode* next_node = next_trace->
		getSymbol()->getCentralVertex()->getItem();
	      
	      if (next_node->addTrace(next_trace, current_frame_d,
				      existing_trace)) {
		
		// set the backpointer
		//
		next_trace->setBackPointer(curr_trace);

		if (debug_level_d >= Integral::ALL) {
		  printNewPath(next_trace, curr_trace);
		}
		
		// add the trace to the next level's trace list
		//
		if (pickup_level_a == (int32)-1) {
		  trace_lists_d(level_num_a + 1).insertLast(next_trace);
		}
		else {
		  if (pickup_level_a != level_num_a + 1) {
		    trace_lists_d(level_num_a + 1).insertLast(next_trace);
		  }
		  else {
		    pickup_list_a->insertLast(next_trace);
		    if (debug_level_d >= Integral::ALL) {
		      printf("catch\n");
		    } // end of debug
		  } // end of adding trace to trace_list
		} // end of adding trace to any list
	      } // end of if the trace was succesfully added
	      
	      // delete the trace since there is existing trace -
	      // same history
	      //
	      else {
		if (debug_level_d >= Integral::ALL) { 
		  printDeletedPath(next_trace, curr_trace);
		}
		num_traces_vit_prun_d(next_trace->getFrame())++;
		delete next_trace;
	      }
	    } // end of 

	    // clean the right contexts list
	    //
	    right_context_list.clear();
	  }
	    
	  // bump the current trace off of the trace list
	  //
	  if (curr_trace->getRefCount() < 1) {
	    Trace::deleteTrace(curr_trace, true);
	  }
	  trace_lists_d(level_num_a).gotoFirst();
	}
	
	// else if there is no subgraph for this node, we extend the search
	// along this level. for now, we don't allow a level to completely
	// skip sub-levels so if we get to the end of the graph at this level
	// we simply stop
	//
	else {
	  bool8 more_paths = curr_vert->gotoFirst();
	  if (more_paths) {
	    status = true;
	  }
	  for ( ; more_paths; more_paths = curr_vert->gotoNext()) {
	    
	    // create a new trace
	    //
	    next_trace = new Trace(*curr_trace);
	    num_traces_gen_d(current_frame_d)++;
	    next_trace->setFrame((ulong)current_frame_d);
	    
	    // update the score with the arc score
	    //
	    GraphArc<SearchNode>* tmp_arc = curr_vert->getCurr();
	    score = next_trace->getScore() + tmp_arc->getWeight(); 	    
	    next_trace->setScore(score);
	    
	    // update the history with the next node at this level
	    //
	    GraphVertex<SearchNode>* target_vertex = tmp_arc->getVertex();
	    next_trace->setSymbol(context_pool_d.
				  shiftAndAllocate(curr_trace->getSymbol(),
						   target_vertex));
	    
	    // add the trace to the search node's trace list to make sure we
	    // propagate it before exiting
	    //
	    Trace* existing_trace = (Trace*)NULL;	    
	    SearchNode* next_node =
	      next_trace->getSymbol()->getCentralVertex()->getItem();

	    if (next_node->addTrace(next_trace, current_frame_d,
				    existing_trace)) {
	      
	      // set the backpointer
	      //
	      next_trace->setBackPointer(curr_trace);

	      // add the trace to this level's trace list
	      //
	      trace_lists_d(level_num_a).insertLast(next_trace);
	      if (debug_level_d >= Integral::ALL) {
		printNewPath(next_trace, curr_trace);
	      }
	    }
	    else {

	      // delete the trace since there is existing trace - same history
	      //
	      if (debug_level_d >= Integral::ALL) {
		printDeletedPath(next_trace, curr_trace);
	      }	  	      
	      num_traces_vit_prun_d(next_trace->getFrame())++;
	      delete next_trace;
	    }
	  }
	  
	  // bump the current trace off of the trace list
	  //
	  if (curr_trace->getRefCount() < 1) {
	    num_traces_vit_prun_d(curr_trace->getFrame())++;
	    Trace::deleteTrace(curr_trace, true);
	  }
	  trace_lists_d(level_num_a).gotoFirst();
	}
      }
    }
  }
  
  // exit gracefully
  //
  return status;
}

// method: beamPruneTrace
//
// arguments:
//  int32 level_num: (input) the level for which we will propagate traces
//
// return: logical error status.
//
bool8 StackSearch::beamPruneTrace(int32 level_num_a) {

  // define local variables
  //
  Trace* curr_trace = (Trace*)NULL;
  float32 threshold = max_frame_scores_d(current_frame_d, level_num_a)
    - getSearchLevel(level_num_a).getBeamThreshold();
  float32 curr_score;

  int32 pruned_traces = 0;
  int32 orig_length = trace_lists_d(level_num_a).length();
  
  // loop over all active traces at this level
  //
  bool8 more_traces = trace_lists_d(level_num_a).gotoFirst();
  
  while (more_traces) {

    curr_trace = trace_lists_d(level_num_a).getCurr();
    curr_score = curr_trace->getScore();

    if (curr_score > max_frame_scores_d(current_frame_d, level_num_a)) {
      return Error::handle(name(), L"beamPrune -- bad maximal score!",
			     Error::ARG, __FILE__, __LINE__);
      }
      
    // prune the traces with too low score
    //
    if (curr_score < threshold)  {

      if (debug_level_d >= Integral::DETAILED) {
	String output(L"score is SMALLER than threshold:");
	output.concat(curr_score);
	output.concat(L" trace:" );
	output.concat(curr_trace);
	Console::put(output);
      }

      if (trace_lists_d(level_num_a).isLast()) {
	more_traces = false;
      }
	  
      // bump the current trace off of the trace list
      //
      trace_lists_d(level_num_a).remove();
      
      if (curr_trace->getRefCount() < 1) {
	Trace::deleteTrace(curr_trace, true);
	num_traces_beam_prun_d(curr_trace->getFrame())++;
      }
      pruned_traces++;
    }
    
    else {
      if (debug_level_d >= Integral::DETAILED) {
	String output(L"score is NOT smaller than threshold:");
	output.concat(curr_score);
	output.concat(L" trace:" );
	output.concat(curr_trace);
	Console::put(output);
      }
      more_traces = trace_lists_d(level_num_a).gotoNext();
    }
    
  }

  if (debug_level_d >= Integral::DETAILED) {

    // print the debugging information
    //
    String output(L"Pf:");
    output.concat(current_frame_d);
    output.concat(L" lv:");
    output.concat(level_num_a);
    output.concat(L" orig_tr:");
    output.concat(orig_length);
    output.concat(L" PR_tr*");
    output.concat(pruned_traces);
    output.concat(L" res_tr:");
    output.concat(trace_lists_d(level_num_a).length());
    output.concat(L" thr:");
    output.concat(threshold);
    output.concat(L" max_sc:");
    output.concat(max_frame_scores_d(current_frame_d, level_num_a));
    output.concat(L"\n");
    Console::put(output);
  }
  
  // exit gracefully
  //
  return true;
}