// file: $isip/class/search/StackSearch/ssrch_15.cc
// version: $Id: ssrch_15.cc 9144 2003-05-16 21:59:27Z jelinek $
//

// isip include files
//
#include "StackSearch.h"
#include "SupportVectorModel.h"

// method: extend
//
// arguments:
//  DoubleLinkedList<Trace>& stack_traces: (input) stack level traces
//  int32 start_frame: (input) start frame
//  
// return: logical error status
//
// extend all word level hypotheses in hypothesis_list by one word
//
bool8 StackSearch::extend(DoubleLinkedList<Trace>& stack_traces_a,
			    int32 start_frame_a) {

  // initialize stack level traces and time synchronously extend all
  // of them by one symbol at the stack level
  //
  setTraces(stack_traces_a, stack_level_d);
  setFrame(start_frame_a);

  // output debugging information
  //
  if (debug_level_d >= Integral::DETAILED) {
    String out(L"\nExtending from start frame ");
    out.concat(start_frame_a); out.concat(L", first propagation");
    Console::put(out);
  }

  // first propagate all the stack level traces to the state level
  // it may include propagation up if the model is skipped
  //
  for (int32 i = stack_level_d; i < getNumLevels() - 1; i++) {
    propagateTracesDown(i);
  }

  // for all next frames generate one word extensions of the
  // hypotheses from the current frame's stack.
  // each extension is moved to the stack corresponding to frame
  // when extension was generated.
  // note that current frame is changed during the model evaluation.
  //
  while ((pathsRemainTrace()) && (current_frame_d < num_frames_d)) {

    // evaluate trace models
    //
    // if the stack level is lowest level, pick up state traces. in
    // SEGMENT evaluation mode pick up only traces that are not
    // looping
    //
    DoubleLinkedList<Trace> pickup_traces(DstrBase::USER);
    evaluateTraceModels(&pickup_traces, stack_level_d);

    // output the debugging information
    //
    if (debug_level_d >= Integral::ALL) {
      String out(L"\n  Start propagate traces one frame from ");
      out.concat(current_frame_d);
      out.concat(L" to ");
      out.concat(current_frame_d + (int32)1);
      out.concat(L" \n\n");
      Console::put(out);
    }
    
    // propagate traces one frame forward and pick up traces from the
    // stack level
    //
    propagateTraces(&pickup_traces, stack_level_d);

    // output debugging information
    //
    if (debug_level_d >= Integral::DETAILED) {
      String out(L"\n  after tr. propagation FROM frame ");
      out.concat(start_frame_a);
      out.concat(L" TO frame ");
      out.concat(current_frame_d);
      out.concat(L" # stack traces: ");
      out.concat(pickup_traces.length());
      if (decoding_mode_d == GLOBAL_STACK) {
	out.concat(L" # global hyps: ");
	out.concat(global_stack_d.length());
      }
      out.concat(L" \n\n");
      Console::put(out);
    }

    // insert valid top level traces into stack
    //
    bool8 more = pickup_traces.gotoFirst();
    while (more) {

      // get the trace from list
      //
      Trace* current_trace = pickup_traces.getCurr();
      float32 current_score = current_trace->getScore();
      
      // if trace is not active or worse than current best valid
      // hypotheses, delete it recursively
      //
      if (!current_trace->isActive()) {
	num_traces_vit_prun_d(current_trace->getFrame())++;
	Trace::deleteTrace(current_trace, true);
      }
      else if (current_score < best_valid_score_d) {
	Trace::deleteTrace(current_trace, true);
      }

      // since the trace is active, insert hypothesis into
      // corresponding stack
      //
      else {

	// create temporary hypothesis
	//
	Hypothesis tmp_hypothesis(current_trace, current_score);

	// find the stack for the hypothesis
	//
	Stack<Hypothesis>* stack = stacks_d(current_frame_d)
	  .get(*current_trace);

	// if there is no stack for this hypothesis yet, create one
	// and put the current hypothesis into it
	//
	if (stack == NULL) {
	  Hypothesis tmp_hyp(current_trace, current_score);
	  Stack<Hypothesis> tmp_stack;
	  tmp_stack.push(&tmp_hyp);
	  stacks_d(current_frame_d).insert(*current_trace, &tmp_stack);
	  if (decoding_mode_d == GLOBAL_STACK) {
	    if (global_stack_d.getCapacity() == global_stack_d.length()) {
	      global_stack_d.setCapacity(2 * global_stack_d.length());
	    }
	    global_stack_d.push(&tmp_hypothesis);
	  }
	}
	
	// if there are less then n-best hypotheses in the stack,
	// insert also current hypotheses
	// 
	else if (stack->length() < n_best_d) {
	  stack->push(&tmp_hypothesis);
	  if (decoding_mode_d == GLOBAL_STACK) {
	    if (global_stack_d.getCapacity() == global_stack_d.length()) {
	      global_stack_d.setCapacity(2 * global_stack_d.length());
	    }
	    global_stack_d.push(&tmp_hypothesis);
	  }
	}

	// if there are n-best hypotheses in the stack,
	// keep the better one and delete worse
	//
	else if (stack->length() == n_best_d) {
	  if (current_score > stack->peek()->getScore()) {
	    Hypothesis worse;
	    stack->pop(&worse);
	    worse.getTrace()->setActive(false);
	    stack->push(&tmp_hypothesis);
	    if ((decoding_mode_d == GLOBAL_STACK)) {
	      if (global_stack_d.getCapacity() == global_stack_d.length()) {
		global_stack_d.setCapacity(2 * global_stack_d.length());
	      }
	      global_stack_d.push(&tmp_hypothesis);
	    }
	  }
	  else {
	    num_traces_vit_prun_d(current_trace->getFrame())++;
	    Trace::deleteTrace(current_trace, true);
	  }
	}

	// update max stack score if it is exceeded
	//
	if (current_score > max_stack_scores_d(current_frame_d)) {
	  max_stack_scores_d(current_frame_d) = current_score;
	}
      } // end of else - active hypothesis was inserted on the stack
      more = pickup_traces.gotoNext();
    } // end of the loop over pickup traces (all are placed on the stack)
  } // end of loop over all end frames

  // remove state level traces after the last frame, because the will
  // not be propagated any more
  //
  DoubleLinkedList<Trace> state_traces(DstrBase::USER);
  getTraces(state_traces, getNumLevels() - 1);
  Trace* current_trace = NULL;
  while (state_traces.removeFirst(current_trace)) {
    Trace::deleteTrace(current_trace, true);
  }

  // after the end of one time-synchronous stack extension,
  // clean the trace lists of all search nodes
  // (later we can clean only those actually visited nodes)
  //
  clearSearchNodesTraceLists();

  // exit gracefully
  //
  return true;
}

// method: decode
//
// arguments: none
//  
// return: logical error status
//
// this is the core decoding method
//
bool8 StackSearch::decode() {
  
  // check whether search levels and statistical models are loaded
  //
  if (getNumLevels() == 0) {
    return Error::handle(name(), L"decode - load lang model first",
			 Error::ARG, __FILE__, __LINE__);
  }
  if (getSearchLevel(getNumLevels() - 1)
      .getStatisticalModels().length() == 0) {
        return Error::handle(name(), L"decode - load stat models first",
			 Error::ARG, __FILE__, __LINE__);
  }
  if ((decoding_mode_d == VITERBI) && (stack_level_d != int32(-1))) {
    return Error::handle(name(),
			 L"decode - stack level must be -1 in VITERBI mode",
			 Error::ARG, __FILE__, __LINE__);
  }
  if (((decoding_mode_d == MULTI_STACK) || (decoding_mode_d == GLOBAL_STACK))
      && (stack_level_d == int32(-1))) {
    return Error::handle(name(),
			 L"decode - stack level can be -1 only in VITERBI mode",
			 Error::ARG, __FILE__, __LINE__);
  }
  if (decoding_mode_d == GLOBAL_STACK) {
    global_stack_d.setCapacity(DEF_GLOB_STACK_CAPACITY);
  }

  // transform skip transitions
  //
  transformSkipModels();

  // set evaluation mode to FRAME_EVAL (SVM) or SEGMENT_EVAL (GAUSSIAN)
  //
  determineEvaluationMode();

  // initialize maximal trace and stack scores
  //
  initSizes(num_frames_d);

  // this will set current frame to 0, clear trace lists, search nodes
  // and generate initial trace
  //
  initializeLinearDecoder();

  // compute SVM features if needed
  //
  if (evaluation_mode_d == SEGMENT_EVAL) {
    normalizeFeatures();
    SupportVectorModel::generateSvmFeatures(svm_features_d, features_d);

    // check the score normalization
    //
    if ((score_normalization_d == NONE) &&
	!user_requested_no_score_normalization_d) {
      score_normalization_d = DURATION;
      Console::put(L"Warning - score normalization changed from default NONE to DURATION because we are in SEGMENT_EVAL mode!");
    }
  }

  if (decoding_mode_d == GLOBAL_STACK) {
    
    // list for of the stack traces generated
    //
    DoubleLinkedList<Trace> pickup_traces(DstrBase::USER);

    // propagate down to get traces at the stack level
    //
    for (int32 i = 0; i < stack_level_d; i++) {
      propagateTracesDown(i);
    }
    getTraces(pickup_traces, stack_level_d);

    // output debugging information
    //
    if (debug_level_d >= Integral::ALL) {
      String out(L"# of pickup traces in the first frame: ");
      out.concat(pickup_traces.length());
      Console::put(out);
    }

    // put all initial traces on the stack 0 and possibly on the
    // global stack
    //
    bool8 more = pickup_traces.gotoFirst();
    while (more) {

      // create temporary hypothesis for each trace
      //
      Trace* curr_trace = pickup_traces.getCurr();
      float32 score = curr_trace->getScore();
      Hypothesis tmp_hypothesis(curr_trace, score);

      // create stack for each word
      //
      Stack<Hypothesis> tmp_stack;
      tmp_stack.push(&tmp_hypothesis);

      // insert the stack into hashtable
      //
      stacks_d(0).insert(*curr_trace, &tmp_stack);
      if (decoding_mode_d == GLOBAL_STACK) {
	if (global_stack_d.getCapacity() == global_stack_d.length()) {
	  global_stack_d.setCapacity(2 * global_stack_d.length());
	}
	global_stack_d.push(&tmp_hypothesis);
      }
      if (debug_level_d >= Integral::ALL) {
	printf("initial catch to stack\n");
      }
      more = pickup_traces.gotoNext();
    }

    if (decoding_mode_d == GLOBAL_STACK) {

      // propagate word hypotheses from all stacks frame by frame
      //
      int32 num_pop = 0;
    
      while (!(global_stack_d.isEmpty() ||
	       best_valid_score_d > global_stack_d.top()->getScore())
	     ) {

	// pop hypotheses from the global stack one by one
	//
	Hypothesis top_hyp;
	global_stack_d.pop(&top_hyp);
	num_pop++;
	//
	// do not delete hyp from its frame stack - keep for Viterbi pruning

	Trace* top_trace = top_hyp.getTrace();
	int32 top_frame = top_trace->getFrame();

	if (top_trace->isActive()) {
	  if (debug_level_d >= Integral::BRIEF) {
	    String out(L"\n    One pop #"); out.concat(num_pop);
	    out.concat(L" of glob size: ");
	    int32 len = global_stack_d.length(); out.concat(len);
	    out.concat(L" sc: ");
	    float32 score = top_hyp.getScore(); out.concat(score);
	    out.concat(L" fr: "); out.concat(top_frame);
	    if (debug_level_d >= Integral::ALL) {
	      out.concat(L" tr: ");
	      out.concat(top_trace);
	    }
	    out.concat(L" pos:");
	    History* top_history = top_trace->getHistory();
	    bool8 more = top_history->gotoFirst();
	    while (more) {
	      out.concat(L" ");
	      top_history->peek()->print(out);
	      more = top_history->gotoNext();
	    }
	    out.concat(L" ");
	    top_trace->getSymbol()->print(out);
	    top_history->gotoFirst();
	    out.concat(L" bvs: ");
	    out.concat(best_valid_score_d);
	    Console::put(out);
	  }

	  DoubleLinkedList<Trace> top_trace_list(DstrBase::USER);
	  top_trace_list.insertLast(top_trace);

	  // extend this hypothesis to the end frame
	  //
	  extend(top_trace_list, top_frame);
	} // end of if top_trace is active
	else {
	  num_traces_vit_prun_d(top_trace->getFrame())++;
	  Trace::deleteTrace(top_trace, true);
	} // end of if top_trace is not active
      } // global stack is empty or best valid is better than top 

      if (global_stack_d.isEmpty()) {
	return Error::handle(name(), L"decode - nothing in global stack",
			     Error::ARG,
			     __FILE__, __LINE__);
      } // end of error report
    } // end of GLOBAL_STACK decoding mode
  
    else if (decoding_mode_d == MULTI_STACK) {

      // propagate word hypotheses from all stacks frame by frame
      //
      for (int32 curr_frame = 0; curr_frame < features_d.length();
	   curr_frame++) {

	// pop word hypotheses from the stack of the current frame one by one
	//
	DoubleLinkedList<Trace> valid_traces(DstrBase::USER);
	if (debug_level_d >= Integral::BRIEF) {
	  String out(L"\n    Popping the stack ");
	  out.concat(curr_frame);
	  out.concat(L" of size ");
	  out.concat(stacks_d(curr_frame).getNumItems());
	  Console::put(out);
	}

	Vector< Stack<Hypothesis> > stacks;
	stacks_d(curr_frame).values(stacks);
	for (int32 i = 0; i < stacks.length(); i++) {
	
	  Stack<Hypothesis>* stack = &stacks(i);
	  Hypothesis curr_hypothesis;
	  while (stack->pop(&curr_hypothesis)) {
	  
	    Trace* curr_trace = curr_hypothesis.getTrace();
	    valid_traces.insertLast(curr_trace);
	  
	    if (debug_level_d > Integral::ALL) {
	      float32 curr_score = curr_hypothesis.getScore();
	      String out(L" ");
	      out.concat(curr_score);
	      out.concat(L" ");
	      out.concat((float32)max_stack_scores_d(curr_frame));
	      Console::put(out);
	    }
	  }
	}

	// extend all traces from stack
	//
	extend(valid_traces, curr_frame);
      }

      // get hypotheses
      //
      DoubleLinkedList<Trace> traces(DstrBase::USER);
      Hypothesis curr_hypothesis;

      Vector< Stack<Hypothesis> > stacks;
      stacks_d(features_d.length()).values(stacks);

      for (int32 i = 0; i < stacks.length(); i++) {
	Stack<Hypothesis>* stack = &stacks(i);
	while (stack->pop(&curr_hypothesis)) {
	  Trace* curr_trace = curr_hypothesis.getTrace();
	  traces.insertLast(curr_trace);
	}
      }
      setTraces(traces, stack_level_d);
    }
  }

  // if decoding mode is VITERBI
  //
  else if (decoding_mode_d == VITERBI) {

    // loop over all frames
    //
    for (int32 i = 0; i < num_frames_d; i++) {

      // decode one frame of data
      //
      if (debug_level_d >= Integral::BRIEF) {
	String out(L"FRAME: ");
	out.concat(i);
	Console::put(out);
      }
      if (!traverseTraceLevels()) {
	break;
      }
    }
  }

  // exit gracefully
  //
  return true;
}