#include "AI_Player_Impl.h"
#include <cmath>

using namespace std;

MOVE_ID AI_Player_Impl::calculate_next_move()
{
	MQV_Map& mqv_map = _player->_qvalues->get_qvalues();
	AI_StateInspector* si = _player->_state_inspector;
	AI_History* history = _player->_history;
	vector<MOVE_ID>& moves = _player->_moves;
	
	double lambda = _player->_lambda;
	double dr = _player->_dr;
	int H = _player->_H;
	
	MOVE_ID next_move = si->get_state_var(USER_MOVE_CHOICE);
	if (next_move == NULL_MOVE)
	{
		MQV_iter it = mqv_map.begin();
		MQV_iter end = mqv_map.end();

		double total_weight = 0;
		for (int i=0; i < moves.size(); i++)
		{
			if ((it == end) || (*it).first > moves[i])
				_weights[i] = 1;
			else {
				_weights[i] = exp((*it).second*lambda);
				++it;
			}
			total_weight += _weights[i];
		}

		double random = rand() / (double) INT_MAX;
		double p = _weights[0] / total_weight;
		int move_index = 0;
		
		while(random > p) p += _weights[++move_index] / total_weight;
		next_move = moves[move_index];
	}

	History_Info* hist_info = new History_Info(next_move);
	hist_info->obj_delta = si->get_state_var(OBJ_DELTA);
	hist_info->obj_delta += si->get_state_var(USER_REINFORCEMENT);
	si->get_state(hist_info->state);
	history->push_front(hist_info);

	if (history->size() == H+1)
	{
		hist_info = history->back();
		history->pop_back();
		hist_iter it = history->end();

		MOVE_ID move = hist_info->move;
		double alpha = 1;
		double nq = 0;
		while (it != history->begin())
		{
			--it;
			nq += alpha*(*it)->obj_delta;
			if ((*it)->move != move)
			{
				move = (*it)->move;
				alpha *= dr;
			}
		}

		_player->_qvalues->update_qvalue(hist_info->state, hist_info->move, nq);
		delete hist_info;
	}

	return next_move;
}