From b6d663e13ff8be5a5b2bad52815443c21d245b8e Mon Sep 17 00:00:00 2001 From: Fabian Oehlmann Date: Thu, 9 Jan 2014 17:05:29 +0000 Subject: [PATCH] made replacing traces available --- src/ats/plugin_ats_ril.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/ats/plugin_ats_ril.c b/src/ats/plugin_ats_ril.c index fe8f43a2d..f96ebb91d 100755 --- a/src/ats/plugin_ats_ril.c +++ b/src/ats/plugin_ats_ril.c @@ -89,7 +89,7 @@ enum RIL_Select enum RIL_E_Modification { - RIL_E_UPDATE, + RIL_E_DISCOUNT, RIL_E_ZERO, RIL_E_ACCUMULATE, RIL_E_REPLACE @@ -125,6 +125,11 @@ struct RIL_Learning_Parameters */ double lambda; + /** + * Whether to accumulate or replace eligibility traces + */ + enum RIL_E_Modification eligibility_trace_mode; + /** * Softmax action-selection temperature */ @@ -615,7 +620,7 @@ agent_modify_eligibility (struct RIL_Peer_Agent *agent, case RIL_E_REPLACE: agent->E[action][i] = (agent->envi->global_discount_variable * agent->envi->parameters.lambda * agent->E[action][i]) > feature[i] ? agent->E[action][i] : feature[i]; //TODO make replacing traces available break; - case RIL_E_UPDATE: + case RIL_E_DISCOUNT: agent->E[action][i] *= agent->envi->global_discount_variable * agent->envi->parameters.lambda; break; case RIL_E_ZERO: @@ -1218,7 +1223,7 @@ agent_select_egreedy (struct RIL_Peer_Agent *agent, double *state) action = agent_get_action_best(agent, state); if (RIL_ALGO_Q == agent->envi->parameters.algorithm) { - agent_modify_eligibility(agent, RIL_E_UPDATE, NULL, action); + agent_modify_eligibility(agent, RIL_E_DISCOUNT, NULL, action); } return action; } @@ -1265,7 +1270,7 @@ agent_select_softmax (struct RIL_Peer_Agent *agent, double *state) if (RIL_ALGO_Q == agent->envi->parameters.algorithm) { if (i == a_max) - agent_modify_eligibility(agent, RIL_E_UPDATE, NULL, i); + agent_modify_eligibility(agent, RIL_E_DISCOUNT, NULL, i); else agent_modify_eligibility(agent, RIL_E_ZERO, NULL, -1); } @@ -1321,7 +1326,7 @@ agent_step (struct RIL_Peer_Agent *agent) //updates weights with selected action (on-policy), if not first step agent_update_weights (agent, reward, s_next, a_next); } - agent_modify_eligibility (agent, RIL_E_UPDATE, s_next, a_next); + agent_modify_eligibility (agent, RIL_E_DISCOUNT, s_next, a_next); break; case RIL_ALGO_Q: @@ -1337,7 +1342,7 @@ agent_step (struct RIL_Peer_Agent *agent) GNUNET_assert(RIL_ACTION_INVALID != a_next); - agent_modify_eligibility (agent, RIL_E_ACCUMULATE, s_next, a_next); + agent_modify_eligibility (agent, agent->envi->parameters.eligibility_trace_mode, s_next, a_next); // GNUNET_log (GNUNET_ERROR_TYPE_INFO, "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n", // agent->step_count, @@ -2008,6 +2013,14 @@ libgnunet_plugin_ats_ril_init (void *cls) { solver->simulate = GNUNET_NO; } + if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(env->cfg, "ats", "RIL_REPLACE_TRACES")) + { + solver->parameters.eligibility_trace_mode = RIL_E_REPLACE; + } + else + { + solver->parameters.eligibility_trace_mode = RIL_E_ACCUMULATE; + } env->sf.s_add = &GAS_ril_address_add; env->sf.s_address_update_property = &GAS_ril_address_property_changed; -- 2.25.1