X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fats%2Fplugin_ats_ril.c;h=6e55f3a1c5805aee89ebc570f8f5be40039af2a7;hb=c29a8124f885f28f287e91ce7a0dabcdd6b17d50;hp=2c268d8ba8e93011e7d8ce79d699becff19eeeb9;hpb=fbfd23141c29c3cb91a72261976963fd497407e7;p=oweals%2Fgnunet.git diff --git a/src/ats/plugin_ats_ril.c b/src/ats/plugin_ats_ril.c old mode 100755 new mode 100644 index 2c268d8ba..6e55f3a1c --- a/src/ats/plugin_ats_ril.c +++ b/src/ats/plugin_ats_ril.c @@ -1,21 +1,21 @@ /* This file is part of GNUnet. - (C) 2011 Christian Grothoff (and other contributing authors) + Copyright (C) 2011-2014 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + Affero General Public License for more details. - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + + SPDX-License-Identifier: AGPL3.0-or-later */ /** @@ -24,28 +24,40 @@ * @author Fabian Oehlmann * @author Matthias Wachs */ -#include "plugin_ats_ril.h" - -#define LOG(kind,...) GNUNET_log_from (kind, "ats-ril",__VA_ARGS__) +#include "platform.h" +#include +#include +#include "gnunet_ats_plugin.h" +#include "gnunet-service-ats_addresses.h" -#define MIN_BW ntohl (GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__) -#define RIL_ACTION_INVALID -1 -#define RIL_INTERVAL_EXPONENT 10 -#define RIL_UTILITY_MAX (double) GNUNET_ATS_MaxBandwidth -#define RIL_DEFAULT_STEP_TIME_MIN GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 500) -#define RIL_DEFAULT_STEP_TIME_MAX GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 3000) -#define RIL_DEFAULT_ALGORITHM RIL_ALGO_SARSA -#define RIL_DEFAULT_DISCOUNT_BETA 1.0 -#define RIL_DEFAULT_DISCOUNT_GAMMA 0.5 -#define RIL_DEFAULT_GRADIENT_STEP_SIZE 0.1 -#define RIL_DEFAULT_TRACE_DECAY 0.5 -#define RIL_DEFAULT_EXPLORE_RATIO 0.1 -#define RIL_DEFAULT_DIVISOR 10 -#define RIL_DEFAULT_GLOBAL_REWARD_SHARE 0.5 +#define LOG(kind,...) GNUNET_log_from (kind, "ats-ril",__VA_ARGS__) -#define RIL_INC_DEC_STEP_SIZE 1 +#define RIL_MIN_BW (5 * ntohl (GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__)) +#define RIL_MAX_BW GNUNET_ATS_MaxBandwidth + +#define RIL_ACTION_INVALID -1 +#define RIL_INTERVAL_EXPONENT 10 +#define RIL_UTILITY_DELAY_MAX 1000 + +#define RIL_DEFAULT_STEP_TIME_MIN GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 200) +#define RIL_DEFAULT_STEP_TIME_MAX GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 2000) +#define RIL_DEFAULT_ALGORITHM RIL_ALGO_SARSA +#define RIL_DEFAULT_SELECT RIL_SELECT_SOFTMAX +#define RIL_DEFAULT_WELFARE RIL_WELFARE_NASH +#define RIL_DEFAULT_DISCOUNT_BETA 0.6 +#define RIL_DEFAULT_DISCOUNT_GAMMA 0.5 +#define RIL_DEFAULT_GRADIENT_STEP_SIZE 0.01 +#define RIL_DEFAULT_TRACE_DECAY 0.5 +#define RIL_DEFAULT_EXPLORE_RATIO 1 +#define RIL_DEFAULT_EXPLORE_DECAY 0.95 +#define RIL_DEFAULT_RBF_DIVISOR 50 +#define RIL_DEFAULT_TEMPERATURE 0.1 +#define RIL_DEFAULT_TEMPERATURE_DECAY 1 + +#define RIL_INC_DEC_STEP_SIZE 1 +#define RIL_NOP_DECAY 0.5 /** * ATS reinforcement learning solver @@ -61,16 +73,16 @@ */ enum RIL_Action_Type { - RIL_ACTION_NOTHING = -1, - RIL_ACTION_BW_IN_DBL = -2, //TODO! put actions back + RIL_ACTION_NOTHING = 0, + RIL_ACTION_BW_IN_DBL = -2, //TODO? Potentially add more actions RIL_ACTION_BW_IN_HLV = -3, - RIL_ACTION_BW_IN_INC = 0, - RIL_ACTION_BW_IN_DEC = 1, + RIL_ACTION_BW_IN_INC = 1, + RIL_ACTION_BW_IN_DEC = 2, RIL_ACTION_BW_OUT_DBL = -4, RIL_ACTION_BW_OUT_HLV = -5, - RIL_ACTION_BW_OUT_INC = -6, - RIL_ACTION_BW_OUT_DEC = -7, - RIL_ACTION_TYPE_NUM = 1 + RIL_ACTION_BW_OUT_INC = 3, + RIL_ACTION_BW_OUT_DEC = 4, + RIL_ACTION_TYPE_NUM = 5 }; enum RIL_Algorithm @@ -79,9 +91,21 @@ enum RIL_Algorithm RIL_ALGO_Q = 1 }; +enum RIL_Select +{ + RIL_SELECT_SOFTMAX = 0, + RIL_SELECT_EGREEDY = 1 +}; + +enum RIL_Welfare +{ + RIL_WELFARE_NASH, + RIL_WELFARE_EGALITARIAN +}; + enum RIL_E_Modification { - RIL_E_SET, + RIL_E_DECAY, RIL_E_ZERO, RIL_E_ACCUMULATE, RIL_E_REPLACE @@ -117,20 +141,55 @@ struct RIL_Learning_Parameters */ double lambda; + /** + * Whether to accumulate or replace eligibility traces + */ + enum RIL_E_Modification eligibility_trace_mode; + + /** + * Initial softmax action-selection temperature + */ + double temperature_init; + + /** + * Softmax action-selection temperature + */ + double temperature; + + /** + * Decay factor of the temperature value + */ + double temperature_decay; + + /** + * Which measure of social welfare should be used + */ + enum RIL_Welfare social_welfare; + /** * State space divisor */ - unsigned long long int divisor; + unsigned long long rbf_divisor; + + /** + * Action selection strategy; + */ + enum RIL_Select select; + + /** + * Initial exploration ratio value + */ + double epsilon_init; /** * Ratio, with what probability an agent should explore in the e-greed policy */ - double explore_ratio; + double epsilon; /** - * How big the share of the global part of the reward signal is + * Decay factor of the explore ratio */ - double reward_global_share; + double epsilon_decay; /** * Minimal interval time between steps in milliseconds @@ -164,6 +223,7 @@ struct RIL_Address_Wrapped struct ATS_Address *address_naked; }; + struct RIL_Peer_Agent { /** @@ -214,7 +274,7 @@ struct RIL_Peer_Agent /** * Last perceived state feature vector */ - double * s_old; + double *s_old; /** * Last chosen action @@ -222,34 +282,39 @@ struct RIL_Peer_Agent int a_old; /** - * Eligibility trace vector + * Eligibility traces */ - double * e; + double ** E; + + /** + * Whether to reset the eligibility traces to 0 after a Q-exploration step + */ + int eligibility_reset; /** * Address in use */ - struct ATS_Address * address_inuse; + struct ATS_Address *address_inuse; /** * Head of addresses DLL */ - struct RIL_Address_Wrapped * addresses_head; + struct RIL_Address_Wrapped *addresses_head; /** * Tail of addresses DLL */ - struct RIL_Address_Wrapped * addresses_tail; + struct RIL_Address_Wrapped *addresses_tail; /** * Inbound bandwidth assigned by the agent */ - unsigned long long bw_in; + uint32_t bw_in; /** * Outbound bandwidth assigned by the agent */ - unsigned long long bw_out; + uint32_t bw_out; /** * Flag whether a suggestion has to be issued @@ -259,35 +324,65 @@ struct RIL_Peer_Agent /** * The address which has to be issued */ - struct ATS_Address * suggestion_address; + struct ATS_Address *suggestion_address; + + /** + * The agent's last objective value + */ + double objective_old; + + /** + * NOP bonus + */ + double nop_bonus; }; -struct RIL_Network +struct RIL_Scope { /** * ATS network type */ - enum GNUNET_ATS_Network_Type type; + enum GNUNET_NetworkType type; /** * Total available inbound bandwidth */ - unsigned long long bw_in_available; + uint32_t bw_in_available; /** * Bandwidth inbound assigned in network after last step */ - unsigned long long bw_in_assigned; + uint32_t bw_in_assigned; + + /** + * Bandwidth inbound actually utilized in the network + */ + uint32_t bw_in_utilized; /** * Total available outbound bandwidth */ - unsigned long long bw_out_available; + uint32_t bw_out_available; /** - * * Bandwidth outbound assigned in network after last step + * Bandwidth outbound assigned in network after last step */ unsigned long long bw_out_assigned; + + /** + * Bandwidth outbound actually utilized in the network + */ + unsigned long long bw_out_utilized; + + /** + * Number of active agents in scope + */ + unsigned int active_agent_count; + + /** + * The social welfare achieved in the scope + */ + double social_welfare; }; /** @@ -298,12 +393,7 @@ struct GAS_RIL_Handle /** * The solver-plugin environment of the solver-plugin API */ - struct GNUNET_ATS_PluginEnvironment *plugin_envi; - - /** - * Statistics handle - */ - struct GNUNET_STATISTICS_Handle *stats; + struct GNUNET_ATS_PluginEnvironment *env; /** * Number of performed steps @@ -318,7 +408,7 @@ struct GAS_RIL_Handle /** * Task identifier of the next time-step to be executed */ - GNUNET_SCHEDULER_TaskIdentifier step_next_task_id; + struct GNUNET_SCHEDULER_Task * step_next_task_id; /** * Variable discount factor, dependent on time between steps @@ -348,7 +438,7 @@ struct GAS_RIL_Handle /** * Array of networks with global assignment state */ - struct RIL_Network * network_entries; + struct RIL_Scope * network_entries; /** * Networks count @@ -373,19 +463,10 @@ struct GAS_RIL_Handle }; /* - * Private functions + * "Private" functions * --------------------------- */ -static int -ril_count_agents(struct GAS_RIL_Handle * solver); - -static double -agent_get_utility (struct RIL_Peer_Agent *agent) -{ - return (double) agent->bw_in; -} - /** * Estimate the current action-value for state s and action a * @@ -395,48 +476,25 @@ agent_get_utility (struct RIL_Peer_Agent *agent) * @return estimation value */ static double -agent_estimate_q (struct RIL_Peer_Agent *agent, double *state, int action) +agent_q (struct RIL_Peer_Agent *agent, + const double *state, + int action) { - int i; - double result = 0; + unsigned int i; + double result = 0.0; for (i = 0; i < agent->m; i++) - { result += state[i] * agent->W[action][i]; - } - - GNUNET_assert(!isnan(result)); + /* prevent crashes if learning diverges */ + if (isnan(result)) + return isnan(result) * UINT32_MAX; if (isinf(result)) - { - return isinf(result) * UINT32_MAX; //TODO! prevent crash when learning diverges - } + return isinf(result) * UINT32_MAX; return result; } -/** - * Decide whether to do exploration (i.e. taking a new action) or exploitation (i.e. taking the - * currently estimated best action) in the current step - * - * @param agent agent performing the step - * @return yes, if exploring - */ -static int -agent_decide_exploration (struct RIL_Peer_Agent *agent) -{ - //TODO? Future Work: Improve exploration/exploitation trade-off by different mechanisms than e-greedy - double r = (double) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, - UINT32_MAX) / (double) UINT32_MAX; - - if (r < agent->envi->parameters.explore_ratio) - { - return GNUNET_YES; - } - return GNUNET_NO; -} - - /** * Get the index of the address in the agent's list. * @@ -469,7 +527,7 @@ agent_address_get_index (struct RIL_Peer_Agent *agent, struct ATS_Address *addre * @return wrapped address */ static struct RIL_Address_Wrapped * -agent_address_get (struct RIL_Peer_Agent *agent, struct ATS_Address *address) +agent_address_get_wrapped (struct RIL_Peer_Agent *agent, struct ATS_Address *address) { struct RIL_Address_Wrapped *cur; @@ -480,6 +538,66 @@ agent_address_get (struct RIL_Peer_Agent *agent, struct ATS_Address *address) } +static int +agent_action_is_possible (struct RIL_Peer_Agent *agent, int action) +{ + int address_index; + + switch (action) + { + case RIL_ACTION_NOTHING: + return GNUNET_YES; + break; + case RIL_ACTION_BW_IN_INC: + case RIL_ACTION_BW_IN_DBL: + if (agent->bw_in >= RIL_MAX_BW) + return GNUNET_NO; + else + return GNUNET_YES; + break; + case RIL_ACTION_BW_IN_DEC: + case RIL_ACTION_BW_IN_HLV: + if (agent->bw_in <= 0) + return GNUNET_NO; + else + return GNUNET_YES; + break; + case RIL_ACTION_BW_OUT_INC: + case RIL_ACTION_BW_OUT_DBL: + if (agent->bw_out >= RIL_MAX_BW) + return GNUNET_NO; + else + return GNUNET_YES; + break; + case RIL_ACTION_BW_OUT_DEC: + case RIL_ACTION_BW_OUT_HLV: + if (agent->bw_out <= 0) + return GNUNET_NO; + else + return GNUNET_YES; + break; + default: + if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) //switch address action + { + address_index = action - RIL_ACTION_TYPE_NUM; + + GNUNET_assert(address_index >= 0); + GNUNET_assert( + address_index <= agent_address_get_index (agent, agent->addresses_tail->address_naked)); + + if ((agent_address_get_index(agent, agent->address_inuse) == address_index) || + agent->address_inuse->active) + return GNUNET_NO; + else + return GNUNET_YES; + break; + } + // error - action does not exist + GNUNET_assert(GNUNET_NO); + } +} + + /** * Gets the action, with the maximal estimated Q-value (i.e. the one currently estimated to bring the * most reward in the future) @@ -489,7 +607,7 @@ agent_address_get (struct RIL_Peer_Agent *agent, struct ATS_Address *address) * @return the action promising most future reward */ static int -agent_get_action_best (struct RIL_Peer_Agent *agent, double *state) +agent_get_action_max (struct RIL_Peer_Agent *agent, double *state) { int i; int max_i = RIL_ACTION_INVALID; @@ -498,11 +616,14 @@ agent_get_action_best (struct RIL_Peer_Agent *agent, double *state) for (i = 0; i < agent->n; i++) { - cur_q = agent_estimate_q (agent, state, i); - if (cur_q > max_q) + if (agent_action_is_possible(agent, i)) { - max_q = cur_q; - max_i = i; + cur_q = agent_q (agent, state, i); + if (cur_q > max_q) + { + max_q = cur_q; + max_i = i; + } } } @@ -511,19 +632,48 @@ agent_get_action_best (struct RIL_Peer_Agent *agent, double *state) return max_i; } - /** - * Gets any action, to explore the action space from that state + * Chooses a random action from the set of possible ones * - * @param agent agent performing the calculation - * @param state the state from which to take the action - * @return any action + * @param agent the agent performing the action + * @return the action index */ static int -agent_get_action_explore (struct RIL_Peer_Agent *agent, double *state) +agent_get_action_random (struct RIL_Peer_Agent *agent) { - // TODO?: Future Work: Choose the action for exploration, which has been explored the least in this state - return GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, agent->n); + int i; + int is_possible[agent->n]; + int sum = 0; + int r; + + for (i = 0; in; i++) + { + if (agent_action_is_possible(agent, i)) + { + is_possible[i] = GNUNET_YES; + sum++; + } + else + { + is_possible[i] = GNUNET_NO; + } + } + + r = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, sum); + + sum = -1; + for (i = 0; in; i++) + { + if (is_possible[i]) + { + sum++; + if (sum == r) + return i; + } + } + + GNUNET_assert(GNUNET_NO); + return RIL_ACTION_INVALID; } @@ -536,86 +686,116 @@ agent_get_action_explore (struct RIL_Peer_Agent *agent, double *state) * @param a_prime the new */ static void -agent_update_weights (struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime) +agent_update (struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime) { int i; + int k; double delta; - double *theta = agent->W[agent->a_old]; + double **theta = agent->W; delta = agent->envi->global_discount_integrated * reward; //reward - delta += agent->envi->global_discount_variable * agent_estimate_q (agent, s_next, a_prime); //discounted future value - delta -= agent_estimate_q (agent, agent->s_old, agent->a_old); //one step + delta += agent->envi->global_discount_variable * agent_q (agent, s_next, a_prime); //discounted future value + delta -= agent_q (agent, agent->s_old, agent->a_old); //one step // LOG(GNUNET_ERROR_TYPE_INFO, "update() Step# %llu Q(s,a): %f a: %f r: %f y: %f Q(s+1,a+1) = %f delta: %f\n", // agent->step_count, -// agent_estimate_q (agent, agent->s_old, agent->a_old), +// agent_q (agent, agent->s_old, agent->a_old), // agent->envi->parameters.alpha, // reward, // agent->envi->global_discount_variable, -// agent_estimate_q (agent, s_next, a_prime), +// agent_q (agent, s_next, a_prime), // delta); - for (i = 0; i < agent->m; i++) + for (k = 0; k < agent->n; k++) { -// LOG(GNUNET_ERROR_TYPE_INFO, "alpha = %f delta = %f e[%d] = %f\n", -// agent->envi->parameters.alpha, -// delta, -// i, -// agent->e[i]); - theta[i] += agent->envi->parameters.alpha * delta * agent->s_old[i];// * agent->e[i]; + for (i = 0; i < agent->m; i++) + { + // LOG(GNUNET_ERROR_TYPE_INFO, "alpha = %f delta = %f e[%d] = %f\n", + // agent->envi->parameters.alpha, + // delta, + // i, + // agent->e[i]); + theta[k][i] += agent->envi->parameters.alpha * delta * agent->E[k][i]; + } } } /** * Changes the eligibility trace vector e in various manners: - * #RIL_E_ACCUMULATE - adds @a f to each component as in accumulating eligibility traces - * #RIL_E_REPLACE - resets each component to @a f as in replacing traces - * #RIL_E_SET - multiplies e with discount factor and lambda as in the update rule + * #RIL_E_ACCUMULATE - adds @a feature to each component as in accumulating eligibility traces + * #RIL_E_REPLACE - resets each component to @a feature as in replacing traces + * #RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule * #RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing * * @param agent the agent handle * @param mod the kind of modification - * @param f how much to change + * @param feature the feature vector + * @param action the action to take */ static void agent_modify_eligibility (struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, - double *f) + double *feature, + int action) { int i; - double *e = agent->e; + int k; for (i = 0; i < agent->m; i++) { switch (mod) { case RIL_E_ACCUMULATE: - e[i] += f[i]; + agent->E[action][i] += feature[i]; break; case RIL_E_REPLACE: - e[i] = f[i]; + agent->E[action][i] = agent->E[action][i] > feature[i] ? agent->E[action][i] : feature[i]; break; - case RIL_E_SET: - e[i] *= agent->envi->global_discount_variable * agent->envi->parameters.lambda; + case RIL_E_DECAY: + for (k = 0; k < agent->n; k++) + { + agent->E[k][i] *= agent->envi->global_discount_variable * agent->envi->parameters.lambda; + } break; case RIL_E_ZERO: - e[i] = 0; + for (k = 0; k < agent->n; k++) + { + agent->E[k][i] = 0; + } break; } } } - +/** + * Informs the environment about the status of the solver + * + * @param solver + * @param op + * @param stat + */ static void ril_inform (struct GAS_RIL_Handle *solver, - enum GAS_Solver_Operation op, - enum GAS_Solver_Status stat) + enum GAS_Solver_Operation op, + enum GAS_Solver_Status stat) { - if (NULL != solver->plugin_envi->info_cb) - solver->plugin_envi->info_cb (solver->plugin_envi->info_cb_cls, op, stat, GAS_INFO_NONE); + solver->env->info_cb (solver->env->cls, + op, + stat, + GAS_INFO_NONE); } +/** + * Calculates the maximum bandwidth an agent can assign in a network scope + * + * @param net + */ +static unsigned long long +ril_get_max_bw (struct RIL_Scope *net) +{ + return GNUNET_MIN(2 * GNUNET_MAX(net->bw_in_available, net->bw_out_available), GNUNET_ATS_MaxBandwidth); +} /** * Changes the active assignment suggestion of the handler and invokes the bw_changed callback to @@ -626,7 +806,7 @@ ril_inform (struct GAS_RIL_Handle *solver, * @param new_address the address which is to be used * @param new_bw_in the new amount of inbound bandwidth set for this address * @param new_bw_out the new amount of outbound bandwidth set for this address - * @param silent disables invocation of the bw_changed callback, if GNUNET_YES + * @param silent disables invocation of the bw_changed callback, if #GNUNET_YES */ static void envi_set_active_suggestion (struct GAS_RIL_Handle *solver, @@ -638,7 +818,9 @@ envi_set_active_suggestion (struct GAS_RIL_Handle *solver, { int notify = GNUNET_NO; - LOG(GNUNET_ERROR_TYPE_DEBUG, " set_active_suggestion() for peer '%s'\n", GNUNET_i2s (&agent->peer)); + LOG(GNUNET_ERROR_TYPE_DEBUG, + " set_active_suggestion() for peer '%s'\n", + GNUNET_i2s (&agent->peer)); //address change if (agent->address_inuse != new_address) @@ -646,15 +828,15 @@ envi_set_active_suggestion (struct GAS_RIL_Handle *solver, if (NULL != agent->address_inuse) { agent->address_inuse->active = GNUNET_NO; - agent->address_inuse->assigned_bw_in.value__ = htonl (0); - agent->address_inuse->assigned_bw_out.value__ = htonl (0); + agent->address_inuse->assigned_bw_in = 0; + agent->address_inuse->assigned_bw_out = 0; } if (NULL != new_address) { LOG(GNUNET_ERROR_TYPE_DEBUG, " set address active: %s\n", agent->is_active ? "yes" : "no"); new_address->active = agent->is_active; - new_address->assigned_bw_in.value__ = htonl (agent->bw_in); - new_address->assigned_bw_out.value__ = htonl (agent->bw_out); + new_address->assigned_bw_in = agent->bw_in; + new_address->assigned_bw_out = agent->bw_out; } notify |= GNUNET_YES; } @@ -672,13 +854,13 @@ envi_set_active_suggestion (struct GAS_RIL_Handle *solver, if (agent->bw_in != new_bw_in) { agent->bw_in = new_bw_in; - new_address->assigned_bw_in.value__ = htonl (new_bw_in); + new_address->assigned_bw_in = new_bw_in; notify |= GNUNET_YES; } if (agent->bw_out != new_bw_out) { agent->bw_out = new_bw_out; - new_address->assigned_bw_out.value__ = htonl (new_bw_out); + new_address->assigned_bw_out = new_bw_out; notify |= GNUNET_YES; } } @@ -693,9 +875,9 @@ envi_set_active_suggestion (struct GAS_RIL_Handle *solver, } else if (agent->address_inuse) { - //disconnect case, no new address - GNUNET_assert(0 == ntohl (agent->address_inuse->assigned_bw_in.value__)); - GNUNET_assert(0 == ntohl (agent->address_inuse->assigned_bw_out.value__)); + /* disconnect case, no new address */ + GNUNET_assert(0 == agent->address_inuse->assigned_bw_in); + GNUNET_assert(0 == agent->address_inuse->assigned_bw_out); agent->bw_in = 0; agent->bw_out = 0; @@ -707,31 +889,6 @@ envi_set_active_suggestion (struct GAS_RIL_Handle *solver, } -static unsigned long long -ril_network_get_assigned (struct GAS_RIL_Handle *solver, enum GNUNET_ATS_Network_Type type, int direction_in) -{ - struct RIL_Peer_Agent *cur; - struct RIL_Network *net; - unsigned long long sum = 0; - - for (cur = solver->agents_head; NULL != cur; cur = cur->next) - { - if (cur->is_active && cur->address_inuse) - { - net = cur->address_inuse->solver_information; - if (net->type == type) - { - if (direction_in) - sum += cur->bw_in; - else - sum += cur->bw_out; - } - } - } - - return sum; -} - /** * Allocates a state vector and fills it with the features present * @param solver the solver handle @@ -750,22 +907,25 @@ envi_get_state (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent) int m; int i; int k; + unsigned long long max_bw; + + state = GNUNET_malloc (sizeof(double) * agent->m); - state = GNUNET_malloc (sizeof(agent->m)); + max_bw = ril_get_max_bw((struct RIL_Scope *) agent->address_inuse->solver_information); y[0] = (double) agent->bw_out; y[1] = (double) agent->bw_in; - m = agent_address_get_index (agent, agent->address_inuse) * (solver->parameters.divisor+1) * (solver->parameters.divisor+1); - for (i = 0; i <= solver->parameters.divisor; i++) + m = agent_address_get_index (agent, agent->address_inuse) * (solver->parameters.rbf_divisor+1) * (solver->parameters.rbf_divisor+1); + for (i = 0; i <= solver->parameters.rbf_divisor; i++) { - for (k = 0; k <= solver->parameters.divisor; k++) + for (k = 0; k <= solver->parameters.rbf_divisor; k++) { - x[0] = i * GNUNET_ATS_MaxBandwidth / solver->parameters.divisor; - x[1] = k * GNUNET_ATS_MaxBandwidth / solver->parameters.divisor; + x[0] = (double) i * (double) max_bw / (double) solver->parameters.rbf_divisor; + x[1] = (double) k * (double) max_bw / (double) solver->parameters.rbf_divisor; d[0] = x[0]-y[0]; d[1] = x[1]-y[1]; - sigma = ((double) GNUNET_ATS_MaxBandwidth / 2) * M_SQRT2; + sigma = (((double) max_bw / ((double) solver->parameters.rbf_divisor + 1)) * 0.5); f = exp(-((d[0]*d[0] + d[1]*d[1]) / (2 * sigma * sigma))); state[m++] = f; } @@ -774,160 +934,106 @@ envi_get_state (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent) return state; } -///* -// * For all networks a peer has an address in, this gets the maximum bandwidth which could -// * theoretically be available in one of the networks. This is used for bandwidth normalization. -// * -// * @param agent the agent handle -// * @param direction_in whether the inbound bandwidth should be considered. Returns the maximum outbound bandwidth if GNUNET_NO -// */ -//static unsigned long long -//ril_get_max_bw (struct RIL_Peer_Agent *agent, int direction_in) -//{ -// /* -// * get the maximum bandwidth possible for a peer, e.g. among all addresses which addresses' -// * network could provide the maximum bandwidth if all that bandwidth was used on that one peer. -// */ -// unsigned long long max = 0; -// struct RIL_Address_Wrapped *cur; -// struct RIL_Network *net; -// -// for (cur = agent->addresses_head; NULL != cur; cur = cur->next) -// { -// net = cur->address_naked->solver_information; -// if (direction_in) -// { -// if (net->bw_in_available > max) -// { -// max = net->bw_in_available; -// } -// } -// else -// { -// if (net->bw_out_available > max) -// { -// max = net->bw_out_available; -// } -// } -// } -// return max; -//} - -///* -// * Get the index of the quality-property in question -// * -// * @param type the quality property type -// * @return the index -// */ -//static int -//ril_find_property_index (uint32_t type) -//{ -// int existing_types[] = GNUNET_ATS_QualityProperties; -// int c; -// for (c = 0; c < GNUNET_ATS_QualityPropertiesCount; c++) -// if (existing_types[c] == type) -// return c; -// return GNUNET_SYSERR; -//} - -//static int -//ril_get_atsi (struct ATS_Address *address, uint32_t type) -//{ -// int c1; -// GNUNET_assert(NULL != address); -// -// if ((NULL == address->atsi) || (0 == address->atsi_count)) -// return 0; -// -// for (c1 = 0; c1 < address->atsi_count; c1++) -// { -// if (ntohl (address->atsi[c1].type) == type) -// return ntohl (address->atsi[c1].value); -// } -// return 0; -//} - -//static double -//envi_reward_global (struct GAS_RIL_Handle *solver) -//{ -// int i; -// struct RIL_Network net; -// unsigned int sum_in_available = 0; -// unsigned int sum_out_available = 0; -// unsigned int sum_in_assigned = 0; -// unsigned int sum_out_assigned = 0; -// double ratio_in; -// double ratio_out; -// -// for (i = 0; i < solver->networks_count; i++) -// { -// net = solver->network_entries[i]; -// sum_in_available += net.bw_in_available; -// sum_in_assigned += net.bw_in_assigned; -// sum_out_available += net.bw_out_available; -// sum_out_assigned += net.bw_out_assigned; -// } -// -// ratio_in = ((double) sum_in_assigned) / ((double) sum_in_available); -// ratio_out = ((double) sum_out_assigned) / ((double) sum_out_available); -// -// // global reward in [1,2] -// return ratio_in +1; -// return ((ratio_in + ratio_out) / 2) + 1; -//} - -//static double -//envi_reward_local (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent) -//{ -// const double *preferences; -// const double *properties; -// int prop_index; -// double pref_match = 0; -// double bw_norm; -// double dl_norm; -// -// preferences = solver->plugin_envi->get_preferences (solver->plugin_envi->get_preference_cls, -// &agent->peer); -// properties = solver->plugin_envi->get_property (solver->plugin_envi->get_property_cls, -// agent->address_inuse); -// -// // delay in [0,1] -// prop_index = ril_find_property_index (GNUNET_ATS_QUALITY_NET_DELAY); -// dl_norm = 2 - properties[prop_index]; //invert property as we want to maximize for lower latencies -// -// // utilization in [0,1] -// bw_norm = (((double) ril_get_atsi (agent->address_inuse, GNUNET_ATS_UTILIZATION_IN) -// / (double) ril_get_max_bw (agent, GNUNET_YES)) -// + ((double) ril_get_atsi (agent->address_inuse, GNUNET_ATS_UTILIZATION_OUT) -// / (double) ril_get_max_bw (agent, GNUNET_NO))) / 2; -// -// // preference matching in [0,4] -// pref_match += (preferences[GNUNET_ATS_PREFERENCE_LATENCY] * dl_norm); -// pref_match += (preferences[GNUNET_ATS_PREFERENCE_BANDWIDTH] * bw_norm); -// -// // local reward in [1,2] -// return (pref_match / 4) +1; -//} +/** + * Returns the utility value of the connection an agent manages + * + * @param agent the agent in question + * @return the utility value + */ +static double +agent_get_utility (struct RIL_Peer_Agent *agent) +{ + const double *preferences; + double delay_atsi; + double delay_norm; + double pref_match; + + preferences = agent->envi->env->get_preferences (agent->envi->env->cls, + &agent->peer); + + delay_atsi = agent->address_inuse->norm_delay.norm; + delay_norm = RIL_UTILITY_DELAY_MAX*exp(-delay_atsi*0.00001); + + pref_match = preferences[GNUNET_ATS_PREFERENCE_LATENCY] * delay_norm; + pref_match += preferences[GNUNET_ATS_PREFERENCE_BANDWIDTH] * + sqrt((double) (agent->bw_in/RIL_MIN_BW) * (double) (agent->bw_out/RIL_MIN_BW)); + return pref_match; +} + +/** + * Calculates the social welfare within a network scope according to what social + * welfare measure is set in the configuration. + * + * @param solver the solver handle + * @param scope the network scope in question + * @return the social welfare value + */ static double -envi_get_collective_utility (struct GAS_RIL_Handle *solver) +ril_network_get_social_welfare (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope) { - //TODO! add nash product struct RIL_Peer_Agent *cur; - double result = RIL_UTILITY_MAX; + double result; - for (cur = solver->agents_head; NULL != cur; cur = cur->next) + switch (solver->parameters.social_welfare) { - if (cur->is_active) + case RIL_WELFARE_EGALITARIAN: + result = DBL_MAX; + for (cur = solver->agents_head; NULL != cur; cur = cur->next) { - if (cur->address_inuse) + if (cur->is_active && cur->address_inuse && (cur->address_inuse->solver_information == scope)) { result = GNUNET_MIN(result, agent_get_utility(cur)); } } + return result; + + case RIL_WELFARE_NASH: + result = 0; + for (cur = solver->agents_head; NULL != cur; cur = cur->next) + { + if (cur->is_active && cur->address_inuse && (cur->address_inuse->solver_information == scope)) + { + result *= pow(agent_get_utility(cur), 1.0 / (double) scope->active_agent_count); + } + } + return result; } + GNUNET_assert(GNUNET_NO); + return 1; +} - return result; +static double +envi_get_penalty (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent) +{ + struct RIL_Scope *net; + unsigned long long over_max; + unsigned long long over_in = 0; + unsigned long long over_out = 0; + + net = agent->address_inuse->solver_information; + + if (net->bw_in_utilized > net->bw_in_available) + { + over_in = net->bw_in_utilized - net->bw_in_available; + if (RIL_ACTION_BW_IN_INC == agent->a_old) + { + /* increase quadratically */ + over_in *= over_in; + } + } + if (net->bw_out_utilized > net->bw_out_available) + { + over_out = net->bw_out_utilized - net->bw_out_available; + if (RIL_ACTION_BW_OUT_INC == agent->a_old) + { + /* increase quadratically */ + over_out *= over_out; + } + } + over_max = (over_in + over_out) / (RIL_MIN_BW * RIL_MIN_BW); + + return -1.0 * (double) over_max; } /** @@ -942,21 +1048,33 @@ envi_get_collective_utility (struct GAS_RIL_Handle *solver) static double envi_get_reward (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent) { - struct RIL_Network *net; - - unsigned long long objective; + struct RIL_Scope *net; + double objective; + double delta; + double steady; + double penalty; + double reward; net = agent->address_inuse->solver_information; - if (net->bw_in_assigned > net->bw_in_available) + + penalty = envi_get_penalty(solver, agent); + objective = (agent_get_utility (agent) + net->social_welfare) / 2; + delta = objective - agent->objective_old; + agent->objective_old = objective; + + if (delta != 0 && penalty == 0) { - objective = net->bw_in_available - net->bw_in_assigned; + agent->nop_bonus = delta * RIL_NOP_DECAY; } else { - objective = envi_get_collective_utility(solver); + agent->nop_bonus *= RIL_NOP_DECAY; } - return objective; + steady = (RIL_ACTION_NOTHING == agent->a_old) ? agent->nop_bonus : 0; + + reward = delta + steady; + return reward + penalty; } /** @@ -972,20 +1090,23 @@ envi_action_bw_double (struct GAS_RIL_Handle *solver, int direction_in) { unsigned long long new_bw; + unsigned long long max_bw; + + max_bw = ril_get_max_bw((struct RIL_Scope *) agent->address_inuse->solver_information); if (direction_in) { new_bw = agent->bw_in * 2; - if (new_bw < agent->bw_in || new_bw > GNUNET_ATS_MaxBandwidth) - new_bw = GNUNET_ATS_MaxBandwidth; + if (new_bw < agent->bw_in || new_bw > max_bw) + new_bw = max_bw; envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw, agent->bw_out, GNUNET_NO); } else { new_bw = agent->bw_out * 2; - if (new_bw < agent->bw_out || new_bw > GNUNET_ATS_MaxBandwidth) - new_bw = GNUNET_ATS_MaxBandwidth; + if (new_bw < agent->bw_out || new_bw > max_bw) + new_bw = max_bw; envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, new_bw, GNUNET_NO); } @@ -1010,16 +1131,16 @@ envi_action_bw_halven (struct GAS_RIL_Handle *solver, if (direction_in) { new_bw = agent->bw_in / 2; - if (new_bw < MIN_BW || new_bw > agent->bw_in) - new_bw = MIN_BW; + if (new_bw <= 0 || new_bw > agent->bw_in) + new_bw = 0; envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw, agent->bw_out, GNUNET_NO); } else { new_bw = agent->bw_out / 2; - if (new_bw < MIN_BW || new_bw > agent->bw_out) - new_bw = MIN_BW; + if (new_bw <= 0 || new_bw > agent->bw_out) + new_bw = 0; envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, new_bw, GNUNET_NO); } @@ -1037,20 +1158,23 @@ static void envi_action_bw_inc (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in) { unsigned long long new_bw; + unsigned long long max_bw; + + max_bw = ril_get_max_bw((struct RIL_Scope *) agent->address_inuse->solver_information); if (direction_in) { - new_bw = agent->bw_in + (RIL_INC_DEC_STEP_SIZE * MIN_BW); - if (new_bw < agent->bw_in || new_bw > GNUNET_ATS_MaxBandwidth) - new_bw = GNUNET_ATS_MaxBandwidth; + new_bw = agent->bw_in + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW); + if (new_bw < agent->bw_in || new_bw > max_bw) + new_bw = max_bw; envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw, agent->bw_out, GNUNET_NO); } else { - new_bw = agent->bw_out + (RIL_INC_DEC_STEP_SIZE * MIN_BW); - if (new_bw < agent->bw_out || new_bw > GNUNET_ATS_MaxBandwidth) - new_bw = GNUNET_ATS_MaxBandwidth; + new_bw = agent->bw_out + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW); + if (new_bw < agent->bw_out || new_bw > max_bw) + new_bw = max_bw; envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, new_bw, GNUNET_NO); } @@ -1072,17 +1196,17 @@ envi_action_bw_dec (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, if (direction_in) { - new_bw = agent->bw_in - (RIL_INC_DEC_STEP_SIZE * MIN_BW); - if (new_bw < MIN_BW || new_bw > agent->bw_in) - new_bw = MIN_BW; + new_bw = agent->bw_in - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW); + if (new_bw <= 0 || new_bw > agent->bw_in) + new_bw = 0; envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw, agent->bw_out, GNUNET_NO); } else { - new_bw = agent->bw_out - (RIL_INC_DEC_STEP_SIZE * MIN_BW); - if (new_bw < MIN_BW || new_bw > agent->bw_out) - new_bw = MIN_BW; + new_bw = agent->bw_out - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW); + if (new_bw <= 0 || new_bw > agent->bw_out) + new_bw = 0; envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, new_bw, GNUNET_NO); } @@ -1103,6 +1227,8 @@ envi_action_address_switch (struct GAS_RIL_Handle *solver, struct RIL_Address_Wrapped *cur; int i = 0; + //cur = agent_address_get_wrapped(agent, agent->address_inuse); + for (cur = agent->addresses_head; NULL != cur; cur = cur->next) { if (i == address_index) @@ -1177,78 +1303,192 @@ envi_do_action (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int } /** - * Performs one step of the Markov Decision Process. Other than in the literature the step starts - * after having done the last action a_old. It observes the new state s_next and the reward - * received. Then the coefficient update is done according to the SARSA or Q-learning method. The - * next action is put into effect. + * Selects the next action using the e-greedy strategy. I.e. with a probability + * of (1-e) the action with the maximum expected return will be chosen + * (=> exploitation) and with probability (e) a random action will be chosen. + * In case the Q-learning rule is set, the function also resets the eligibility + * traces in the exploration case (after Watkin's Q-learning). * - * @param agent the agent performing the step + * @param agent the agent selecting an action + * @param state the current state-feature vector + * @return the action index */ -static void -agent_step (struct RIL_Peer_Agent *agent) +static int +agent_select_egreedy (struct RIL_Peer_Agent *agent, double *state) { - int a_next = RIL_ACTION_INVALID; - int explore; - double *s_next; - double reward; + int action; + double r = (double) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + UINT32_MAX) / (double) UINT32_MAX; - LOG(GNUNET_ERROR_TYPE_DEBUG, " agent_step() Peer '%s', algorithm %s\n", - GNUNET_i2s (&agent->peer), - agent->envi->parameters.algorithm ? "Q" : "SARSA"); + if (r < agent->envi->parameters.epsilon) //explore + { + action = agent_get_action_random(agent); + if (RIL_ALGO_Q == agent->envi->parameters.algorithm) + { + agent->eligibility_reset = GNUNET_YES; + } + agent->envi->parameters.epsilon *= agent->envi->parameters.epsilon_decay; + return action; + } + else //exploit + { + action = agent_get_action_max(agent, state); + return action; + } +} - s_next = envi_get_state (agent->envi, agent); - reward = envi_get_reward (agent->envi, agent); - explore = agent_decide_exploration (agent); +/** + * Selects the next action with a probability corresponding to its value. The + * probability is calculated using a Boltzmann distribution with a temperature + * value. The higher the temperature, the more are the action selection + * probabilities the same. With a temperature of 0, the selection is greedy, + * i.e. always the action with the highest value is chosen. + * @param agent + * @param state + * @return + */ +static int +agent_select_softmax (struct RIL_Peer_Agent *agent, double *state) +{ + int i; + int a_max; + double eqt[agent->n]; + double p[agent->n]; + double sum = 0; + double r; - switch (agent->envi->parameters.algorithm) + a_max = agent_get_action_max(agent, state); + + for (i=0; in; i++) { - case RIL_ALGO_SARSA: - if (explore) + if (agent_action_is_possible(agent, i)) + { + eqt[i] = exp(agent_q(agent,state,i) / agent->envi->parameters.temperature); + if (isinf (eqt[i])) + eqt[i] = isinf(eqt[i]) * UINT32_MAX; + sum += eqt[i]; + } + } + for (i=0; in; i++) + { + if (agent_action_is_possible(agent, i)) { - a_next = agent_get_action_explore (agent, s_next); + p[i] = eqt[i]/sum; } else { - a_next = agent_get_action_best (agent, s_next); + p[i] = 0; + } + } + r = (double) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + UINT32_MAX) / (double) UINT32_MAX; + sum = 0; + for (i=0; in; i++) + { + if (sum + p[i] > r) + { + if (i != a_max) + { + if (RIL_ALGO_Q == agent->envi->parameters.algorithm) + agent->eligibility_reset = GNUNET_YES; + agent->envi->parameters.temperature *= agent->envi->parameters.temperature_decay; + } + return i; } + sum += p[i]; + } + GNUNET_assert(GNUNET_NO); + return RIL_ACTION_INVALID; +} + +/** + * Select the next action of an agent either according to the e-greedy strategy + * or the softmax strategy. + * + * @param agent the agent in question + * @param state the current state-feature vector + * @return the action index + */ +static int +agent_select_action (struct RIL_Peer_Agent *agent, double *state) +{ + if (agent->envi->parameters.select == RIL_SELECT_EGREEDY) + { + return agent_select_egreedy(agent, state); + } + else + { + return agent_select_softmax(agent, state); + } +} + +/** + * Performs one step of the Markov Decision Process. Other than in the literature the step starts + * after having done the last action a_old. It observes the new state s_next and the reward + * received. Then the coefficient update is done according to the SARSA or Q-learning method. The + * next action is put into effect. + * + * @param agent the agent performing the step + */ +static void +agent_step (struct RIL_Peer_Agent *agent) +{ + int a_next = RIL_ACTION_INVALID; + int a_max; + double *s_next; + double reward; + + LOG(GNUNET_ERROR_TYPE_DEBUG, " agent_step() Peer '%s', algorithm %s\n", + GNUNET_i2s (&agent->peer), + agent->envi->parameters.algorithm ? "Q" : "SARSA"); + + s_next = envi_get_state (agent->envi, agent); + reward = envi_get_reward (agent->envi, agent); + + if (agent->eligibility_reset) + { + agent_modify_eligibility(agent, RIL_E_ZERO, NULL, -1); + agent->eligibility_reset = GNUNET_NO; + } + else + { + agent_modify_eligibility (agent, RIL_E_DECAY, NULL, -1); + } + if (RIL_ACTION_INVALID != agent->a_old) + { + agent_modify_eligibility (agent, agent->envi->parameters.eligibility_trace_mode, agent->s_old, agent->a_old); + } + + switch (agent->envi->parameters.algorithm) + { + case RIL_ALGO_SARSA: + a_next = agent_select_action (agent, s_next); if (RIL_ACTION_INVALID != agent->a_old) { //updates weights with selected action (on-policy), if not first step - agent_update_weights (agent, reward, s_next, a_next); - agent_modify_eligibility (agent, RIL_E_SET, s_next); + agent_update (agent, reward, s_next, a_next); } break; case RIL_ALGO_Q: - a_next = agent_get_action_best (agent, s_next); + a_max = agent_get_action_max (agent, s_next); if (RIL_ACTION_INVALID != agent->a_old) { //updates weights with best action, disregarding actually selected action (off-policy), if not first step - agent_update_weights (agent, reward, s_next, a_next); - } - if (explore) - { - a_next = agent_get_action_explore (agent, s_next); - agent_modify_eligibility (agent, RIL_E_ZERO, NULL); - } - else - { - a_next = agent_get_action_best (agent, s_next); - agent_modify_eligibility (agent, RIL_E_SET, s_next); + agent_update (agent, reward, s_next, a_max); } + a_next = agent_select_action (agent, s_next); break; } GNUNET_assert(RIL_ACTION_INVALID != a_next); - agent_modify_eligibility (agent, RIL_E_ACCUMULATE, s_next); - -// GNUNET_log (GNUNET_ERROR_TYPE_INFO, "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n", -// agent->step_count, -// reward, -// agent->bw_in/1024, -// agent->bw_out/1024, -// a_next); + LOG (GNUNET_ERROR_TYPE_DEBUG, "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n", + agent->step_count, + reward, + agent->bw_in/1024, + agent->bw_out/1024, + a_next); envi_do_action (agent->envi, agent, a_next); @@ -1259,30 +1499,43 @@ agent_step (struct RIL_Peer_Agent *agent) agent->step_count += 1; } +/** + * Prototype of the ril_step() procedure + * + * @param solver the solver handle + */ static void ril_step (struct GAS_RIL_Handle *solver); + /** * Task for the scheduler, which performs one step and lets the solver know that * no further step is scheduled. * * @param cls the solver handle - * @param tc the task context for the scheduler */ static void -ril_step_scheduler_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +ril_step_scheduler_task (void *cls) { struct GAS_RIL_Handle *solver = cls; - solver->step_next_task_id = GNUNET_SCHEDULER_NO_TASK; + solver->step_next_task_id = NULL; ril_step (solver); } +/** + * Determines how much of the available bandwidth is assigned. If more is + * assigned than available it returns 1. The function is used to determine the + * step size of the adaptive stepping. + * + * @param solver the solver handle + * @return the ratio + */ static double ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver) { int i; - struct RIL_Network net; + struct RIL_Scope net; unsigned long long sum_assigned = 0; unsigned long long sum_available = 0; double ratio; @@ -1292,8 +1545,8 @@ ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver) net = solver->network_entries[i]; if (net.bw_in_assigned > 0) //only consider scopes where an address is actually active { - sum_assigned += net.bw_in_assigned; - sum_assigned += net.bw_out_assigned; + sum_assigned += net.bw_in_utilized; + sum_assigned += net.bw_out_utilized; sum_available += net.bw_in_available; sum_available += net.bw_out_available; } @@ -1307,7 +1560,7 @@ ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver) ratio = 0; } - return ratio > 1 ? 1 : ratio; //overutilization possible, cap at 1 + return ratio > 1 ? 1 : ratio; //overutilization is possible, cap at 1 } /** @@ -1317,7 +1570,7 @@ ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver) * @param type the network type * @return the network struct */ -static struct RIL_Network * +static struct RIL_Scope * ril_get_network (struct GAS_RIL_Handle *s, uint32_t type) { int i; @@ -1332,10 +1585,19 @@ ril_get_network (struct GAS_RIL_Handle *s, uint32_t type) return NULL ; } +/** + * Determines whether more connections are allocated in a network scope, than + * they would theoretically fit. This is used as a heuristic to determine, + * whether a new connection can be allocated or not. + * + * @param solver the solver handle + * @param network the network scope in question + * @return GNUNET_YES if there are theoretically enough resources left + */ static int -ril_network_is_not_full (struct GAS_RIL_Handle *solver, enum GNUNET_ATS_Network_Type network) +ril_network_is_not_full (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network) { - struct RIL_Network *net; + struct RIL_Scope *net; struct RIL_Peer_Agent *agent; unsigned long long address_count = 0; @@ -1352,14 +1614,26 @@ ril_network_is_not_full (struct GAS_RIL_Handle *solver, enum GNUNET_ATS_Network_ } net = ril_get_network (solver, network); - return (net->bw_in_available > MIN_BW * address_count) && (net->bw_out_available > MIN_BW * address_count); + return (net->bw_in_available > RIL_MIN_BW * address_count) && (net->bw_out_available > RIL_MIN_BW * address_count); } +/** + * Unblocks an agent for which a connection request is there, that could not + * be satisfied. Iterates over the addresses of the agent, if one of its + * addresses can now be allocated in its scope the agent is unblocked, + * otherwise it remains unchanged. + * + * @param solver the solver handle + * @param agent the agent in question + * @param silent + */ static void ril_try_unblock_agent (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent) { struct RIL_Address_Wrapped *addr_wrap; - struct RIL_Network *net; + struct RIL_Scope *net; + unsigned long long start_in; + unsigned long long start_out; for (addr_wrap = agent->addresses_head; NULL != addr_wrap; addr_wrap = addr_wrap->next) { @@ -1367,13 +1641,23 @@ ril_try_unblock_agent (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *age if (ril_network_is_not_full(solver, net->type)) { if (NULL == agent->address_inuse) - envi_set_active_suggestion (solver, agent, addr_wrap->address_naked, MIN_BW, MIN_BW, silent); + { + start_in = net->bw_in_available < net->bw_in_utilized ? (net->bw_in_available - net->bw_in_utilized) / 2 : RIL_MIN_BW; + start_out = net->bw_out_available < net->bw_out_utilized ? (net->bw_out_available - net->bw_out_utilized) / 2 : RIL_MIN_BW; + envi_set_active_suggestion (solver, agent, addr_wrap->address_naked, start_in, start_out, silent); + } return; } } agent->address_inuse = NULL; } +/** + * Determines how much the reward needs to be discounted depending on the amount + * of time, which has passed since the last time-step. + * + * @param solver the solver handle + */ static void ril_calculate_discount (struct GAS_RIL_Handle *solver) { @@ -1381,7 +1665,7 @@ ril_calculate_discount (struct GAS_RIL_Handle *solver) struct GNUNET_TIME_Relative time_delta; double tau; - // MDP case - remove when debugged + // MDP case only for debugging purposes if (solver->simulate) { solver->global_discount_variable = solver->parameters.gamma; @@ -1404,17 +1688,120 @@ ril_calculate_discount (struct GAS_RIL_Handle *solver) / (double) solver->parameters.beta; } +/** + * Count the number of active agents/connections in a network scope + * + * @param solver the solver handle + * @param scope the network scope in question + * @return the number of allocated connections + */ +static int +ril_network_count_active_agents (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope) +{ + int c = 0; + struct RIL_Peer_Agent *cur_agent; + + for (cur_agent = solver->agents_head; NULL != cur_agent; cur_agent = cur_agent->next) + { + if (cur_agent->is_active && cur_agent->address_inuse && (cur_agent->address_inuse->solver_information == scope)) + { + c++; + } + } + return c; +} + +/** + * Calculates how much bandwidth is assigned in sum in a network scope, either + * in the inbound or in the outbound direction. + * + * @param solver the solver handle + * @param type the type of the network scope in question + * @param direction_in GNUNET_YES if the inbound direction should be summed up, + * otherwise the outbound direction will be summed up + * @return the sum of the assigned bandwidths + */ +static unsigned long long +ril_network_get_assigned (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in) +{ + struct RIL_Peer_Agent *cur; + struct RIL_Scope *net; + unsigned long long sum = 0; + + for (cur = solver->agents_head; NULL != cur; cur = cur->next) + { + if (cur->is_active && cur->address_inuse) + { + net = cur->address_inuse->solver_information; + if (net->type == type) + { + if (direction_in) + sum += cur->bw_in; + else + sum += cur->bw_out; + } + } + } + + return sum; +} + +/** + * Calculates how much bandwidth is actually utilized in sum in a network scope, + * either in the inbound or in the outbound direction. + * + * @param solver the solver handle + * @param type the type of the network scope in question + * @param direction_in GNUNET_YES if the inbound direction should be summed up, + * otherwise the outbound direction will be summed up + * @return the sum of the utilized bandwidths (in bytes/second) + */ +static unsigned long long +ril_network_get_utilized (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in) +{ + struct RIL_Peer_Agent *cur; + struct RIL_Scope *net; + unsigned long long sum = 0; + + for (cur = solver->agents_head; NULL != cur; cur = cur->next) + { + if (cur->is_active && cur->address_inuse) + { + net = cur->address_inuse->solver_information; + if (net->type == type) + { + if (direction_in) + sum += cur->address_inuse->norm_utilization_in.norm; + else + sum += cur->address_inuse->norm_utilization_out.norm; + } + } + } + + return sum; +} + +/** + * Retrieves the state of the network scope, so that its attributes are up-to- + * date. + * + * @param solver the solver handle + */ static void -ril_calculate_assigned_bwnet (struct GAS_RIL_Handle *solver) +ril_networks_update_state (struct GAS_RIL_Handle *solver) { int c; - struct RIL_Network *net; + struct RIL_Scope *net; for (c = 0; c < solver->networks_count; c++) { net = &solver->network_entries[c]; net->bw_in_assigned = ril_network_get_assigned(solver, net->type, GNUNET_YES); + net->bw_in_utilized = ril_network_get_utilized(solver, net->type, GNUNET_YES); net->bw_out_assigned = ril_network_get_assigned(solver, net->type, GNUNET_NO); + net->bw_out_utilized = ril_network_get_utilized(solver, net->type, GNUNET_NO); + net->active_agent_count = ril_network_count_active_agents(solver, net); + net->social_welfare = ril_network_get_social_welfare(solver, net); } } @@ -1445,17 +1832,23 @@ ril_step_schedule_next (struct GAS_RIL_Handle *solver) offset = (double) solver->parameters.step_time_min.rel_value_us; y = factor * pow (used_ratio, RIL_INTERVAL_EXPONENT) + offset; - GNUNET_assert(y <= (double ) solver->parameters.step_time_max.rel_value_us); - GNUNET_assert(y >= (double ) solver->parameters.step_time_min.rel_value_us); + GNUNET_assert(y <= (double) solver->parameters.step_time_max.rel_value_us); + GNUNET_assert(y >= (double) solver->parameters.step_time_min.rel_value_us); - time_next = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, (unsigned long long) y); + time_next = GNUNET_TIME_relative_saturating_multiply (GNUNET_TIME_UNIT_MICROSECONDS, (unsigned long long) y); + +// LOG (GNUNET_ERROR_TYPE_INFO, "ratio: %f, factor: %f, offset: %f, y: %f\n", +// used_ratio, +// factor, +// offset, +// y); if (solver->simulate) { time_next = GNUNET_TIME_UNIT_ZERO; } - if ((GNUNET_SCHEDULER_NO_TASK == solver->step_next_task_id) && (GNUNET_NO == solver->done)) + if ((NULL == solver->step_next_task_id) && (GNUNET_NO == solver->done)) { solver->step_next_task_id = GNUNET_SCHEDULER_add_delayed (time_next, &ril_step_scheduler_task, solver); @@ -1464,6 +1857,7 @@ ril_step_schedule_next (struct GAS_RIL_Handle *solver) /** * Triggers one step per agent + * * @param solver */ static void @@ -1487,10 +1881,7 @@ ril_step (struct GAS_RIL_Handle *solver) } ril_calculate_discount (solver); - ril_calculate_assigned_bwnet (solver); - - //calculate network state vector -// envi_state_networks(solver); + ril_networks_update_state (solver); //trigger one step per active, unblocked agent for (cur = solver->agents_head; NULL != cur; cur = cur->next) @@ -1508,9 +1899,9 @@ ril_step (struct GAS_RIL_Handle *solver) } } - ril_calculate_assigned_bwnet (solver); + ril_networks_update_state (solver); - solver->step_count += 1; + solver->step_count++; ril_step_schedule_next (solver); ril_inform (solver, GAS_OP_SOLVE_STOP, GAS_STAT_SUCCESS); @@ -1519,51 +1910,30 @@ ril_step (struct GAS_RIL_Handle *solver) for (cur = solver->agents_head; NULL != cur; cur = cur->next) { if (cur->suggestion_issue) { - solver->plugin_envi->bandwidth_changed_cb(solver->plugin_envi->bw_changed_cb_cls, cur->suggestion_address); + solver->env->bandwidth_changed_cb (solver->env->cls, + cur->suggestion_address); cur->suggestion_issue = GNUNET_NO; } } ril_inform (solver, GAS_OP_SOLVE_UPDATE_NOTIFICATION_STOP, GAS_STAT_SUCCESS); } -static int -ril_count_agents (struct GAS_RIL_Handle *solver) -{ - int c = 0; - struct RIL_Peer_Agent *cur_agent; - - for (cur_agent = solver->agents_head; NULL != cur_agent; cur_agent = cur_agent->next) - { - c++; - } - return c; -} - +/** + * Initializes the matrix W of parameter vectors theta with small random numbers. + * + * @param agent The respective agent + */ static void -agent_w_start (struct RIL_Peer_Agent *agent) +agent_w_init (struct RIL_Peer_Agent *agent) { - int count; - struct RIL_Peer_Agent *other; int i; int k; - count = ril_count_agents(agent->envi); - for (i = 0; i < agent->n; i++) { for (k = 0; k < agent->m; k++) { - if (0 == count) { - agent->W[i][k] = agent->envi->parameters.alpha * (1.0 - 2.0*((double) GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX)/(double)UINT32_MAX)); - } - else { - for (other = agent->envi->agents_head; NULL != other; other = other->next) - { - agent->W[i][k] += (other->W[i][k] / (double) count); - } - } - - GNUNET_assert(!isinf(agent->W[i][k])); + agent->W[i][k] = agent->envi->parameters.alpha * (1.0 - 2.0 * ((double) GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX)/(double) UINT32_MAX)); } } } @@ -1586,21 +1956,25 @@ agent_init (void *s, const struct GNUNET_PeerIdentity *peer) agent->peer = *peer; agent->step_count = 0; agent->is_active = GNUNET_NO; - agent->bw_in = MIN_BW; - agent->bw_out = MIN_BW; + agent->bw_in = RIL_MIN_BW; + agent->bw_out = RIL_MIN_BW; agent->suggestion_issue = GNUNET_NO; agent->n = RIL_ACTION_TYPE_NUM; agent->m = 0; agent->W = (double **) GNUNET_malloc (sizeof (double *) * agent->n); + agent->E = (double **) GNUNET_malloc (sizeof (double *) * agent->n); for (i = 0; i < agent->n; i++) { agent->W[i] = (double *) GNUNET_malloc (sizeof (double) * agent->m); + agent->E[i] = (double *) GNUNET_malloc (sizeof (double) * agent->m); } - agent_w_start(agent); + agent_w_init(agent); + agent->eligibility_reset = GNUNET_NO; agent->a_old = RIL_ACTION_INVALID; agent->s_old = GNUNET_malloc (sizeof (double) * agent->m); - agent->e = (double *) GNUNET_malloc (sizeof (double) * agent->m); - agent_modify_eligibility (agent, RIL_E_ZERO, NULL); + agent->address_inuse = NULL; + agent->objective_old = 0; + agent->nop_bonus = 0; return agent; } @@ -1618,11 +1992,12 @@ agent_die (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent) for (i = 0; i < agent->n; i++) { - GNUNET_free(agent->W[i]); + GNUNET_free_non_null(agent->W[i]); + GNUNET_free_non_null(agent->E[i]); } - GNUNET_free(agent->W); - GNUNET_free(agent->e); - GNUNET_free(agent->s_old); + GNUNET_free_non_null(agent->W); + GNUNET_free_non_null(agent->E); + GNUNET_free_non_null(agent->s_old); GNUNET_free(agent); } @@ -1665,12 +2040,12 @@ ril_get_agent (struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity * * @return whether or not the network is considered active */ static int -ril_network_is_active (struct GAS_RIL_Handle *solver, enum GNUNET_ATS_Network_Type network) +ril_network_is_active (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network) { - struct RIL_Network *net; + struct RIL_Scope *net; net = ril_get_network (solver, network); - return net->bw_out_available >= MIN_BW; + return net->bw_out_available >= RIL_MIN_BW; } /** @@ -1698,7 +2073,7 @@ ril_cut_from_vector (void **old, unsigned int bytes_hole; unsigned int bytes_after; - GNUNET_assert(old_length > hole_length); + GNUNET_assert(old_length >= hole_length); GNUNET_assert(old_length >= (hole_start + hole_length)); size = element_size * (old_length - hole_length); @@ -1714,8 +2089,8 @@ ril_cut_from_vector (void **old, else { tmpptr = GNUNET_malloc (size); - memcpy (tmpptr, oldptr, bytes_before); - memcpy (tmpptr + bytes_before, oldptr + (bytes_before + bytes_hole), bytes_after); + GNUNET_memcpy (tmpptr, oldptr, bytes_before); + GNUNET_memcpy (tmpptr + bytes_before, oldptr + (bytes_before + bytes_hole), bytes_after); } if (NULL != *old) { @@ -1737,268 +2112,65 @@ ril_cut_from_vector (void **old, * @param kind the kind to change the preference * @param pref_rel the normalized preference value for this kind over all clients */ -void +static void GAS_ril_address_change_preference (void *solver, - const struct GNUNET_PeerIdentity *peer, - enum GNUNET_ATS_PreferenceKind kind, - double pref_rel) + const struct GNUNET_PeerIdentity *peer, + enum GNUNET_ATS_PreferenceKind kind, + double pref_rel) { LOG(GNUNET_ERROR_TYPE_DEBUG, "API_address_change_preference() Preference '%s' for peer '%s' changed to %.2f \n", GNUNET_ATS_print_preference_type (kind), GNUNET_i2s (peer), pref_rel); - ril_step (solver); + struct GAS_RIL_Handle *s = solver; + + s->parameters.temperature = s->parameters.temperature_init; + s->parameters.epsilon = s->parameters.epsilon_init; + ril_step (s); } + /** - * Entry point for the plugin + * Add a new address for a peer to the solver * - * @param cls pointer to the 'struct GNUNET_ATS_PluginEnvironment' + * The address is already contained in the addresses hashmap! + * + * @param solver the solver Handle + * @param address the address to add + * @param network network type of this address */ -void * -libgnunet_plugin_ats_ril_init (void *cls) +static void +GAS_ril_address_add (void *solver, + struct ATS_Address *address, + uint32_t network) { - struct GNUNET_ATS_PluginEnvironment *env = cls; - struct GAS_RIL_Handle *solver = GNUNET_new (struct GAS_RIL_Handle); - struct RIL_Network * cur; - int c; - char *string; + struct GAS_RIL_Handle *s = solver; + struct RIL_Peer_Agent *agent; + struct RIL_Address_Wrapped *address_wrapped; + struct RIL_Scope *net; + unsigned int m_new; + unsigned int m_old; + unsigned int n_new; + unsigned int n_old; + int i; + unsigned int zero; - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_init() Initializing RIL solver\n"); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "API_address_add()\n"); - GNUNET_assert(NULL != env); - GNUNET_assert(NULL != env->cfg); - GNUNET_assert(NULL != env->stats); - GNUNET_assert(NULL != env->bandwidth_changed_cb); - GNUNET_assert(NULL != env->get_preferences); - GNUNET_assert(NULL != env->get_property); + net = ril_get_network (s, network); + address->solver_information = net; - if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number(env->cfg, "ats", "RIL_DIVISOR", &solver->parameters.divisor)) - { - solver->parameters.divisor = RIL_DEFAULT_DIVISOR; - } - if (GNUNET_OK - != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats", "RIL_STEP_TIME_MIN", - &solver->parameters.step_time_min)) - { - solver->parameters.step_time_min = RIL_DEFAULT_STEP_TIME_MIN; - } - if (GNUNET_OK - != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats", "RIL_STEP_TIME_MAX", - &solver->parameters.step_time_max)) - { - solver->parameters.step_time_max = RIL_DEFAULT_STEP_TIME_MAX; - } - if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_ALGORITHM", &string)) + if (!ril_network_is_active (s, network)) { - solver->parameters.algorithm = !strcmp (string, "SARSA") ? RIL_ALGO_SARSA : RIL_ALGO_Q; - GNUNET_free (string); - } - else - { - solver->parameters.algorithm = RIL_DEFAULT_ALGORITHM; - } - if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_DISCOUNT_BETA", &string)) - { - solver->parameters.beta = strtod (string, NULL); - GNUNET_free (string); - if (!(solver->parameters.beta > 0)) - { - LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_DISCOUNT_BETA not configured as positive number. Set to default value of %f instead.\n", RIL_DEFAULT_DISCOUNT_BETA); - solver->parameters.beta = RIL_DEFAULT_DISCOUNT_BETA; - } - } - else - { - solver->parameters.beta = RIL_DEFAULT_DISCOUNT_BETA; - } - if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_DISCOUNT_GAMMA", &string)) - { - solver->parameters.gamma = strtod (string, NULL); - GNUNET_free (string); - if (!(solver->parameters.gamma < 1) || (solver->parameters.gamma < 0)) - { - LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_DISCOUNT_GAMMA not configured in range [0,1[. Set to default value of %f instead.\n", RIL_DEFAULT_DISCOUNT_GAMMA); - solver->parameters.gamma = RIL_DEFAULT_DISCOUNT_GAMMA; - } - } - else - { - solver->parameters.gamma = RIL_DEFAULT_DISCOUNT_GAMMA; - } - if (GNUNET_OK - == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_GRADIENT_STEP_SIZE", &string)) - { - solver->parameters.alpha = strtod (string, NULL); - GNUNET_free (string); - if (!(solver->parameters.alpha > 0) || solver->parameters.alpha > 1) - { - LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_GRADIENT_STEP_SIZE not configured in range ]0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_GRADIENT_STEP_SIZE); - solver->parameters.alpha = RIL_DEFAULT_GRADIENT_STEP_SIZE; - } - } - else - { - solver->parameters.alpha = RIL_DEFAULT_GRADIENT_STEP_SIZE; - } - if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_TRACE_DECAY", &string)) - { - solver->parameters.lambda = strtod (string, NULL); - GNUNET_free (string); - if (solver->parameters.lambda < 0 || solver->parameters.lambda > 1) - { - LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_TRACE_DECAY not configured in range [0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_TRACE_DECAY); - solver->parameters.lambda = RIL_DEFAULT_TRACE_DECAY; - } - } - else - { - solver->parameters.lambda = RIL_DEFAULT_TRACE_DECAY; - } - if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_EXPLORE_RATIO", &string)) - { - solver->parameters.explore_ratio = strtod (string, NULL); - GNUNET_free (string); - if (solver->parameters.explore_ratio < 0 || solver->parameters.explore_ratio > 1) - { - LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_EXPLORE_RATIO not configured in range [0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_EXPLORE_RATIO); - solver->parameters.explore_ratio = RIL_DEFAULT_EXPLORE_RATIO; - } - } - else - { - solver->parameters.explore_ratio = RIL_DEFAULT_EXPLORE_RATIO; - } - if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_GLOBAL_REWARD_SHARE", &string)) - { - solver->parameters.reward_global_share = strtod (string, NULL); - GNUNET_free (string); - if (solver->parameters.reward_global_share < 0 || solver->parameters.reward_global_share > 1) - { - LOG (GNUNET_ERROR_TYPE_WARNING, "RIL_GLOBAL_REWARD_SHARE not configured in range [0,1]. Set to default value of %f instead.\n", RIL_DEFAULT_GLOBAL_REWARD_SHARE); - solver->parameters.reward_global_share = RIL_DEFAULT_GLOBAL_REWARD_SHARE; - } - } - else - { - solver->parameters.reward_global_share = RIL_DEFAULT_GLOBAL_REWARD_SHARE; - } - if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (env->cfg, "ats", "RIL_SIMULATE", &solver->simulate)) - { - solver->simulate = GNUNET_NO; - } - - env->sf.s_add = &GAS_ril_address_add; - env->sf.s_address_update_property = &GAS_ril_address_property_changed; - env->sf.s_address_update_session = &GAS_ril_address_session_changed; - env->sf.s_address_update_inuse = &GAS_ril_address_inuse_changed; - env->sf.s_address_update_network = &GAS_ril_address_change_network; - env->sf.s_get = &GAS_ril_get_preferred_address; - env->sf.s_get_stop = &GAS_ril_stop_get_preferred_address; - env->sf.s_pref = &GAS_ril_address_change_preference; - env->sf.s_feedback = &GAS_ril_address_preference_feedback; - env->sf.s_del = &GAS_ril_address_delete; - env->sf.s_bulk_start = &GAS_ril_bulk_start; - env->sf.s_bulk_stop = &GAS_ril_bulk_stop; - - solver->plugin_envi = env; - solver->networks_count = env->network_count; - solver->network_entries = GNUNET_malloc (env->network_count * sizeof (struct RIL_Network)); - solver->step_count = 0; - solver->done = GNUNET_NO; - - for (c = 0; c < env->network_count; c++) - { - cur = &solver->network_entries[c]; - cur->type = env->networks[c]; - cur->bw_in_available = env->in_quota[c]; - cur->bw_out_available = env->out_quota[c]; - LOG(GNUNET_ERROR_TYPE_INFO, "init() Quotas for %s network: IN %llu - OUT %llu\n", GNUNET_ATS_print_network_type(cur->type), cur->bw_in_available/1024, cur->bw_out_available/1024); + LOG(GNUNET_ERROR_TYPE_DEBUG, + "API_address_add() Did not add %s address %s for peer '%s', network does not have enough bandwidth\n", + address->plugin, address->addr, GNUNET_i2s (&address->peer)); + return; } - LOG(GNUNET_ERROR_TYPE_INFO, "init() Parameters:\n"); - LOG(GNUNET_ERROR_TYPE_INFO, "init() Algorithm = %s, alpha = %f, beta = %f, lambda = %f\n", - solver->parameters.algorithm ? "Q" : "SARSA", - solver->parameters.alpha, - solver->parameters.beta, - solver->parameters.lambda); - LOG(GNUNET_ERROR_TYPE_INFO, "init() explore = %f, global_share = %f\n", - solver->parameters.explore_ratio, - solver->parameters.reward_global_share); - - return solver; -} - -/** - * Exit point for the plugin - * - * @param cls the solver handle - */ -void * -libgnunet_plugin_ats_ril_done (void *cls) -{ - struct GAS_RIL_Handle *s = cls; - struct RIL_Peer_Agent *cur_agent; - struct RIL_Peer_Agent *next_agent; - - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_done() Shutting down RIL solver\n"); - - s->done = GNUNET_YES; - - cur_agent = s->agents_head; - while (NULL != cur_agent) - { - next_agent = cur_agent->next; - GNUNET_CONTAINER_DLL_remove(s->agents_head, s->agents_tail, cur_agent); - agent_die (s, cur_agent); - cur_agent = next_agent; - } - - if (GNUNET_SCHEDULER_NO_TASK != s->step_next_task_id) - { - GNUNET_SCHEDULER_cancel (s->step_next_task_id); - } - GNUNET_free(s->network_entries); - GNUNET_free(s); - - return NULL; -} - -/** - * Add a new address for a peer to the solver - * - * The address is already contained in the addresses hashmap! - * - * @param solver the solver Handle - * @param address the address to add - * @param network network type of this address - */ -void -GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network) -{ - struct GAS_RIL_Handle *s = solver; - struct RIL_Peer_Agent *agent; - struct RIL_Address_Wrapped *address_wrapped; - struct RIL_Network *net; - unsigned int m_new; - unsigned int m_old; - unsigned int n_new; - unsigned int n_old; - int i; - unsigned int zero; - - LOG (GNUNET_ERROR_TYPE_DEBUG, "API_address_add()\n"); - - net = ril_get_network (s, network); - address->solver_information = net; - - if (!ril_network_is_active (s, network)) - { - LOG(GNUNET_ERROR_TYPE_DEBUG, - "API_address_add() Did not add %s address %s for peer '%s', network does not have enough bandwidth\n", - address->plugin, address->addr, GNUNET_i2s (&address->peer)); - return; - } + s->parameters.temperature = s->parameters.temperature_init; + s->parameters.epsilon = s->parameters.epsilon_init; agent = ril_get_agent (s, &address->peer, GNUNET_YES); @@ -2008,23 +2180,29 @@ GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network GNUNET_CONTAINER_DLL_insert_tail(agent->addresses_head, agent->addresses_tail, address_wrapped); //increase size of W - m_new = agent->m + ((s->parameters.divisor+1) * (s->parameters.divisor+1)); + m_new = agent->m + ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)); m_old = agent->m; n_new = agent->n + 1; n_old = agent->n; GNUNET_array_grow(agent->W, agent->n, n_new); + agent->n = n_old; + GNUNET_array_grow(agent->E, agent->n, n_new); for (i = 0; i < n_new; i++) { if (i < n_old) { agent->m = m_old; GNUNET_array_grow(agent->W[i], agent->m, m_new); + agent->m = m_old; + GNUNET_array_grow(agent->E[i], agent->m, m_new); } else { zero = 0; GNUNET_array_grow(agent->W[i], zero, m_new); + zero = 0; + GNUNET_array_grow(agent->E[i], zero, m_new); } } @@ -2032,9 +2210,6 @@ GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network agent->m = m_old; GNUNET_array_grow(agent->s_old, agent->m, m_new); - agent->m = m_old; - GNUNET_array_grow(agent->e, agent->m, m_new); - ril_try_unblock_agent(s, agent, GNUNET_NO); ril_step (s); @@ -2051,66 +2226,74 @@ GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network * * @param solver the solver handle * @param address the address to remove - * @param session_only delete only session not whole address */ -void -GAS_ril_address_delete (void *solver, struct ATS_Address *address, int session_only) +static void +GAS_ril_address_delete (void *solver, + struct ATS_Address *address) { struct GAS_RIL_Handle *s = solver; struct RIL_Peer_Agent *agent; struct RIL_Address_Wrapped *address_wrapped; - int address_was_used = address->active; int address_index; unsigned int m_new; unsigned int n_new; int i; - struct RIL_Network *net; + struct RIL_Scope *net; - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_address_delete() Delete %s%s %s address %s for peer '%s'\n", - session_only ? "session for " : "", address->active ? "active" : "inactive", address->plugin, - address->addr, GNUNET_i2s (&address->peer)); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "API_address_delete() Delete %s %s address %s for peer '%s'\n", + address->active ? "active" : "inactive", + address->plugin, + address->addr, + GNUNET_i2s (&address->peer)); agent = ril_get_agent (s, &address->peer, GNUNET_NO); if (NULL == agent) { net = address->solver_information; - GNUNET_assert(!ril_network_is_active (s, net->type)); - LOG(GNUNET_ERROR_TYPE_DEBUG, - "No agent allocated for peer yet, since address was in inactive network\n"); + GNUNET_assert(! ril_network_is_active (s, net->type)); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "No agent allocated for peer yet, since address was in inactive network\n"); return; } + s->parameters.temperature = s->parameters.temperature_init; + s->parameters.epsilon = s->parameters.epsilon_init; + address_index = agent_address_get_index (agent, address); - address_wrapped = agent_address_get (agent, address); + address_wrapped = agent_address_get_wrapped (agent, address); if (NULL == address_wrapped) { net = address->solver_information; - GNUNET_assert(!ril_network_is_active (s, net->type)); - LOG(GNUNET_ERROR_TYPE_DEBUG, - "Address not considered by agent, address was in inactive network\n"); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Address not considered by agent, address was in inactive network\n"); return; } - - GNUNET_CONTAINER_DLL_remove(agent->addresses_head, agent->addresses_tail, address_wrapped); - GNUNET_free(address_wrapped); + GNUNET_CONTAINER_DLL_remove (agent->addresses_head, + agent->addresses_tail, + address_wrapped); + GNUNET_free (address_wrapped); //decrease W - m_new = agent->m - ((s->parameters.divisor+1) * (s->parameters.divisor+1)); + m_new = agent->m - ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)); n_new = agent->n - 1; - LOG(GNUNET_ERROR_TYPE_DEBUG, "first\n"); - for (i = 0; i < agent->n; i++) { ril_cut_from_vector ((void **) &agent->W[i], sizeof(double), - address_index * ((s->parameters.divisor+1) * (s->parameters.divisor+1)), - ((s->parameters.divisor+1) * (s->parameters.divisor+1)), agent->m); - } - GNUNET_free(agent->W[RIL_ACTION_TYPE_NUM + address_index]); - LOG(GNUNET_ERROR_TYPE_DEBUG, "second\n"); + address_index * ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), + ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), agent->m); + ril_cut_from_vector ((void **) &agent->E[i], sizeof(double), + address_index * ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), + ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), agent->m); + } + GNUNET_free_non_null(agent->W[RIL_ACTION_TYPE_NUM + address_index]); + GNUNET_free_non_null(agent->E[RIL_ACTION_TYPE_NUM + address_index]); ril_cut_from_vector ((void **) &agent->W, sizeof(double *), RIL_ACTION_TYPE_NUM + address_index, 1, agent->n); + ril_cut_from_vector ((void **) &agent->E, sizeof(double *), RIL_ACTION_TYPE_NUM + address_index, + 1, agent->n); //correct last action if (agent->a_old > (RIL_ACTION_TYPE_NUM + address_index)) { @@ -2120,141 +2303,63 @@ GAS_ril_address_delete (void *solver, struct ATS_Address *address, int session_o { agent->a_old = RIL_ACTION_INVALID; } - //decrease old state vector and eligibility vector - LOG(GNUNET_ERROR_TYPE_DEBUG, "third\n"); + //decrease old state vector ril_cut_from_vector ((void **) &agent->s_old, sizeof(double), - address_index * ((s->parameters.divisor+1) * (s->parameters.divisor+1)), - ((s->parameters.divisor+1) * (s->parameters.divisor+1)), agent->m); - ril_cut_from_vector ((void **) &agent->e, sizeof(double), - address_index * ((s->parameters.divisor+1) * (s->parameters.divisor+1)), - ((s->parameters.divisor+1) * (s->parameters.divisor+1)), agent->m); + address_index * ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), + ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), agent->m); agent->m = m_new; agent->n = n_new; - if (address_was_used) + if (agent->address_inuse == address) { if (NULL != agent->addresses_head) //if peer has an address left, use it { - envi_set_active_suggestion (s, agent, agent->addresses_head->address_naked, MIN_BW, MIN_BW, - GNUNET_NO); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Active address died, suggesting alternative!\n"); + envi_set_active_suggestion (s, + agent, + agent->addresses_head->address_naked, + agent->bw_in, + agent->bw_out, + GNUNET_YES); } else { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Active address died, suggesting disconnect!\n"); envi_set_active_suggestion (s, agent, NULL, 0, 0, GNUNET_NO); } } - ril_step (solver); + if (agent->suggestion_address == address) + { + agent->suggestion_issue = GNUNET_NO; + agent->suggestion_address = NULL; + } + GNUNET_assert (agent->address_inuse != address); } + /** * Update the properties of an address in the solver * * @param solver solver handle * @param address the address - * @param type the ATSI type in HBO - * @param abs_value the absolute value of the property - * @param rel_value the normalized value */ -void +static void GAS_ril_address_property_changed (void *solver, - struct ATS_Address *address, - uint32_t type, - uint32_t abs_value, - double rel_value) -{ - LOG(GNUNET_ERROR_TYPE_DEBUG, - "API_address_property_changed() Property '%s' for peer '%s' address %s changed " - "to %.2f \n", GNUNET_ATS_print_property_type (type), GNUNET_i2s (&address->peer), - address->addr, rel_value); - - ril_step (solver); -} - -/** - * Update the session of an address in the solver - * - * NOTE: values in addresses are already updated - * - * @param solver solver handle - * @param address the address - * @param cur_session the current session - * @param new_session the new session - */ -void -GAS_ril_address_session_changed (void *solver, - struct ATS_Address *address, - uint32_t cur_session, - uint32_t new_session) + struct ATS_Address *address) { - /* - * TODO? Future Work: Potentially add session activity as a feature in state vector - */ - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_address_session_changed()\n"); -} + struct GAS_RIL_Handle *s = solver; -/** - * Notify the solver that an address is (not) actively used by transport - * to communicate with a remote peer - * - * NOTE: values in addresses are already updated - * - * @param solver solver handle - * @param address the address - * @param in_use usage state - */ -void -GAS_ril_address_inuse_changed (void *solver, struct ATS_Address *address, int in_use) -{ - /* - * TODO? Future Work: Potentially add usage variable to state vector - */ LOG(GNUNET_ERROR_TYPE_DEBUG, - "API_address_inuse_changed() Usage for %s address of peer '%s' changed to %s\n", - address->plugin, GNUNET_i2s (&address->peer), (GNUNET_YES == in_use) ? "USED" : "UNUSED"); + "Properties for peer '%s' address changed\n", + GNUNET_i2s (&address->peer)); + s->parameters.temperature = s->parameters.temperature_init; + s->parameters.epsilon = s->parameters.epsilon_init; + ril_step (s); } -/** - * Notify solver that the network an address is located in has changed - * - * NOTE: values in addresses are already updated - * - * @param solver solver handle - * @param address the address - * @param current_network the current network - * @param new_network the new network - */ -void -GAS_ril_address_change_network (void *solver, - struct ATS_Address *address, - uint32_t current_network, - uint32_t new_network) -{ - struct GAS_RIL_Handle *s = solver; - struct RIL_Peer_Agent *agent; - - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_address_change_network() Network type changed, moving " - "%s address of peer %s from '%s' to '%s'\n", - (GNUNET_YES == address->active) ? "active" : "inactive", GNUNET_i2s (&address->peer), - GNUNET_ATS_print_network_type (current_network), GNUNET_ATS_print_network_type (new_network)); - - if (address->active && !ril_network_is_active (solver, new_network)) - { - GAS_ril_address_delete (solver, address, GNUNET_NO); - return; - } - - agent = ril_get_agent (s, &address->peer, GNUNET_NO); - if (NULL == agent) - { - GNUNET_assert(!ril_network_is_active (solver, current_network)); - - GAS_ril_address_add (s, address, new_network); - return; - } - - address->solver_information = ril_get_network(solver, new_network); -} /** * Give feedback about the current assignment @@ -2266,46 +2371,54 @@ GAS_ril_address_change_network (void *solver, * @param kind the kind to change the preference * @param score the score */ -void +static void GAS_ril_address_preference_feedback (void *solver, - void *application, - const struct GNUNET_PeerIdentity *peer, - const struct GNUNET_TIME_Relative scope, - enum GNUNET_ATS_PreferenceKind kind, - double score) + struct GNUNET_SERVICE_Client *application, + const struct GNUNET_PeerIdentity *peer, + const struct GNUNET_TIME_Relative scope, + enum GNUNET_ATS_PreferenceKind kind, + double score) { - LOG(GNUNET_ERROR_TYPE_DEBUG, - "API_address_preference_feedback() Peer '%s' got a feedback of %+.3f from application %s for " - "preference %s for %d seconds\n", GNUNET_i2s (peer), "UNKNOWN", - GNUNET_ATS_print_preference_type (kind), scope.rel_value_us / 1000000); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "API_address_preference_feedback() Peer '%s' got a feedback of %+.3f from application %s for " + "preference %s for %d seconds\n", + GNUNET_i2s (peer), + "UNKNOWN", + GNUNET_ATS_print_preference_type (kind), + scope.rel_value_us / 1000000); } + /** * Start a bulk operation * * @param solver the solver */ -void +static void GAS_ril_bulk_start (void *solver) { struct GAS_RIL_Handle *s = solver; - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_bulk_start() lock: %d\n", s->bulk_lock+1); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "API_bulk_start() lock: %d\n", s->bulk_lock+1); s->bulk_lock++; } + /** * Bulk operation done * * @param solver the solver handle */ -void +static void GAS_ril_bulk_stop (void *solver) { struct GAS_RIL_Handle *s = solver; - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_bulk_stop() lock: %d\n", s->bulk_lock-1); + LOG(GNUNET_ERROR_TYPE_DEBUG, + "API_bulk_stop() lock: %d\n", + s->bulk_lock - 1); if (s->bulk_lock < 1) { @@ -2321,6 +2434,7 @@ GAS_ril_bulk_stop (void *solver) } } + /** * Tell solver to notify ATS if the address to use changes for a specific * peer using the bandwidth changed callback @@ -2331,12 +2445,10 @@ GAS_ril_bulk_stop (void *solver) * @param solver the solver handle * @param peer the identity of the peer */ -const struct ATS_Address * -GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer) +static void +GAS_ril_get_preferred_address (void *solver, + const struct GNUNET_PeerIdentity *peer) { - /* - * activate agent, return currently chosen address - */ struct GAS_RIL_Handle *s = solver; struct RIL_Peer_Agent *agent; @@ -2360,11 +2472,15 @@ GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *p LOG(GNUNET_ERROR_TYPE_DEBUG, "API_get_preferred_address() Activated agent for peer '%s', but no address available\n", GNUNET_i2s (peer)); + s->parameters.temperature = s->parameters.temperature_init; + s->parameters.epsilon = s->parameters.epsilon_init; } - - return agent->address_inuse; + if (NULL != agent->address_inuse) + s->env->bandwidth_changed_cb (s->env->cls, + agent->address_inuse); } + /** * Tell solver stop notifying ATS about changes for this peers * @@ -2374,13 +2490,15 @@ GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *p * @param solver the solver handle * @param peer the peer */ -void -GAS_ril_stop_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer) +static void +GAS_ril_stop_get_preferred_address (void *solver, + const struct GNUNET_PeerIdentity *peer) { struct GAS_RIL_Handle *s = solver; struct RIL_Peer_Agent *agent; - LOG(GNUNET_ERROR_TYPE_DEBUG, "API_stop_get_preferred_address()"); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "API_stop_get_preferred_address()"); agent = ril_get_agent (s, peer, GNUNET_NO); @@ -2395,6 +2513,9 @@ GAS_ril_stop_get_preferred_address (void *solver, const struct GNUNET_PeerIdenti return; } + s->parameters.temperature = s->parameters.temperature_init; + s->parameters.epsilon = s->parameters.epsilon_init; + agent->is_active = GNUNET_NO; envi_set_active_suggestion (s, agent, agent->address_inuse, agent->bw_in, agent->bw_out, @@ -2403,8 +2524,327 @@ GAS_ril_stop_get_preferred_address (void *solver, const struct GNUNET_PeerIdenti ril_step (s); LOG(GNUNET_ERROR_TYPE_DEBUG, - "API_stop_get_preferred_address() Paused agent for peer '%s' with %s address\n", - GNUNET_i2s (peer), agent->address_inuse->plugin); + "API_stop_get_preferred_address() Paused agent for peer '%s'\n", + GNUNET_i2s (peer)); +} + + +/** + * Entry point for the plugin + * + * @param cls pointer to the 'struct GNUNET_ATS_PluginEnvironment' + */ +void * +libgnunet_plugin_ats_ril_init (void *cls) +{ + static struct GNUNET_ATS_SolverFunctions sf; + struct GNUNET_ATS_PluginEnvironment *env = cls; + struct GAS_RIL_Handle *solver = GNUNET_new (struct GAS_RIL_Handle); + struct RIL_Scope * cur; + int c; + char *string; + float f_tmp; + + LOG (GNUNET_ERROR_TYPE_DEBUG, + "API_init() Initializing RIL solver\n"); + + GNUNET_assert (NULL != env); + GNUNET_assert (NULL != env->cfg); + GNUNET_assert (NULL != env->stats); + GNUNET_assert (NULL != env->bandwidth_changed_cb); + GNUNET_assert (NULL != env->get_preferences); + + if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number(env->cfg, "ats", "RIL_RBF_DIVISOR", &solver->parameters.rbf_divisor)) + { + solver->parameters.rbf_divisor = RIL_DEFAULT_RBF_DIVISOR; + } + + if (GNUNET_OK + != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats", "RIL_STEP_TIME_MIN", + &solver->parameters.step_time_min)) + { + solver->parameters.step_time_min = RIL_DEFAULT_STEP_TIME_MIN; + } + + if (GNUNET_OK + != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats", "RIL_STEP_TIME_MAX", + &solver->parameters.step_time_max)) + { + solver->parameters.step_time_max = RIL_DEFAULT_STEP_TIME_MAX; + } + + if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_ALGORITHM", &string)) + { + GNUNET_STRINGS_utf8_toupper (string, string); + if (0 == strcmp (string, "SARSA")) + { + solver->parameters.algorithm = RIL_ALGO_SARSA; + } + if (0 == strcmp (string, "Q-LEARNING")) + { + solver->parameters.algorithm = RIL_ALGO_Q; + } + + GNUNET_free (string); + } + else + { + solver->parameters.algorithm = RIL_DEFAULT_ALGORITHM; + } + + if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_SELECT", &string)) + { + solver->parameters.select = !strcmp (string, "EGREEDY") ? RIL_SELECT_EGREEDY : RIL_SELECT_SOFTMAX; + GNUNET_free (string); + } + else + { + solver->parameters.select = RIL_DEFAULT_SELECT; + } + + + solver->parameters.beta = RIL_DEFAULT_DISCOUNT_BETA; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_DISCOUNT_BETA", &f_tmp)) + { + if (f_tmp < 0.0) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_DISCOUNT_BETA", f_tmp); + } + else + { + solver->parameters.beta = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_DISCOUNT_BETA", f_tmp); + } + } + + solver->parameters.gamma = RIL_DEFAULT_DISCOUNT_GAMMA; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_DISCOUNT_GAMMA", &f_tmp)) + { + if ((f_tmp < 0.0) || (f_tmp > 1.0)) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_DISCOUNT_GAMMA", f_tmp); + } + else + { + solver->parameters.gamma = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_DISCOUNT_GAMMA", f_tmp); + } + } + + solver->parameters.alpha = RIL_DEFAULT_GRADIENT_STEP_SIZE; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_GRADIENT_STEP_SIZE", &f_tmp)) + { + if ((f_tmp < 0.0) || (f_tmp > 1.0)) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_GRADIENT_STEP_SIZE", f_tmp); + } + else + { + solver->parameters.alpha = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_GRADIENT_STEP_SIZE", f_tmp); + } + } + + solver->parameters.lambda = RIL_DEFAULT_TRACE_DECAY; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_TRACE_DECAY", &f_tmp)) + { + if ((f_tmp < 0.0) || (f_tmp > 1.0)) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_TRACE_DECAY", f_tmp); + } + else + { + solver->parameters.lambda = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_TRACE_DECAY", f_tmp); + } + } + + solver->parameters.epsilon_init = RIL_DEFAULT_EXPLORE_RATIO; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_EXPLORE_RATIO", &f_tmp)) + { + if ((f_tmp < 0.0) || (f_tmp > 1.0)) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_EXPLORE_RATIO", f_tmp); + } + else + { + solver->parameters.epsilon_init = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_EXPLORE_RATIO", f_tmp); + } + } + + solver->parameters.epsilon_decay = RIL_DEFAULT_EXPLORE_DECAY; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_EXPLORE_DECAY", &f_tmp)) + { + if ((f_tmp < 0.0) || (f_tmp > 1.0)) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_EXPLORE_DECAY", f_tmp); + } + else + { + solver->parameters.epsilon_decay = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_EXPLORE_DECAY", f_tmp); + } + } + + solver->parameters.temperature_init = RIL_DEFAULT_TEMPERATURE; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_TEMPERATURE", &f_tmp)) + { + if (f_tmp <= 0.0) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_TEMPERATURE", f_tmp); + } + else + { + solver->parameters.temperature_init = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_TEMPERATURE", f_tmp); + } + } + + solver->parameters.temperature_decay = RIL_DEFAULT_TEMPERATURE_DECAY; + if (GNUNET_SYSERR != GNUNET_CONFIGURATION_get_value_float (env->cfg, "ats", + "RIL_TEMPERATURE_DECAY", &f_tmp)) + { + if ((f_tmp <= 0.0) || solver->parameters.temperature_decay > 1) + { + LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"), + "RIL_TEMPERATURE_DECAY", f_tmp); + } + else + { + solver->parameters.temperature_decay = f_tmp; + LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n", + "RIL_TEMPERATURE_DECAY", f_tmp); + } + } + + if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (env->cfg, "ats", "RIL_SIMULATE", &solver->simulate)) + { + solver->simulate = GNUNET_NO; + } + + if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(env->cfg, "ats", "RIL_REPLACE_TRACES")) + { + solver->parameters.eligibility_trace_mode = RIL_E_REPLACE; + } + else + { + solver->parameters.eligibility_trace_mode = RIL_E_ACCUMULATE; + } + + if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_SOCIAL_WELFARE", &string)) + { + solver->parameters.social_welfare = !strcmp (string, "NASH") ? RIL_WELFARE_NASH : RIL_WELFARE_EGALITARIAN; + GNUNET_free (string); + } + else + { + solver->parameters.social_welfare = RIL_DEFAULT_WELFARE; + } + + solver->env = env; + sf.cls = solver; + sf.s_add = &GAS_ril_address_add; + sf.s_address_update_property = &GAS_ril_address_property_changed; + sf.s_get = &GAS_ril_get_preferred_address; + sf.s_get_stop = &GAS_ril_stop_get_preferred_address; + sf.s_pref = &GAS_ril_address_change_preference; + sf.s_feedback = &GAS_ril_address_preference_feedback; + sf.s_del = &GAS_ril_address_delete; + sf.s_bulk_start = &GAS_ril_bulk_start; + sf.s_bulk_stop = &GAS_ril_bulk_stop; + + solver->networks_count = env->network_count; + solver->network_entries = GNUNET_malloc (env->network_count * sizeof (struct RIL_Scope)); + solver->step_count = 0; + solver->done = GNUNET_NO; + + for (c = 0; c < env->network_count; c++) + { + cur = &solver->network_entries[c]; + cur->type = c; + cur->bw_in_available = env->in_quota[c]; + cur->bw_out_available = env->out_quota[c]; + LOG (GNUNET_ERROR_TYPE_DEBUG, + "init() Quotas for %s network: IN %llu - OUT %llu\n", + GNUNET_NT_to_string(cur->type), + cur->bw_in_available/1024, + cur->bw_out_available/1024); + } + + LOG(GNUNET_ERROR_TYPE_DEBUG, "init() Parameters:\n"); + LOG(GNUNET_ERROR_TYPE_DEBUG, "init() Algorithm = %s, alpha = %f, beta = %f, lambda = %f\n", + solver->parameters.algorithm ? "Q" : "SARSA", + solver->parameters.alpha, + solver->parameters.beta, + solver->parameters.lambda); + LOG(GNUNET_ERROR_TYPE_DEBUG, "init() exploration_ratio = %f, temperature = %f, ActionSelection = %s\n", + solver->parameters.epsilon, + solver->parameters.temperature, + solver->parameters.select ? "EGREEDY" : "SOFTMAX"); + LOG(GNUNET_ERROR_TYPE_DEBUG, "init() RBF_DIVISOR = %llu\n", + solver->parameters.rbf_divisor); + + return &sf; } + +/** + * Exit point for the plugin + * + * @param cls the solver handle + */ +void * +libgnunet_plugin_ats_ril_done (void *cls) +{ + struct GNUNET_ATS_SolverFunctions *sf = cls; + struct GAS_RIL_Handle *s = sf->cls; + struct RIL_Peer_Agent *cur_agent; + struct RIL_Peer_Agent *next_agent; + + LOG(GNUNET_ERROR_TYPE_DEBUG, "API_done() Shutting down RIL solver\n"); + + s->done = GNUNET_YES; + + cur_agent = s->agents_head; + while (NULL != cur_agent) + { + next_agent = cur_agent->next; + GNUNET_CONTAINER_DLL_remove(s->agents_head, s->agents_tail, cur_agent); + agent_die (s, cur_agent); + cur_agent = next_agent; + } + + if (NULL != s->step_next_task_id) + { + GNUNET_SCHEDULER_cancel (s->step_next_task_id); + } + GNUNET_free(s->network_entries); + GNUNET_free(s); + + return NULL; +} + + /* end of plugin_ats_ril.c */