Reinforcement Learning Example
public class SimpleWorldQLearningAgent extends SimpleWorldRobotController {
private TDLearner brain;
private Action left;
private Action right;
private Action forward;
private int prevLifeForce;
/**
* Setup the TDLearner for use with a SimpleWorld agent.
*/
public void startUp() {
// define the actions that the agent may take.
ArrayList<Action> actions = new ArrayList<Action>();
left = new StringAction("TurnLeft");
actions.add(left);
right = new StringAction("TurnRight");
actions.add(right);
forward = new StringAction("GoForward");
actions.add(forward);
// Construct the Q-Table with initial value 1 for each entry.
QTable qTable = new QTable(new ConstantQInitializationPolicy(1));
// Select a random action 5% of the time, otherwise use greedy strategy.
ActionSelectionPolicy selector =
new EpsilonGreedyActionSelectionPolicy(qTable, 0.05);
// Update the Q-Table using Q-Learning.
LearningRateFunction lr = new ConstantLearningRate(0.01);
QUpdatePolicy updater = new QLearningPolicy(qTable, lr, 0.75);
// Create the TDLearner that will be used by the agent.
brain = new TDLearner(actions, qTable, selector, updater);
prevLifeForce = 0;
}
/**
* Every time step happens, use the TDLearner to select the action to be
* taken. This also trains the TDLearner based on the agent's experiences.
*/
public void step(SimpleWorldRobot robot) {
SimpleWorldAgentState s =
new SimpleWorldAgentState(robot.getLeftColor(),
robot.getFrontColor(), robot.getRightColor(), robot.getSmell());
Action a = null;
if (prevLifeForce != 0 && robot.getLifeForce() > prevLifeForce) {
a = brain.getNextAction(s, 50); // found cheese!
}
else {
a = brain.getNextAction(s, 0);
}
prevLifeForce = robot.getLifeForce();
if (a.equals(left)) {
robot.rotateLeft();
}
else if (a.equals(right)) {
robot.rotateRight();
}
else {
robot.moveForward();
}
}
/*
* A State sub-class to make it easier to work with the SimpleWorld
* agent's sensor values.
*/
private static class SimpleWorldAgentState extends StringState {
private static final long serialVersionUID = 1L;
public SimpleWorldAgentState(int left, int front, int right, int smell) {
super(left + "" + front + "" + right + "" + smell);
}
}
}
The example below illustrates the use of dLife’s reinforcement learning package by having an agent learn to find food in dLife’s Microworld.