a3c.py: fix self.reward_buf bug

rename send/recv_ts_diff to send/recv_interval
add more .gitignore

Author: francisyyan

a3c/a3c.py

@@ -11,16 +11,16 @@
 def normalize_states(states):
     norm_states = np.array(states, dtype=np.float32)
 
-    # queuing_delay, target range [0, 200]
-    queuing_delays = norm_states[:, 0]
-    queuing_delays /= 100.0
-    queuing_delays -= 1.0
+    # queuing_delay, target range [0, 2000]
+    queuing_delay = norm_states[:, 0]
+    queuing_delay /= 1000.0
+    queuing_delay -= 1.0
 
-    # send_ts_diff and recv_ts_diff, target range [0, 100]
+    # send_interval and recv_interval, target range [0, 500]
     for i in [1, 2]:
-        ts_diffs = norm_states[:, i]
-        ts_diffs /= 50.0
-        ts_diffs -= 1.0
+        interval = norm_states[:, i]
+        interval /= 250.0
+        interval -= 1.0
 
     # cwnd, target range [0, 100]
     cwnd = norm_states[:, 3]
@@ -109,7 +109,7 @@ def build_loss(self):
         entropy = -tf.reduce_mean(pi.action_probs * log_action_probs)
 
         # total loss and gradients
-        loss = policy_loss + 0.5 * value_loss - 0.2 * entropy
+        loss = policy_loss + 0.5 * value_loss - 0.01 * entropy
         grads = tf.gradients(loss, pi.trainable_vars)
         grads, _ = tf.clip_by_global_norm(grads, 10.0)
 
@@ -196,10 +196,10 @@ def rollout(self):
         if self.gamma == 1.0:
             self.reward_buf = np.full(episode_len, final_reward)
         else:
-            reward_buf = np.zeros(episode_len)
-            reward_buf[-1] = final_reward
+            self.reward_buf = np.zeros(episode_len)
+            self.reward_buf[-1] = final_reward
             for i in reversed(xrange(episode_len - 1)):
-                reward_buf[i] = reward_buf[i + 1] * self.gamma
+                self.reward_buf[i] = self.reward_buf[i + 1] * self.gamma
 
         # compute advantages
         self.adv_buf = self.reward_buf - np.asarray(self.value_buf)

a3c/worker.py

@@ -26,7 +26,7 @@ def prepare_traces(bandwidth):
         uplink_trace = path.join(trace_dir, '%dmbps.trace' % bandwidth)
         downlink_trace = uplink_trace
     else:
-        trace_path = '/usr/share/mahimahi/traces/' + bandwidth
+        trace_path = path.join(trace_dir, bandwidth)
         # intentionally switch uplink and downlink traces due to sender first
         uplink_trace = trace_path + '.down'
         downlink_trace = trace_path + '.up'
@@ -35,9 +35,9 @@ def prepare_traces(bandwidth):
 
 
 def create_env(task_index):
-    bandwidth = 12  # or 'Verizon-LTE-driving'
+    bandwidth = 12
     delay = 20
-    queue = 200  # or None
+    queue = 200
 
     uplink_trace, downlink_trace = prepare_traces(bandwidth)
     mm_cmd = ('mm-delay %d mm-link %s %s' %

env/sender.py

@@ -133,8 +133,8 @@ def update_state(self, ack):
             self.prev_send_ts = send_ts
             self.prev_recv_ts = recv_ts
 
-        send_ts_diff = send_ts - self.prev_send_ts
-        recv_ts_diff = recv_ts - self.prev_recv_ts
+        send_interval = send_ts - self.prev_send_ts
+        recv_interval = recv_ts - self.prev_recv_ts
         self.prev_send_ts = send_ts
         self.prev_recv_ts = recv_ts
 
@@ -152,7 +152,8 @@ def update_state(self, ack):
             if curr_ts_ms() - self.runtime_start > self.max_runtime:
                 self.running = False
 
-        return [queuing_delay, send_ts_diff, recv_ts_diff, self.cwnd]
+        state = [queuing_delay, send_interval, recv_interval, self.cwnd]
+        return state
 
     def take_action(self, action):
         self.cwnd += self.action_mapping[action]

helpers/.gitignore

@@ -0,0 +1,2 @@
+*.trace
+TABLE