commit a80cf00c8d884ddec4302f06badaeb00c7533ca9
Author: Marcos Assuncao <assuncao@acm.org>
Date:   Mon Sep 8 17:52:17 2014 +0200

    kadeploy plot script

diff --git a/papers/2014/reservation/experiments/src/python/lib/kadeploy.py b/papers/2014/reservation/experiments/src/python/lib/kadeploy.py
index ae991ea..b16bf95 100644
--- a/papers/2014/reservation/experiments/src/python/lib/kadeploy.py
+++ b/papers/2014/reservation/experiments/src/python/lib/kadeploy.py
@@ -19,7 +19,8 @@ TZINFO = pytz.timezone("Europe/Paris")
 
 class DeploymentStat:
 
-    def __init__(self, host, start, step1_dur, step2_dur, step3_dur, success):
+    def __init__(self, id, host, start, step1_dur, step2_dur, step3_dur, success):
+        self._id = id
         self._host = host
         self._start = start
         self._step1_dur = int(step1_dur)
@@ -29,6 +30,16 @@ class DeploymentStat:
         self._retry_1 = 0
         self._retry_2 = 0
         self._retry_3 = 0
+        self._cluster = None
+        self._env = None
+
+    @property
+    def id(self):
+        return self._id
+
+    @id.setter
+    def id(self, id):
+        self._id = id
 
     @property
     def hostname(self):
@@ -56,7 +67,8 @@ class DeploymentStat:
 
     @property
     def success(self):
-        return self._success
+        success = (self._step1_dur > 5 and self.step_2_duration > 10 and self.step_3_duration > 10)
+        return self._success and success
 
     @property
     def num_retries_step_1(self):
@@ -88,6 +100,72 @@ class DeploymentStat:
                 self.num_retries_step_3 > 0 or
                 self.num_retries_step_3 > 0)
 
+    @property
+    def cluster(self):
+        return self._cluster
+
+    @cluster.setter
+    def cluster(self, c):
+        self._cluster = c
+
+    @property
+    def environment(self):
+        return self._env
+
+    @environment.setter
+    def environment(self, env):
+        self._env = env
+
+
+class Deployment(list):
+    """
+    Stores information about an entire deployment,
+    with stats about deployment on all nodes
+    """
+
+    def __init__(self, id):
+        super(Deployment, self).__init__()
+        self._id = id
+
+    def append_stat(self, stat):
+        """
+        Adds a deployment stat to this list
+        :param stat: the stat to be added
+        :return: nil
+        """
+        self.append(stat)
+
+    @property
+    def id(self):
+        """
+        Returns the deployment ID
+        :return: the deployment ID
+        """
+        return self._id
+
+    @property
+    def duration(self):
+        """
+        Returns the time length of the longest deployment stat
+        :return: the deployment duration
+        """
+        last_stat = 0
+        for stat in self:
+            last_stat = max(last_stat, stat.deployment_duration)
+        return last_stat
+
+    @property
+    def success(self):
+        """
+        Checks whether the deployment has been successful
+        :return: False if any deployment on a machine failed
+        """
+        for stat in self:
+            if not stat.success:
+                return False
+        return True
+
+
 def parse_date(str_date):
     d = datetime.strptime(str_date, "%Y-%m-%d %H:%M:%S")
     d = TZINFO.localize(d)
@@ -99,10 +177,10 @@ def parse_kdeploy_opt():
                         help='the input request trace file')
     parser.add_argument('--output', dest='output', type=str, required=True,
                         help='the output directory')
-    # parser.add_argument('--start-date', dest='start_date', type=parse_date, required=True,
-    #                     help='the start date of the log (i.e. YYYY-MM-DD HH:mm:ss)')
     parser.add_argument('--cluster', dest='cluster', type=str, required=True,
                         help='the name of the cluster')
+    parser.add_argument('--plot', dest='plot', type=str, required=True, choices=['deployment'],
+                        help='type of graphs that should be plotted')
 
     args = parser.parse_args()
     return args
@@ -110,9 +188,10 @@ def parse_kdeploy_opt():
 def parse_log(log, cluster):
     f = open(log, 'r')
 
-    deployments = []
+    stats = []
     for line in f:
         fields = line.split(',')
+        id = fields[0]
         hostname = fields[2]
 
         if cluster in hostname:
@@ -122,25 +201,26 @@ def parse_log(log, cluster):
             step2_dur = int(fields[5])
             step3_dur = int(fields[6])
             success = "true" in fields[10]
-            dep = DeploymentStat(hostname, start, step1_dur, step2_dur, step3_dur, success)
+            env = fields[11].strip()
+            dep = DeploymentStat(id, hostname, start, step1_dur, step2_dur, step3_dur, success)
             retry_1 = int(fields[7])
             retry_2 = int(fields[8])
             retry_3 = int(fields[9])
             dep.num_retries_step_1 = retry_1
             dep.num_retries_step_2 = retry_2
             dep.num_retries_step_3 = retry_3
-            deployments.append(dep)
+            dep.cluster = cluster
+            dep.environment = env
+            stats.append(dep)
 
     f.close()
-    return deployments
+    return stats
 
-def plot_deployment_time(deployments, cluster, out_dir):
+def plot_deployment_time(stats, cluster, out_dir):
     values = []
-    for d in deployments:
+    for d in stats:
         if d.success and not d.has_retries:
-            value = d.deployment_duration
-            if d.step_1_duration > 5 and d.step_2_duration > 10 and d.step_3_duration > 10:
-                values.append(value)
+            values.append(d.deployment_duration)
 
     # title = 'Deployment on Cluster %s' % cluster
     title = 'Deployment on %s' % cluster
@@ -148,17 +228,54 @@ def plot_deployment_time(deployments, cluster, out_dir):
     hist_graph(values, title, 'Deployment Time (seconds)', '', out_file)
 
 
-def plot_boot_time(deployments, cluster, out_dir):
-    values = []
-    for d in deployments:
-        if d.success and not d.has_retries:
-            if d.step_3_duration > 30:
-                values.append(d.step_3_duration)
+def group_by_deployment(stats, cluster):
+    """
+    Group deployment stats by deployment.
+
+    :param stats: a list of deployment stats
+    :param cluster: the name of the cluster to consider
+    :return: a dictionary of <id, deployment> values
+    """
+    deployments = {}
+    for s in stats:
+        if s.cluster != cluster:
+            continue
+
+        if s.id in deployments:
+            dep = deployments[s.id]
+        else:
+            dep = Deployment(s.id)
+            deployments[s.id] = dep
+        dep.append_stat(s)
+    return deployments
+
 
-    title = 'Machine Boot Time on Cluster %s' % (cluster)
-    out_file = out_dir + ("/boot_time_%s.pdf" % (cluster))
-    hist_graph(values, title, 'Boot Time (seconds)', '', out_file)
+def plot_deployment_size(deployments, cluster):
+    sizes = {}
+    for d in deployments.itervalues():
+        size = len(d)
+        if size not in sizes:
+            sizes[size] = []
 
+        if size > 0 and d.success:
+            sizes[size].append(d.duration)
+
+    fig, ax = plt.subplots()
+    x = []
+    means = []
+    stds = []
+    for size, value in sizes.iteritems():
+        if len(value) > 0:
+            x.append(size)
+            means.append(np.mean(value))
+            stds.append(np.std(value))
+
+    ax.set_xlim(0, np.max(x) + 1)
+    ax.set_ylabel('Deployment Duration (s)')
+    ax.set_xlabel('Deployment Size (\# Machines)')
+
+    plt.errorbar(x, means, yerr=stds)
+    plt.show()
 
 def hist_graph(values, title, x_label, y_label, out_file):
     ar1 = np.array(values)
@@ -211,8 +328,12 @@ def hist_graph(values, title, x_label, y_label, out_file):
     #plt.show()
     plt.savefig(out_file)
 
+
 def plot_graphs():
     opts = parse_kdeploy_opt()
-    deployments = parse_log(opts.input, opts.cluster)
-    plot_deployment_time(deployments, opts.cluster, opts.output)
-#    plot_boot_time(deployments, opts.cluster, opts.output)
+    if opts.plot == 'deployment':
+        print 'Printing deployment graphs...'
+        stats = parse_log(opts.input, opts.cluster)
+        # plot_deployment_time(stats, opts.cluster, opts.output)
+        deployments = group_by_deployment(stats, opts.cluster)
+        plot_deployment_size(deployments, opts.cluster)
