Modified histogram shell plot script, added python dynamics plot script

Supports:

- bwe_plot.sh
---- Histogram plots
---- Baseline histograms
---- Vertical error lines
---- Dashed horizontal lines
---- Different colors for different algorithms
---- Legends read as input
---- Extra horizontal space in case there are TCP flows plotted
---- Multiple windows
---- Auto vertical scale, except for latency plots which are kept constant for all plots

- plot_dynamics.py
---- Dynamic plots
---- Different colors for different flows and algorithms
---- Dashed line plot for available capacity
---- Throughput, latency and loss on separated boxes

R=stefan@webrtc.org

Review URL: https://codereview.webrtc.org/1237903003 .

Cr-Commit-Position: refs/heads/master@{#9614}
diff --git a/webrtc/modules/remote_bitrate_estimator/test/plot_bars.sh b/webrtc/modules/remote_bitrate_estimator/test/plot_bars.sh
new file mode 100644
index 0000000..e733bf6
--- /dev/null
+++ b/webrtc/modules/remote_bitrate_estimator/test/plot_bars.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+
+# Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+# To set up in e.g. Eclipse, run a separate shell and pipe the output from the
+# test into this script.
+#
+# In Eclipse, that amounts to creating a Run Configuration which starts
+# "/bin/bash" with the arguments "-c [trunk_path]/out/Debug/modules_unittests
+# --gtest_filter=*BweTest* | [trunk_path]/webrtc/modules/
+# remote_bitrate_estimator/test/plot_bars.sh
+
+# This script supports multiple figures (windows), the figure is specified as an
+# identifier at the first argument after the PLOT command. Each figure has a
+# single y axis and a dual y axis mode. If any line specifies an axis by ending
+# with "#<axis number (1 or 2)>" two y axis will be used, the first will be
+# assumed to represent bitrate (in kbps) and the second will be assumed to
+# represent time deltas (in ms).
+
+log=$(</dev/stdin)
+
+# Plot histograms.
+function gen_gnuplot_bar_input {
+  x_start=1
+  x_end=3.75
+  bars=$(echo "$log" | grep "BAR")
+
+  labels=$(echo "$log" | grep "^LABEL")
+  figures=($(echo "$bars" | cut -f 2 | sort | uniq))
+
+  echo "reset"
+
+  echo "set title font 'Verdana,22'"
+  echo "set xtics font 'Verdana,24'"
+  echo "set ytics font 'Verdana,14'"
+  echo "set ylabel font 'Verdana,16'"
+
+  echo "set xrange[$x_start:$x_end]"
+  echo "set style fill solid 0.5"
+  echo "set style fill solid border -1"
+
+  ydist=11  # Used to correctly offset the y label.
+  for figure in "${figures[@]}" ; do
+
+    echo "set terminal wxt $figure size 440,440 dashed"
+    echo "set ylabel offset $ydist, -3"
+    ((ydist--))
+
+    title=$(echo "$labels" | grep "^LABEL.$figure" | cut -f 3 | \
+                                         head -n 1 | sed 's/_/ /g')
+    y_label=$(echo "$labels" | grep "^LABEL.$figure" | cut -f 4 | \
+                                         head -n 1 | sed 's/_/ /g')
+
+    # RMCAT flows.
+    num_flows=$(echo "$labels" | grep "^LABEL.$figure" | cut -f 5 | \
+                                         head -n 1)
+
+    # RMCAT algorithm 1.
+    x_label_1=$(echo "$log" | grep "BAR.$figure" | cut -f 3 | sed 's/_/\t/g' \
+                                 | cut -f 1  | sort | uniq | head -n 1 )
+
+    # RMCAT algorithm 2.
+    x_label_2=$(echo "$log" | grep "BAR.$figure" | cut -f 3 | sed 's/_/\t/g' \
+                                 | cut -f 1  | sort | uniq |  sed -n 2p)
+
+    x_labels="('$x_label_1' 2, '$x_label_2' 3)"
+    tcp_flow=false
+
+    tcp_space=0.2  # Extra horizontal space between bars.
+
+    # Parse labels if there are other flows in addition to RMCAT ones.
+    IFS='x' read -ra split_label_1 <<< "$x_label_1"
+
+    if (( ${#split_label_1[@]} > "1" )); then
+      tcp_flow=true
+      box_width=$(echo "(1.0-$tcp_space/2)/$num_flows" | bc -l)
+      echo "set xtics font 'Verdana,16'"
+      x_labels="("
+      delimiter=""
+      abscissa=$(echo $x_start + 0.5 + 0.5*$box_width | bc)
+      for label in "${split_label_1[@]}" ; do
+        x_labels+="$delimiter'$label' $abscissa"
+        abscissa=$(echo $abscissa + $box_width | bc)
+        delimiter=", "
+      done
+      abscissa=$(echo $abscissa + $tcp_space | bc)
+      IFS='x' read -ra split_label_2 <<< "$x_label_2"
+      for label in "${split_label_2[@]}" ; do
+        x_labels+="$delimiter'$label' $abscissa"
+        abscissa=$(echo $abscissa + $box_width | bc)
+      done
+      x_labels="$x_labels)"
+    else
+      box_width=$(echo 1.0/$num_flows | bc -l)
+    fi
+
+    echo "set boxwidth $box_width"
+
+    # Plots can be directly exported to image files.
+    file_name=$(echo "$labels" | grep "^LABEL.$figure" | cut -f 5 | head -n 1)
+
+    y_max=0  # Used to scale the plot properly.
+
+    # Since only the optimal bitrate for the first flow is being ploted,
+    # consider only this one for scalling purposes.
+    data_sets=$(echo "$bars" | grep "LIMITERRORBAR.$figure" | cut -f 3 | \
+                                   sed 's/_/\t/g' | cut -f 1 | sort | uniq)
+
+    if (( ${#data_sets} > "0" )); then
+      for set in $data_sets ; do
+        y=$(echo "$bars" | grep "LIMITERRORBAR.$figure.$set" | cut -f 8 | \
+                                                                head -n 1)
+        if (( $(bc <<< "$y > $y_max") == 1 )); then
+          y_max=$y
+        fi
+      done
+    fi
+
+    data_sets=$(echo "$bars" | grep "ERRORBAR.$figure" | cut -f 3 | sort | uniq)
+    if (( ${#data_sets} > "0" )); then
+      for set in $data_sets ; do
+        y=$(echo "$bars" | grep "ERRORBAR.$figure.$set" | cut -f 6 | head -n 1)
+        if (( $(bc <<< "$y > $y_max") == 1 )) ; then
+          y_max=$y
+        fi
+      done
+    fi
+
+    data_sets=$(echo "$bars" | grep "BAR.$figure" | cut -f 3 | sort | uniq)
+
+    for set in $data_sets ; do
+      y=$(echo "$bars" | grep "BAR.$figure.$set" | cut -f 4 | head -n 1)
+      if (( $(bc <<< "$y > $y_max") == 1 )) ; then
+        y_max=$y
+      fi
+    done
+
+    y_max=$(echo $y_max*1.1 | bc)
+
+    # Scale all latency plots with the same vertical scale.
+    delay_figure=5
+    if (( $figure==$delay_figure )) ; then
+      y_max=250
+    fi
+
+    echo "set ylabel \"$y_label\""
+    echo "set yrange[0:$y_max]"
+
+    echo "set multiplot"
+
+    # Plot bars.
+    data_sets=$(echo "$bars" | grep "BAR.$figure" | cut -f 3 | sort | uniq)
+
+    echo "set xtics $x_labels"
+    echo "plot '-' using 1:4:2 with boxes lc variable notitle"
+
+    echo
+
+    color=11  # Green.
+    x_bar=$(echo $x_start + 0.5 + 0.5*$box_width | bc)
+    for set in $data_sets ; do
+      echo -n "$x_bar  $color  "
+      echo "$bars" | grep "BAR.$figure.$set" | cut -f 3,4
+
+      # Add extra space if TCP flows are being plotted.
+      if $tcp_flow && \
+          (( $(bc <<< "$x_bar < $x_start + 1.5 - 0.5*$tcp_space") == 1 )) && \
+          (( $(bc <<< "$x_bar + $box_width > $x_start + 1.5 + 0.5*$tcp_space") \
+           == 1 )); then
+          x_bar=$(echo $x_bar + $tcp_space | bc)
+      fi
+
+      x_bar=$(echo $x_bar + $box_width | bc)
+
+      if (( $(bc <<< "$x_bar > 2.5") == 1 )) ; then
+        color=12  # Blue.
+      fi
+      # Different bar color for TCP flows:
+      if $tcp_flow && \
+         (( $(bc <<< "(100*$x_bar)%100 < 50") == 1 ))
+      then
+        color=18  # Gray.
+      fi
+    done
+    echo "e"
+
+    # Plot Baseline bars, e.g. one-way path delay on latency plots.
+    data_sets=$(echo "$log" | grep "BASELINE.$figure" | cut -f 3 | sort | uniq)
+
+    echo "set xtics $x_labels"
+    echo "plot '-' using 1:4:2 with boxes lc variable notitle"
+
+    echo
+
+    color=18  # Gray.
+    x_bar=$(echo $x_start + 0.5 + 0.5*$box_width | bc)
+    for set in $data_sets ; do
+      echo -n "$x_bar  $color  "
+      echo "$log" | grep "BASELINE.$figure.$set" | cut -f 3,4
+
+      # Add extra space if TCP flows are being plotted.
+      if $tcp_flow && \
+          (( $(bc <<< "$x_bar < $x_start + 1.5 - 0.5*$tcp_space") == 1 )) && \
+          (( $(bc <<< "$x_bar + $box_width > $x_start + 1.5 + 0.5*$tcp_space") \
+           == 1 )); then
+          x_bar=$(echo $x_bar + $tcp_space | bc)
+      fi
+
+      x_bar=$(echo $x_bar + $box_width | bc)
+
+    done
+    echo "e"
+
+    # Plot vertical error lines, e.g. y +- sigma.
+    data_sets=$(echo "$bars" | grep "ERRORBAR.$figure" | cut -f 3 | sort | uniq)
+
+    if (( ${#data_sets} > "0" )); then
+
+      echo "set key left"
+      error_title=$(echo "$bars" | grep "ERRORBAR.$figure" | cut -f 7 | \
+                                                 head -n 1 | sed 's/_/ /g')
+
+      echo "set xtics $x_labels"
+      echo "plot '-' using 1:3:4:5 title '$error_title' with yerr"
+
+      x_error_line=$(echo $x_start + 0.5 + 0.5*$box_width | bc)
+      for set in $data_sets ; do
+        echo -n "$x_error_line  "
+        echo "$bars" | grep "ERRORBAR.$figure.$set" | cut -f 3,4,5,6
+
+        # Add extra space if TCP flows are being plotted.
+        if $tcp_flow && \
+          (( $(bc <<< "$x_error_line < $x_start + 1.5 - 0.5*$tcp_space") == 1 \
+          )) && (( $(bc <<< "$x_error_line + $box_width > $x_start + 1.5 \
+          + 0.5*$tcp_space") == 1 )); then
+          x_error_line=$(echo $x_error_line + $tcp_space | bc)
+        fi
+
+        x_error_line=$(echo $x_error_line + $box_width | bc)
+      done
+      echo "e"
+    fi
+
+    # Plot horizontal dashed lines, e.g. y = optimal bitrate.
+    data_sets=$(echo "$bars" | grep "LIMITERRORBAR.$figure" | cut -f 3 \
+                                                     | sort | uniq)
+    if (( ${#data_sets} > "0" )); then
+
+      echo "set style line 1 lt 1 lw 3 pt 3 ps 0 linecolor rgb 'black'"
+
+      limit_titles=$(echo "$bars" | grep "LIMITERRORBAR.$figure" | cut -f 9 \
+                                                         | sort | uniq)
+
+      for title in $limit_titles ; do
+        y_max=$(echo "$bars" | grep "LIMITERRORBAR.$figure" | grep "$title" \
+                                               | cut -f 8 | head -n 1)
+
+        retouched_title=$(echo "$title" | sed 's/#/\t/g' | cut -f 1 \
+                                                         | sed 's/_/ /g')
+
+        echo "set key right top"
+        echo "set xtics $x_labels"
+        echo "plot $y_max lt 7 lw 1 linecolor rgb 'black' \
+                                  title '$retouched_title'"
+      done
+
+    fi
+
+    echo "unset multiplot"
+  done
+}
+gen_gnuplot_bar_input | gnuplot -persist
diff --git a/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py b/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py
new file mode 100644
index 0000000..f825be4
--- /dev/null
+++ b/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+# This script is used to plot simulation dynamics.
+# Able to plot each flow separately. Other plot boxes can be added,
+# currently one for Throughput, one for Latency and one for Packet Loss.
+
+import matplotlib
+import matplotlib.pyplot as plt
+import math
+import numpy
+import re
+import sys
+
+
+class Variable:
+  def __init__(self, variable):
+    self._ID = variable[0]
+    self._xlabel = variable[1]
+    self._ylabel = variable[2]
+    self._subplot = variable[3]
+    self._y_max = variable[4]
+    self._samples = dict()
+
+  def getID(self):
+    return self._ID
+
+  def getXLabel(self):
+    return self._xlabel
+
+  def getYLabel(self):
+    return self._ylabel
+
+  def getSubplot(self):
+    return self._subplot
+
+  def getYMax(self):
+    return self._y_max
+
+  def getNumberOfFlows(self):
+    return len(self._samples)
+
+
+  def addSample(self, line):
+    groups = re.search(r'/\d_(\w+)#\d@(\w*)', line)
+    var_name = groups.group(1)
+    alg_name = groups.group(2)
+
+    if alg_name not in self._samples.keys():
+      self._samples[alg_name] = {}
+
+    if var_name not in self._samples[alg_name].keys():
+      self._samples[alg_name][var_name] = []
+
+    sample = re.search(r'(\d+\.\d+)\t([-]?\d+\.\d+)', line)
+    s = (sample.group(1),sample.group(2))
+    self._samples[alg_name][var_name].append(s)
+
+def plotVar(v, ax, show_legend, show_x_label):
+  if show_x_label:
+    ax.set_xlabel(v.getXLabel(), fontsize='large')
+  ax.set_ylabel(v.getYLabel(), fontsize='large')
+
+  for alg in v._samples.keys():
+    i = 1
+    for series in v._samples[alg].keys():
+      x = [sample[0] for sample in v._samples[alg][series]]
+      y = [sample[1] for sample in v._samples[alg][series]]
+      x = numpy.array(x)
+      y = numpy.array(y)
+      line = plt.plot(x, y, label=alg, linewidth=4.0)
+      colormap = {'Available1':'#AAAAAA',
+                  'Available2':'#AAAAAA',
+                  'GCC1':'#80D000',
+                  'GCC2':'#008000',
+                  'GCC3':'#00F000',
+                  'GCC4':'#00B000',
+                  'GCC5':'#70B020',
+                  'NADA1':'#0000AA',
+                  'NADA2':'#A0A0FF',
+                  'NADA3':'#0000FF',
+                  'NADA4':'#C0A0FF',
+                  'NADA5':'#9060B0',
+                  'TCP1':'#AAAAAA',
+                  'TCP2':'#AAAAAA',
+                  'TCP3':'#AAAAAA',
+                  'TCP4':'#AAAAAA',
+                  'TCP5':'#AAAAAA',
+                  'TCP6':'#AAAAAA',
+                  'TCP7':'#AAAAAA',
+                  'TCP8':'#AAAAAA',
+                  'TCP9':'#AAAAAA',
+                  'TCP10':'#AAAAAA',}
+
+      plt.setp(line, color=colormap[alg + str(i)])
+      if alg.startswith('Available'):
+        plt.setp(line, linestyle='--')
+      plt.grid(True)
+
+      x1, x2, y1, y2 = plt.axis()
+      if v.getYMax() >= 0:
+        y2 = v.getYMax()
+      plt.axis((0, x2, 0, y2))
+      i += 1
+
+    if show_legend:
+      legend = plt.legend(loc='upper right', shadow=True,
+                          fontsize='large', ncol=len(v._samples))
+
+if __name__ == '__main__':
+
+  variables = [
+          ('Throughput_kbps', "Time (s)", "Throughput (kbps)", 1, 4000),
+          ('Delay_ms', "Time (s)", "One-way Delay (ms)", 2, 500),
+          ('Packet_Loss', "Time (s)", "Packet Loss Ratio", 3, 1.0),
+          ]
+
+  var = []
+
+  # Create objects.
+  for variable in variables:
+    var.append(Variable(variable))
+
+  # Add samples to the objects.
+  for line in sys.stdin:
+    if line.startswith("[ RUN      ]"):
+        test_name = re.search('\.(\w+)', line).group(1)
+    if line.startswith("PLOT"):
+      for v in var:
+        if v.getID() in line:
+          v.addSample(line)
+
+  matplotlib.rcParams.update({'font.size': 20})
+
+  # Plot variables.
+  fig = plt.figure()
+
+  # Offest and threshold on the same plot.
+  n = var[-1].getSubplot()
+  i = 0
+  for v in var:
+    ax = fig.add_subplot(n, 1, v.getSubplot())
+    plotVar(v, ax, i == 0, i == n - 1)
+    i += 1
+
+  #fig.savefig(test_name+".jpg")
+  plt.show()
+  
\ No newline at end of file
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh b/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.sh
old mode 100755
new mode 100644
similarity index 84%
rename from webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh
rename to webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.sh
index 66b7417..cb31825
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh
+++ b/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.sh
@@ -14,9 +14,9 @@
 # In Eclipse, that amounts to creating a Run Configuration which starts
 # "/bin/bash" with the arguments "-c [trunk_path]/out/Debug/modules_unittests
 # --gtest_filter=*BweTest* | [trunk_path]/webrtc/modules/
-# remote_bitrate_estimator/bwe_plot.
+# remote_bitrate_estimator/test/plot_dynamics.sh
 
-# bwe_plot.sh supports multiple figures (windows), the figure is specified as an
+# This script supports multiple figures (windows), the figure is specified as an
 # identifier at the first argument after the PLOT command. Each figure has a
 # single y axis and a dual y axis mode. If any line specifies an axis by ending
 # with "#<axis number (1 or 2)>" two y axis will be used, the first will be
@@ -25,6 +25,7 @@
 
 log=$(</dev/stdin)
 
+# Plot dynamics.
 function gen_gnuplot_input {
   colors=(a7001f 0a60c2 b2582b 21a66c d6604d 4393c3 f4a582 92c5de edcbb7 b1c5d0)
   plots=$(echo "$log" | grep "^PLOT")
@@ -34,14 +35,14 @@
     data_sets=$(echo "$plots" | grep "^PLOT.$figure" | cut -f 3 | sort | uniq)
     linetypes=($(echo "$data_sets" | grep "#" | cut -d '#' -f 2 | \
       cut -d ' ' -f 1))
-    echo -n "reset; "
-    echo -n "set terminal wxt $figure size 1440,900 font \"Arial,9\"; "
-    echo -n "set xlabel \"Seconds\"; "
+    echo "reset; "
+    echo "set terminal wxt $figure size 1440,900 font \"Arial,9\"; "
+    echo "set xlabel \"Seconds\"; "
     if (( "${#linetypes[@]}" > "0" )); then
-      echo -n "set ylabel 'bitrate (kbps)';"
-      echo -n "set ytics nomirror;"
-      echo -n "set y2label 'time delta (ms)';"
-      echo -n "set y2tics nomirror;"
+      echo "set ylabel 'bitrate (kbps)';"
+      echo "set ytics nomirror;"
+      echo "set y2label 'time delta (ms)';"
+      echo "set y2tics nomirror;"
     fi
     echo -n "plot "
     i=0
@@ -69,5 +70,4 @@
     done
   done
 }
-
 gen_gnuplot_input | gnuplot -persist