Merge pull request #3627 from f18m/master

Scripts for graphing benchmark results

perf/generate_csv.sh

@@ -3,16 +3,27 @@
 #
 # This script assumes that 2 machines are used to generate performance results.
 # First machine is assumed to be the one where this script runs.
-# Second machine is the "REMOTE_IP" machine; we assume to have passwordless SSH access.
+# Second machine is the "REMOTE_IP_SSH" machine; we assume to have passwordless SSH access.
+#
+# Usage example:
+#    export REMOTE_IP_SSH=10.0.0.1
+#    export LOCAL_TEST_ENDPOINT="tcp://192.168.1.1:1234"
+#    export REMOTE_TEST_ENDPOINT="tcp://192.168.1.2:1234"
+#    export REMOTE_LIBZMQ_PATH="/home/fmontorsi/libzmq/perf"
+#    ./generate_csv.sh
 #
 
 set -u
 
-# configurable values:
+# configurable values (via environment variables):
-REMOTE_IP=${REMOTE_IP:-127.0.0.1}
+REMOTE_IP_SSH=${REMOTE_IP_SSH:-127.0.0.1}
-REMOTE_PATH=/home/francesco/work/libzmq-orig/perf  #/root/libzmq/perf
+REMOTE_LIBZMQ_PATH=${REMOTE_LIBZMQ_PATH:-/root/libzmq/perf}
-TEST_ENDPOINT=tcp://127.0.0.1:1234
+LOCAL_TEST_ENDPOINT=${LOCAL_TEST_ENDPOINT:-tcp://192.168.1.1:1234}
-MESSAGE_SIZE_LIST="1 16 64 128 512 1024 4096 16384 65536"
+REMOTE_TEST_ENDPOINT=${REMOTE_TEST_ENDPOINT:-tcp://192.168.1.2:1234}
+
+# constant values:
+MESSAGE_SIZE_LIST="8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072"
+OUTPUT_DIR="results"
 OUTPUT_FILE_PREFIX="results.txt"
 OUTPUT_FILE_CSV_PREFIX="results.csv"
 
@@ -21,19 +32,19 @@ OUTPUT_FILE_CSV_PREFIX="results.csv"
 
 function verify_ssh()
 {
-    ssh $REMOTE_IP "ls /" >/dev/null
+    ssh $REMOTE_IP_SSH "ls /" >/dev/null
     if [ $? -ne 0 ]; then
-        echo "Cannot connect via SSH passwordless to the REMOTE_IP $REMOTE_IP. Please fix the problem and retry."
+        echo "Cannot connect via SSH passwordless to the REMOTE_IP_SSH $REMOTE_IP_SSH. Please fix the problem and retry."
         exit 2
     fi
 
-    ssh $REMOTE_IP "ls $REMOTE_PATH" >/dev/null
+    ssh $REMOTE_IP_SSH "ls $REMOTE_LIBZMQ_PATH" >/dev/null
     if [ $? -ne 0 ]; then
-        echo "The folder $REMOTE_PATH is not valid. Please fix the problem and retry."
+        echo "The folder $REMOTE_LIBZMQ_PATH is not valid. Please fix the problem and retry."
         exit 2
     fi
 
-    echo "SSH connection to the remote $REMOTE_IP is working fine."
+    echo "SSH connection to the remote $REMOTE_IP_SSH is working fine."
 }
 
 function run_remote_perf_util()
@@ -42,8 +53,8 @@ function run_remote_perf_util()
     local REMOTE_PERF_UTIL="$2"
     local NUM_MESSAGES="$3"
 
-    echo "Launching on $REMOTE_IP the utility [$REMOTE_PERF_UTIL] for messages ${MESSAGE_SIZE_BYTES}B long"
+    echo "Launching on $REMOTE_IP_SSH the utility [$REMOTE_PERF_UTIL] for messages ${MESSAGE_SIZE_BYTES}B long"
-    ssh $REMOTE_IP "$REMOTE_PATH/$REMOTE_PERF_UTIL $TEST_ENDPOINT $MESSAGE_SIZE_BYTES $NUM_MESSAGES" &
+    ssh $REMOTE_IP_SSH "$REMOTE_LIBZMQ_PATH/$REMOTE_PERF_UTIL $TEST_ENDPOINT $MESSAGE_SIZE_BYTES $NUM_MESSAGES" &
     if [ $? -ne 0 ]; then
         echo "Failed to launch remote perf util."
         exit 2
@@ -59,17 +70,18 @@ function generate_output_file()
     local CSV_HEADER_LINE="$5"
 
     # derived values:
-    local OUTPUT_FILE_TXT="${OUTPUT_FILE_PREFIX}.txt"         # useful just for human-friendly debugging
+    local OUTPUT_FILE_TXT="${OUTPUT_DIR}/${OUTPUT_FILE_PREFIX}.txt"         # useful just for human-friendly debugging
-    local OUTPUT_FILE_CSV="${OUTPUT_FILE_PREFIX}.csv"     # actually used to later produce graphs
+    local OUTPUT_FILE_CSV="${OUTPUT_DIR}/${OUTPUT_FILE_PREFIX}.csv"     # actually used to later produce graphs
     local MESSAGE_SIZE_ARRAY=($MESSAGE_SIZE_LIST)
 
     echo "Killing still-running ZMQ performance utils, if any"
     pkill $LOCAL_PERF_UTIL                       # in case it's running from a previous test
     if [ ! -z "$REMOTE_PERF_UTIL" ]; then
-        ssh $REMOTE_IP "pkill $REMOTE_PERF_UTIL"     # in case it's running from a previous test
+        ssh $REMOTE_IP_SSH "pkill $REMOTE_PERF_UTIL"     # in case it's running from a previous test
     fi
 
     echo "Resetting output file $OUTPUT_FILE_TXT and $OUTPUT_FILE_CSV"
+    mkdir -p ${OUTPUT_DIR}
     > $OUTPUT_FILE_TXT
     echo "$CSV_HEADER_LINE" > $OUTPUT_FILE_CSV
 
@@ -103,30 +115,32 @@ verify_ssh
 THROUGHPUT_CSV_HEADER_LINE="# message_size,message_count,PPS[msg/s],throughput[Mb/s]"
 
 # PUSH/PULL TCP throughput CSV file:
+TEST_ENDPOINT="$LOCAL_TEST_ENDPOINT"
 generate_output_file "local_thr" "remote_thr" \
     "pushpull_tcp_thr_results" \
-    "100000" \
+    "1000000" \
     "$THROUGHPUT_CSV_HEADER_LINE"
 
 # PUSH/PULL INPROC throughput CSV file:
 # NOTE: in this case there is no remote utility to run and no ENDPOINT to provide:
-TEST_ENDPOINT=""
+TEST_ENDPOINT=""  # inproc does not require any endpoint
 generate_output_file "inproc_thr" "" \
     "pushpull_inproc_thr_results" \
-    "1000000" \
+    "10000000" \
     "$THROUGHPUT_CSV_HEADER_LINE"
 
 # PUB/SUB proxy INPROC throughput CSV file:
 # NOTE: in this case there is no remote utility to run and no ENDPOINT to provide:
-TEST_ENDPOINT=""
+TEST_ENDPOINT="" # inproc does	not require any	endpoint
 generate_output_file "proxy_thr" "" \
     "pubsubproxy_inproc_thr_results" \
-    "1000000" \
+    "10000000" \
     "$THROUGHPUT_CSV_HEADER_LINE"
 
 
 # REQ/REP TCP latency CSV file:
 # NOTE: in this case it's the remote_lat utility that prints out the data, so we swap the local/remote arguments to the bash func:
+TEST_ENDPOINT="$REMOTE_TEST_ENDPOINT"
 generate_output_file "remote_lat" "local_lat" \
     "reqrep_tcp_lat_results" \
     "10000" \

perf/generate_graphs.py

@@ -1,17 +1,24 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 #
-# This script assumes that a CSV file produced by "generate_csv.sh" is provided as input
+# This script assumes that the set of CSV files produced by "generate_csv.sh" is provided as input
+# and that locally there is the "results" folder.
 #
 
-# configurable values:
+# results for TCP:
-INPUT_FILE_PUSHPULL_TCP_THROUGHPUT="pushpull_tcp_thr_results.csv"
+INPUT_FILE_PUSHPULL_TCP_THROUGHPUT="results/pushpull_tcp_thr_results.csv"
-INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT="pushpull_inproc_thr_results.csv"
+INPUT_FILE_REQREP_TCP_LATENCY="results/reqrep_tcp_lat_results.csv"
-INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT="pubsubproxy_inproc_thr_results.csv"
+TCP_LINK_GPBS=100
-INPUT_FILE_REQREP_TCP_LATENCY="reqrep_tcp_lat_results.csv"
+
+# results for INPROC:
+INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT="results/pushpull_inproc_thr_results.csv"
+INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT="results/pubsubproxy_inproc_thr_results.csv"
 
 
 # dependencies
+#
+# pip3 install matplotlib
+#
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -19,31 +26,48 @@ import numpy as np
 
 # functions
 
-def plot_throughput(csv_filename, title):
+def plot_throughput(csv_filename, title, is_tcp=False):
     message_size_bytes, message_count, pps, mbps = np.loadtxt(csv_filename, delimiter=',', unpack=True)
-    plt.semilogx(message_size_bytes, pps / 1e6, label='PPS [Mmsg/s]', marker='x')
+
-    plt.semilogx(message_size_bytes, mbps / 1e3, label='Throughput [Mb/s]', marker='o')
+    fig, ax1 = plt.subplots()
+
+    # PPS axis
+    color = 'tab:red'
+    ax1.set_xlabel('Message size [B]')
+    ax1.set_ylabel('PPS [Mmsg/s]', color=color)
+    ax1.semilogx(message_size_bytes, pps / 1e6, label='PPS [Mmsg/s]', marker='x', color=color)
+    ax1.tick_params(axis='y', labelcolor=color)
+
+    # GBPS axis
+    color = 'tab:blue'
+    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis
+    ax2.set_ylabel('Throughput [Gb/s]', color=color)
+    ax2.semilogx(message_size_bytes, mbps / 1e3, label='Throughput [Gb/s]', marker='o')
+    if is_tcp:
+        ax2.set_yticks(np.arange(0, TCP_LINK_GPBS + 1, TCP_LINK_GPBS/10)) 
+    ax2.tick_params(axis='y', labelcolor=color)
+    ax2.grid(True)
     
-    plt.xlabel('Message size [B]')
     plt.title(title)
-    plt.legend()
+    fig.tight_layout()  # otherwise the right y-label is slightly clippe
-    plt.show()
     plt.savefig(csv_filename.replace('.csv', '.png'))
+    plt.show()
 
 def plot_latency(csv_filename, title):
     message_size_bytes, message_count, lat = np.loadtxt(csv_filename, delimiter=',', unpack=True)
     plt.semilogx(message_size_bytes, lat, label='Latency [us]', marker='o')
     
     plt.xlabel('Message size [B]')
+    plt.ylabel('Latency [us]')
+    plt.grid(True)
     plt.title(title)
-    plt.legend()
-    plt.show()
     plt.savefig(csv_filename.replace('.csv', '.png'))
+    plt.show()
 
 
 # main
 
-plot_throughput(INPUT_FILE_PUSHPULL_TCP_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, TCP transport')
+plot_throughput(INPUT_FILE_PUSHPULL_TCP_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, TCP transport', is_tcp=True)
 plot_throughput(INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, INPROC transport')
 plot_throughput(INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT, 'ZeroMQ PUB/SUB PROXY socket throughput, INPROC transport')
 plot_latency(INPUT_FILE_REQREP_TCP_LATENCY, 'ZeroMQ REQ/REP socket latency, TCP transport')