From 9cdef8aa0c7d5f4c266e4d3e02aeba18b3c08170 Mon Sep 17 00:00:00 2001 From: fmontorsi Date: Thu, 8 Aug 2019 11:18:05 +0200 Subject: [PATCH 1/2] Better script for performance tests --- perf/generate_csv.sh | 55 +++++++++++++++++++++++++---------------- perf/generate_graphs.py | 51 +++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 36 deletions(-) diff --git a/perf/generate_csv.sh b/perf/generate_csv.sh index 362c4aa3..226baa3b 100755 --- a/perf/generate_csv.sh +++ b/perf/generate_csv.sh @@ -3,16 +3,27 @@ # # This script assumes that 2 machines are used to generate performance results. # First machine is assumed to be the one where this script runs. -# Second machine is the "REMOTE_IP" machine; we assume to have passwordless SSH access. +# Second machine is the "REMOTE_IP_SSH" machine; we assume to have passwordless SSH access. +# +# Usage example: +# export REMOTE_IP_SSH=10.0.0.1 +# export LOCAL_TEST_ENDPOINT="tcp://192.168.1.1:1234" +# export REMOTE_TEST_ENDPOINT="tcp://192.168.1.2:1234" +# export REMOTE_LIBZMQ_PATH="/home/fmontorsi/libzmq/perf" +# ./generate_csv.sh # set -u -# configurable values: -REMOTE_IP=${REMOTE_IP:-127.0.0.1} -REMOTE_PATH=/home/francesco/work/libzmq-orig/perf #/root/libzmq/perf -TEST_ENDPOINT=tcp://127.0.0.1:1234 -MESSAGE_SIZE_LIST="1 16 64 128 512 1024 4096 16384 65536" +# configurable values (via environment variables): +REMOTE_IP_SSH=${REMOTE_IP_SSH:-127.0.0.1} +REMOTE_LIBZMQ_PATH=${REMOTE_LIBZMQ_PATH:-/root/libzmq/perf} +LOCAL_TEST_ENDPOINT=${LOCAL_TEST_ENDPOINT:-tcp://192.168.1.1:1234} +REMOTE_TEST_ENDPOINT=${REMOTE_TEST_ENDPOINT:-tcp://192.168.1.2:1234} + +# constant values: +MESSAGE_SIZE_LIST="8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072" +OUTPUT_DIR="results" OUTPUT_FILE_PREFIX="results.txt" OUTPUT_FILE_CSV_PREFIX="results.csv" @@ -21,19 +32,19 @@ OUTPUT_FILE_CSV_PREFIX="results.csv" function verify_ssh() { - ssh $REMOTE_IP "ls /" >/dev/null + ssh $REMOTE_IP_SSH "ls /" >/dev/null if [ $? -ne 0 ]; then - echo "Cannot connect via SSH passwordless to the REMOTE_IP $REMOTE_IP. Please fix the problem and retry." + echo "Cannot connect via SSH passwordless to the REMOTE_IP_SSH $REMOTE_IP_SSH. Please fix the problem and retry." exit 2 fi - ssh $REMOTE_IP "ls $REMOTE_PATH" >/dev/null + ssh $REMOTE_IP_SSH "ls $REMOTE_LIBZMQ_PATH" >/dev/null if [ $? -ne 0 ]; then - echo "The folder $REMOTE_PATH is not valid. Please fix the problem and retry." + echo "The folder $REMOTE_LIBZMQ_PATH is not valid. Please fix the problem and retry." exit 2 fi - echo "SSH connection to the remote $REMOTE_IP is working fine." + echo "SSH connection to the remote $REMOTE_IP_SSH is working fine." } function run_remote_perf_util() @@ -42,8 +53,8 @@ function run_remote_perf_util() local REMOTE_PERF_UTIL="$2" local NUM_MESSAGES="$3" - echo "Launching on $REMOTE_IP the utility [$REMOTE_PERF_UTIL] for messages ${MESSAGE_SIZE_BYTES}B long" - ssh $REMOTE_IP "$REMOTE_PATH/$REMOTE_PERF_UTIL $TEST_ENDPOINT $MESSAGE_SIZE_BYTES $NUM_MESSAGES" & + echo "Launching on $REMOTE_IP_SSH the utility [$REMOTE_PERF_UTIL] for messages ${MESSAGE_SIZE_BYTES}B long" + ssh $REMOTE_IP_SSH "$REMOTE_LIBZMQ_PATH/$REMOTE_PERF_UTIL $TEST_ENDPOINT $MESSAGE_SIZE_BYTES $NUM_MESSAGES" & if [ $? -ne 0 ]; then echo "Failed to launch remote perf util." exit 2 @@ -59,14 +70,14 @@ function generate_output_file() local CSV_HEADER_LINE="$5" # derived values: - local OUTPUT_FILE_TXT="${OUTPUT_FILE_PREFIX}.txt" # useful just for human-friendly debugging - local OUTPUT_FILE_CSV="${OUTPUT_FILE_PREFIX}.csv" # actually used to later produce graphs + local OUTPUT_FILE_TXT="${OUTPUT_DIR}/${OUTPUT_FILE_PREFIX}.txt" # useful just for human-friendly debugging + local OUTPUT_FILE_CSV="${OUTPUT_DIR}/${OUTPUT_FILE_PREFIX}.csv" # actually used to later produce graphs local MESSAGE_SIZE_ARRAY=($MESSAGE_SIZE_LIST) echo "Killing still-running ZMQ performance utils, if any" pkill $LOCAL_PERF_UTIL # in case it's running from a previous test if [ ! -z "$REMOTE_PERF_UTIL" ]; then - ssh $REMOTE_IP "pkill $REMOTE_PERF_UTIL" # in case it's running from a previous test + ssh $REMOTE_IP_SSH "pkill $REMOTE_PERF_UTIL" # in case it's running from a previous test fi echo "Resetting output file $OUTPUT_FILE_TXT and $OUTPUT_FILE_CSV" @@ -103,30 +114,32 @@ verify_ssh THROUGHPUT_CSV_HEADER_LINE="# message_size,message_count,PPS[msg/s],throughput[Mb/s]" # PUSH/PULL TCP throughput CSV file: +TEST_ENDPOINT="$LOCAL_TEST_ENDPOINT" generate_output_file "local_thr" "remote_thr" \ "pushpull_tcp_thr_results" \ - "100000" \ + "1000000" \ "$THROUGHPUT_CSV_HEADER_LINE" # PUSH/PULL INPROC throughput CSV file: # NOTE: in this case there is no remote utility to run and no ENDPOINT to provide: -TEST_ENDPOINT="" +TEST_ENDPOINT="" # inproc does not require any endpoint generate_output_file "inproc_thr" "" \ "pushpull_inproc_thr_results" \ - "1000000" \ + "10000000" \ "$THROUGHPUT_CSV_HEADER_LINE" # PUB/SUB proxy INPROC throughput CSV file: # NOTE: in this case there is no remote utility to run and no ENDPOINT to provide: -TEST_ENDPOINT="" +TEST_ENDPOINT="" # inproc does not require any endpoint generate_output_file "proxy_thr" "" \ "pubsubproxy_inproc_thr_results" \ - "1000000" \ + "10000000" \ "$THROUGHPUT_CSV_HEADER_LINE" # REQ/REP TCP latency CSV file: # NOTE: in this case it's the remote_lat utility that prints out the data, so we swap the local/remote arguments to the bash func: +TEST_ENDPOINT="$REMOTE_TEST_ENDPOINT" generate_output_file "remote_lat" "local_lat" \ "reqrep_tcp_lat_results" \ "10000" \ diff --git a/perf/generate_graphs.py b/perf/generate_graphs.py index ae2271ed..6efa94f8 100755 --- a/perf/generate_graphs.py +++ b/perf/generate_graphs.py @@ -1,17 +1,21 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -# This script assumes that a CSV file produced by "generate_csv.sh" is provided as input +# This script assumes that the set of CSV files produced by "generate_csv.sh" is provided as input +# and that locally there is the "results" folder. # # configurable values: -INPUT_FILE_PUSHPULL_TCP_THROUGHPUT="pushpull_tcp_thr_results.csv" -INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT="pushpull_inproc_thr_results.csv" -INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT="pubsubproxy_inproc_thr_results.csv" -INPUT_FILE_REQREP_TCP_LATENCY="reqrep_tcp_lat_results.csv" +INPUT_FILE_PUSHPULL_TCP_THROUGHPUT="results/pushpull_tcp_thr_results.csv" +INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT="results/pushpull_inproc_thr_results.csv" +INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT="results/pubsubproxy_inproc_thr_results.csv" +INPUT_FILE_REQREP_TCP_LATENCY="results/reqrep_tcp_lat_results.csv" # dependencies +# +# pip3 install matplotlib +# import matplotlib.pyplot as plt import numpy as np @@ -19,31 +23,48 @@ import numpy as np # functions -def plot_throughput(csv_filename, title): +def plot_throughput(csv_filename, title, force_10gb_limits=False): message_size_bytes, message_count, pps, mbps = np.loadtxt(csv_filename, delimiter=',', unpack=True) - plt.semilogx(message_size_bytes, pps / 1e6, label='PPS [Mmsg/s]', marker='x') - plt.semilogx(message_size_bytes, mbps / 1e3, label='Throughput [Mb/s]', marker='o') + + fig, ax1 = plt.subplots() + + # PPS axis + color = 'tab:red' + ax1.set_xlabel('Message size [B]') + ax1.set_ylabel('PPS [Mmsg/s]', color=color) + ax1.semilogx(message_size_bytes, pps / 1e6, label='PPS [Mmsg/s]', marker='x', color=color) + ax1.tick_params(axis='y', labelcolor=color) + + # GBPS axis + color = 'tab:blue' + ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis + ax2.set_ylabel('Throughput [Gb/s]', color=color) + ax2.semilogx(message_size_bytes, mbps / 1e3, label='Throughput [Gb/s]', marker='o') + if force_10gb_limits: + ax2.set_yticks(np.arange(0, 11, 1)) + ax2.tick_params(axis='y', labelcolor=color) + ax2.grid(True) - plt.xlabel('Message size [B]') plt.title(title) - plt.legend() - plt.show() + fig.tight_layout() # otherwise the right y-label is slightly clippe plt.savefig(csv_filename.replace('.csv', '.png')) + plt.show() def plot_latency(csv_filename, title): message_size_bytes, message_count, lat = np.loadtxt(csv_filename, delimiter=',', unpack=True) plt.semilogx(message_size_bytes, lat, label='Latency [us]', marker='o') plt.xlabel('Message size [B]') + plt.ylabel('Latency [us]') + plt.grid(True) plt.title(title) - plt.legend() - plt.show() plt.savefig(csv_filename.replace('.csv', '.png')) + plt.show() # main -plot_throughput(INPUT_FILE_PUSHPULL_TCP_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, TCP transport') +plot_throughput(INPUT_FILE_PUSHPULL_TCP_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, TCP transport', force_10gb_limits=True) plot_throughput(INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, INPROC transport') plot_throughput(INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT, 'ZeroMQ PUB/SUB PROXY socket throughput, INPROC transport') plot_latency(INPUT_FILE_REQREP_TCP_LATENCY, 'ZeroMQ REQ/REP socket latency, TCP transport') From 5f95e53e0f954e6c093260bf1f9eb8fd0946a728 Mon Sep 17 00:00:00 2001 From: fmontorsi Date: Thu, 8 Aug 2019 11:40:33 +0200 Subject: [PATCH 2/2] Allow to graph results from TCP links at 100Gbps nicely. --- perf/generate_csv.sh | 1 + perf/generate_graphs.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/perf/generate_csv.sh b/perf/generate_csv.sh index 226baa3b..d307f29e 100755 --- a/perf/generate_csv.sh +++ b/perf/generate_csv.sh @@ -81,6 +81,7 @@ function generate_output_file() fi echo "Resetting output file $OUTPUT_FILE_TXT and $OUTPUT_FILE_CSV" + mkdir -p ${OUTPUT_DIR} > $OUTPUT_FILE_TXT echo "$CSV_HEADER_LINE" > $OUTPUT_FILE_CSV diff --git a/perf/generate_graphs.py b/perf/generate_graphs.py index 6efa94f8..20651b71 100755 --- a/perf/generate_graphs.py +++ b/perf/generate_graphs.py @@ -5,11 +5,14 @@ # and that locally there is the "results" folder. # -# configurable values: +# results for TCP: INPUT_FILE_PUSHPULL_TCP_THROUGHPUT="results/pushpull_tcp_thr_results.csv" +INPUT_FILE_REQREP_TCP_LATENCY="results/reqrep_tcp_lat_results.csv" +TCP_LINK_GPBS=100 + +# results for INPROC: INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT="results/pushpull_inproc_thr_results.csv" INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT="results/pubsubproxy_inproc_thr_results.csv" -INPUT_FILE_REQREP_TCP_LATENCY="results/reqrep_tcp_lat_results.csv" # dependencies @@ -23,7 +26,7 @@ import numpy as np # functions -def plot_throughput(csv_filename, title, force_10gb_limits=False): +def plot_throughput(csv_filename, title, is_tcp=False): message_size_bytes, message_count, pps, mbps = np.loadtxt(csv_filename, delimiter=',', unpack=True) fig, ax1 = plt.subplots() @@ -40,8 +43,8 @@ def plot_throughput(csv_filename, title, force_10gb_limits=False): ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis ax2.set_ylabel('Throughput [Gb/s]', color=color) ax2.semilogx(message_size_bytes, mbps / 1e3, label='Throughput [Gb/s]', marker='o') - if force_10gb_limits: - ax2.set_yticks(np.arange(0, 11, 1)) + if is_tcp: + ax2.set_yticks(np.arange(0, TCP_LINK_GPBS + 1, TCP_LINK_GPBS/10)) ax2.tick_params(axis='y', labelcolor=color) ax2.grid(True) @@ -64,7 +67,7 @@ def plot_latency(csv_filename, title): # main -plot_throughput(INPUT_FILE_PUSHPULL_TCP_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, TCP transport', force_10gb_limits=True) +plot_throughput(INPUT_FILE_PUSHPULL_TCP_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, TCP transport', is_tcp=True) plot_throughput(INPUT_FILE_PUSHPULL_INPROC_THROUGHPUT, 'ZeroMQ PUSH/PULL socket throughput, INPROC transport') plot_throughput(INPUT_FILE_PUBSUBPROXY_INPROC_THROUGHPUT, 'ZeroMQ PUB/SUB PROXY socket throughput, INPROC transport') plot_latency(INPUT_FILE_REQREP_TCP_LATENCY, 'ZeroMQ REQ/REP socket latency, TCP transport')