blob: 658f08c51f733f3e4b59a918b5ad4f1c69e19894 [file] [log] [blame]
#!/bin/bash
if [ $# -ne 1 ]; then
echo "Usage: $0 <output_filename>.md"
exit 1
fi
output_markdown="results/$1"
benchmark_script="server.py"
output_file="results/temp_output.txt"
checkpoint_file="resnet18-f37072fd.pth"
downloaded_checkpoint=false
batch_size_values=(1 32 64 128 256)
compile_values=(true false)
if [ -f $checkpoint_file ]; then
echo "Checkpoint exists."
else
downloaded_checkpoint=true
echo "Downloading checkpoint..."
wget https://download.pytorch.org/models/resnet18-f37072fd.pth
fi
if [ -e "$output_file" ]; then
rm "$output_file"
fi
touch $output_file
for batch_size in "${batch_size_values[@]}"; do
for compile in "${compile_values[@]}"; do
if [ "$compile" = true ]; then
python -W ignore "$benchmark_script" --batch_size "$batch_size" --compile >> $output_file
else
python -W ignore "$benchmark_script" --batch_size "$batch_size" --no-compile >> $output_file
fi
done
done
echo "| bs, compile | torch.load() / s | warmup latency / s | avg latency / s | max latency / s | min latency / s | throughput samples/s | GPU util % |" > $output_markdown
echo "| ----------- | ---------------- | ------------------ | --------------- | --------------- | --------------- | -------------------- | ---------- |" >> $output_markdown
while IFS= read -r line; do
batch_size=$(echo "$line" | jq -r '.batch_size')
compile=$(echo "$line" | jq -r '.compile')
torch_load=$(echo "$line" | jq -r '.torch_load_time' | awk '{printf "%.5f", $0}')
warmup_latency=$(echo "$line" | jq -r '.warmup_latency' | awk '{printf "%.5f", $0}')
avg_latency=$(echo "$line" | jq -r '.average_latency' | awk '{printf "%.5f", $0}')
max_latency=$(echo "$line" | jq -r '.max_latency' | awk '{printf "%.5f", $0}')
min_latency=$(echo "$line" | jq -r '.min_latency' | awk '{printf "%.5f", $0}')
throughput=$(echo "$line" | jq -r '.throughput' | awk '{printf "%.5f", $0}')
gpu_util=$(echo "$line" | jq -r '.GPU_utilization' | awk '{printf "%.5f", $0}')
echo "| $batch_size, $compile | $torch_load | $warmup_latency | $avg_latency | $max_latency | $min_latency | $throughput | $gpu_util |"
done < $output_file >> $output_markdown
rm "$output_file"
if [ "$downloaded_checkpoint" = true ]; then
echo "Cleaning up checkpoint..."
rm "$checkpoint_file"
else
echo "No cleanup needed"
fi