tools/syz-benchcmp/benchcmp.go - platform/external/syzkaller - Git at Google

 // Copyright 2017 syzkaller project authors. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

 // syz-benchcmp visualizes syz-manager benchmarking results.
 // First, run syz-manager with -bench=old flag.
 // Then, do experimental modifications and run syz-manager again with -bench=new flag.
 // Then, run syz-benchcmp old new.
 package main

 import (
 	"bufio"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"html/template"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"sort"
 )

 var (
 	flagSkip = flag.Int("skip", -30, "skip that many seconds after start (skip first 20% by default)")
 )

 type Graph struct {
 	Name    string
 	Headers []string
 	Points  []Point
 }

 type Point struct {
 	Time uint64
 	Vals []uint64
 }

 func main() {
 	flag.Parse()
 	if len(flag.Args()) == 0 {
 		fmt.Fprintf(os.Stderr, "usage: syz-benchcmp [flags] bench_file0 [bench_file1 [bench_file2]]...\n")
 		flag.PrintDefaults()
 		os.Exit(1)
 	}

 	graphs := []*Graph{
 		{Name: "coverage"},
 		{Name: "corpus"},
 		{Name: "exec total"},
 		{Name: "crash types"},
 	}
 	for i, fname := range flag.Args() {
 		data := readFile(fname)
 		addExecSpeed(data)
 		for _, g := range graphs {
 			g.Headers = append(g.Headers, filepath.Base(fname))
 			for _, v := range data {
 				pt := Point{
 					Time: v["fuzzing"],
 					Vals: make([]uint64, len(flag.Args())),
 				}
 				pt.Vals[i] = v[g.Name]
 				g.Points = append(g.Points, pt)
 			}
 		}
 	}
 	for _, g := range graphs {
 		if len(g.Points) == 0 {
 			failf("no data points")
 		}
 		sort.Sort(pointSlice(g.Points))
 		skipStart(g)
 		restoreMissingPoints(g)
 	}
 	printFinalStats(graphs)
 	display(graphs)
 }

 func readFile(fname string) (data []map[string]uint64) {
 	f, err := os.Open(fname)
 	if err != nil {
 		failf("failed to open input file: %v", err)
 	}
 	defer f.Close()
 	dec := json.NewDecoder(bufio.NewReader(f))
 	for dec.More() {
 		v := make(map[string]uint64)
 		if err := dec.Decode(&v); err != nil {
 			failf("failed to decode input file %v: %v", fname, err)
 		}
 		data = append(data, v)
 	}
 	return
 }

 func addExecSpeed(data []map[string]uint64) {
 	// Speed between consecutive samples is very unstable.
 	const (
 		window = 100
 		step   = 10
 	)
 	for i := window; i < len(data); i += step {
 		cur := data[i]
 		prev := data[i-window]
 		dx := cur["exec total"] - prev["exec total"]
 		dt := cur["fuzzing"] - prev["fuzzing"]
 		cur["exec speed"] = dx * 1000 / dt
 	}
 }

 func skipStart(g *Graph) {
 	skipTime := uint64(*flagSkip)
 	if *flagSkip < 0 {
 		// Negative skip means percents.
 		max := g.Points[len(g.Points)-1].Time
 		skipTime = max * -skipTime / 100
 	}
 	if skipTime > 0 {
 		skip := sort.Search(len(g.Points), func(i int) bool {
 			return g.Points[i].Time > skipTime
 		})
 		g.Points = g.Points[skip:]
 	}
 }

 func restoreMissingPoints(g *Graph) {
 	for i := range g.Headers {
 		// Find previous and next non-zero point for each zero point,
 		// and restore its value with linear inerpolation.
 		type Pt struct {
 			Time uint64
 			Val  uint64
 		}
 		var prev Pt
 		prevs := make(map[uint64]Pt)
 		for _, pt := range g.Points {
 			if pt.Vals[i] != 0 {
 				prev = Pt{pt.Time, pt.Vals[i]}
 				continue
 			}
 			prevs[pt.Time] = prev
 		}
 		var next Pt
 		for pti := len(g.Points) - 1; pti >= 0; pti-- {
 			pt := g.Points[pti]
 			if pt.Vals[i] != 0 {
 				next = Pt{pt.Time, pt.Vals[i]}
 				continue
 			}
 			prev := prevs[pt.Time]
 			if prev.Val == 0 || next.Val == 0 {
 				continue
 			}
 			pt.Vals[i] = prev.Val
 			if next.Time != prev.Time {
 				// Use signed calculations as corpus can go backwards.
 				pt.Vals[i] += uint64(int64(next.Val-prev.Val) * int64(pt.Time-prev.Time) / int64(next.Time-prev.Time))
 			}
 		}
 	}
 }

 func printFinalStats(graphs []*Graph) {
 	for i := 1; i < len(graphs[0].Headers); i++ {
 		fmt.Printf("%-12v%16v%16v%16v\n", "", graphs[0].Headers[0], graphs[0].Headers[i], "diff")
 		for _, g := range graphs {
 			lastNonZero := func(x int) uint64 {
 				for j := len(g.Points) - 1; j >= 0; j-- {
 					if v := g.Points[j].Vals[x]; v != 0 {
 						return v
 					}
 				}
 				return 0
 			}
 			old := lastNonZero(0)
 			new := lastNonZero(i)
 			fmt.Printf("%-12v%16v%16v%+16d\n", g.Name, old, new, int64(new-old))
 		}
 		fmt.Printf("\n")
 	}
 }

 func display(graphs []*Graph) {
 	outf, err := ioutil.TempFile("", "")
 	if err != nil {
 		failf("failed to create temp file: %v", err)
 	}
 	if err := htmlTemplate.Execute(outf, graphs); err != nil {
 		failf("failed to execute template: %v", err)
 	}
 	outf.Close()
 	name := outf.Name() + ".html"
 	if err := os.Rename(outf.Name(), name); err != nil {
 		failf("failed to rename file: %v", err)
 	}
 	if err := exec.Command("xdg-open", name).Start(); err != nil {
 		failf("failed to start browser: %v", err)
 	}
 }

 type pointSlice []Point

 func (a pointSlice) Len() int           { return len(a) }
 func (a pointSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a pointSlice) Less(i, j int) bool { return a[i].Time < a[j].Time }

 func failf(msg string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, msg+"\n", args...)
 	os.Exit(1)
 }

 var htmlTemplate = template.Must(
 	template.New("").Parse(`
 <!doctype html>
 <html>
   <head>
     <title>Syzkaller Bench</title>
     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
     <script type="text/javascript">
       google.load("visualization", "1", {packages:["corechart"]});
       google.setOnLoadCallback(drawCharts);
       function drawCharts() {
         {{range $id, $graph := .}}
         {
           var data = new google.visualization.DataTable();
           data.addColumn({type: 'number'});
           {{range $graph.Headers}}
             data.addColumn({type: 'number', label: '{{.}}'});
           {{end}}
           data.addRows([
             {{range $graph.Points}} [ {{.Time}}, {{range .Vals}} {{if .}} {{.}} {{end}}, {{end}}
           ],
           {{end}}
           ]);
           new google.visualization.LineChart(document.getElementById('graph_div_{{$id}}')).
             draw(data, {
               title: '{{$graph.Name}}',
               width: "100%",
               height: document.documentElement.clientHeight * 0.48,
               legend: {position: "in"},
               focusTarget: "category",
               hAxis: {title: "Time, sec"},
               chartArea: {left: "5%", top: "5%", width: "90%", height:"85%"}
             })
         }
         {{end}}
       }
     </script>
 </head>
 <body>
   <table style="width: 100%; height: 98vh">
     <tr>
       <td style="width: 50%"> <div id="graph_div_0"></div> </td>
       <td style="width: 50%"> <div id="graph_div_2"></div> </td>
     </tr>
     <tr>
       <td style="width: 50%"> <div id="graph_div_1"></div> </td>
       <td style="width: 50%"> <div id="graph_div_3"></div> </td>
     </tr>
   </table>
 </body>
 </html>
 `))
	// Copyright 2017 syzkaller project authors. All rights reserved.
	// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

	// syz-benchcmp visualizes syz-manager benchmarking results.
	// First, run syz-manager with -bench=old flag.
	// Then, do experimental modifications and run syz-manager again with -bench=new flag.
	// Then, run syz-benchcmp old new.
	package main

	import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	)

	var (
	flagSkip = flag.Int("skip", -30, "skip that many seconds after start (skip first 20% by default)")
	)

	type Graph struct {
	Name string
	Headers []string
	Points []Point
	}

	type Point struct {
	Time uint64
	Vals []uint64
	}

	func main() {
	flag.Parse()
	if len(flag.Args()) == 0 {
	fmt.Fprintf(os.Stderr, "usage: syz-benchcmp [flags] bench_file0 [bench_file1 [bench_file2]]...\n")
	flag.PrintDefaults()
	os.Exit(1)
	}

	graphs := []*Graph{
	{Name: "coverage"},
	{Name: "corpus"},
	{Name: "exec total"},
	{Name: "crash types"},
	}
	for i, fname := range flag.Args() {
	data := readFile(fname)
	addExecSpeed(data)
	for _, g := range graphs {
	g.Headers = append(g.Headers, filepath.Base(fname))
	for _, v := range data {
	pt := Point{
	Time: v["fuzzing"],
	Vals: make([]uint64, len(flag.Args())),
	}
	pt.Vals[i] = v[g.Name]
	g.Points = append(g.Points, pt)
	}
	}
	}
	for _, g := range graphs {
	if len(g.Points) == 0 {
	failf("no data points")
	}
	sort.Sort(pointSlice(g.Points))
	skipStart(g)
	restoreMissingPoints(g)
	}
	printFinalStats(graphs)
	display(graphs)
	}

	func readFile(fname string) (data []map[string]uint64) {
	f, err := os.Open(fname)
	if err != nil {
	failf("failed to open input file: %v", err)
	}
	defer f.Close()
	dec := json.NewDecoder(bufio.NewReader(f))
	for dec.More() {
	v := make(map[string]uint64)
	if err := dec.Decode(&v); err != nil {
	failf("failed to decode input file %v: %v", fname, err)
	}
	data = append(data, v)
	}
	return
	}

	func addExecSpeed(data []map[string]uint64) {
	// Speed between consecutive samples is very unstable.
	const (
	window = 100
	step = 10
	)
	for i := window; i < len(data); i += step {
	cur := data[i]
	prev := data[i-window]
	dx := cur["exec total"] - prev["exec total"]
	dt := cur["fuzzing"] - prev["fuzzing"]
	cur["exec speed"] = dx * 1000 / dt
	}
	}

	func skipStart(g *Graph) {
	skipTime := uint64(*flagSkip)
	if *flagSkip < 0 {
	// Negative skip means percents.
	max := g.Points[len(g.Points)-1].Time
	skipTime = max * -skipTime / 100
	}
	if skipTime > 0 {
	skip := sort.Search(len(g.Points), func(i int) bool {
	return g.Points[i].Time > skipTime
	})
	g.Points = g.Points[skip:]
	}
	}

	func restoreMissingPoints(g *Graph) {
	for i := range g.Headers {
	// Find previous and next non-zero point for each zero point,
	// and restore its value with linear inerpolation.
	type Pt struct {
	Time uint64
	Val uint64
	}
	var prev Pt
	prevs := make(map[uint64]Pt)
	for _, pt := range g.Points {
	if pt.Vals[i] != 0 {
	prev = Pt{pt.Time, pt.Vals[i]}
	continue
	}
	prevs[pt.Time] = prev
	}
	var next Pt
	for pti := len(g.Points) - 1; pti >= 0; pti-- {
	pt := g.Points[pti]
	if pt.Vals[i] != 0 {
	next = Pt{pt.Time, pt.Vals[i]}
	continue
	}
	prev := prevs[pt.Time]
	if prev.Val == 0 \|\| next.Val == 0 {
	continue
	}
	pt.Vals[i] = prev.Val
	if next.Time != prev.Time {
	// Use signed calculations as corpus can go backwards.
	pt.Vals[i] += uint64(int64(next.Val-prev.Val) * int64(pt.Time-prev.Time) / int64(next.Time-prev.Time))
	}
	}
	}
	}

	func printFinalStats(graphs []*Graph) {
	for i := 1; i < len(graphs[0].Headers); i++ {
	fmt.Printf("%-12v%16v%16v%16v\n", "", graphs[0].Headers[0], graphs[0].Headers[i], "diff")
	for _, g := range graphs {
	lastNonZero := func(x int) uint64 {
	for j := len(g.Points) - 1; j >= 0; j-- {
	if v := g.Points[j].Vals[x]; v != 0 {
	return v
	}
	}
	return 0
	}
	old := lastNonZero(0)
	new := lastNonZero(i)
	fmt.Printf("%-12v%16v%16v%+16d\n", g.Name, old, new, int64(new-old))
	}
	fmt.Printf("\n")
	}
	}

	func display(graphs []*Graph) {
	outf, err := ioutil.TempFile("", "")
	if err != nil {
	failf("failed to create temp file: %v", err)
	}
	if err := htmlTemplate.Execute(outf, graphs); err != nil {
	failf("failed to execute template: %v", err)
	}
	outf.Close()
	name := outf.Name() + ".html"
	if err := os.Rename(outf.Name(), name); err != nil {
	failf("failed to rename file: %v", err)
	}
	if err := exec.Command("xdg-open", name).Start(); err != nil {
	failf("failed to start browser: %v", err)
	}
	}

	type pointSlice []Point

	func (a pointSlice) Len() int { return len(a) }
	func (a pointSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
	func (a pointSlice) Less(i, j int) bool { return a[i].Time < a[j].Time }

	func failf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
	}

	var htmlTemplate = template.Must(
	template.New("").Parse(`
	<!doctype html>
	<html>
	<head>
	<title>Syzkaller Bench</title>
	<script type="text/javascript" src="https://www.google.com/jsapi"></script>
	<script type="text/javascript">
	google.load("visualization", "1", {packages:["corechart"]});
	google.setOnLoadCallback(drawCharts);
	function drawCharts() {
	{{range $id, $graph := .}}
	{
	var data = new google.visualization.DataTable();
	data.addColumn({type: 'number'});
	{{range $graph.Headers}}
	data.addColumn({type: 'number', label: '{{.}}'});
	{{end}}
	data.addRows([
	{{range $graph.Points}} [ {{.Time}}, {{range .Vals}} {{if .}} {{.}} {{end}}, {{end}}
	],
	{{end}}
	]);
	new google.visualization.LineChart(document.getElementById('graph_div_{{$id}}')).
	draw(data, {
	title: '{{$graph.Name}}',
	width: "100%",
	height: document.documentElement.clientHeight * 0.48,
	legend: {position: "in"},
	focusTarget: "category",
	hAxis: {title: "Time, sec"},
	chartArea: {left: "5%", top: "5%", width: "90%", height:"85%"}
	})
	}
	{{end}}
	}
	</script>
	</head>
	<body>
	<table style="width: 100%; height: 98vh">
	<tr>
	<td style="width: 50%"> <div id="graph_div_0"></div> </td>
	<td style="width: 50%"> <div id="graph_div_2"></div> </td>
	</tr>
	<tr>
	<td style="width: 50%"> <div id="graph_div_1"></div> </td>
	<td style="width: 50%"> <div id="graph_div_3"></div> </td>
	</tr>
	</table>
	</body>
	</html>
	`))