Add profile times to treble_build
Add profiling times to treble_build to compare operating modes.
Increase and make build graph timeouts configurable by the client.
Bug: 237815549
Bug: 237015743
Test: m treble_build
Test: treble_build -json -o out/droid_report.json droid
Test: treble_build -v -o out/droid_report.txt droid
Change-Id: Icb3c50bcc517bd9bd69d9d81f482a461a1ddeb95
diff --git a/build/treble_build/cmd/main.go b/build/treble_build/cmd/main.go
index 0264ff3..b89fe9e 100644
--- a/build/treble_build/cmd/main.go
+++ b/build/treble_build/cmd/main.go
@@ -26,6 +26,7 @@
"os"
"runtime"
"strings"
+ "time"
"tools/treble/build/report/app"
"tools/treble/build/report/local"
@@ -67,11 +68,14 @@
// Common flags
ninjaDbPtr = flag.String("ninja", local.DefNinjaDb(), "Set the .ninja file to use when building metrics")
ninjaExcPtr = flag.String("ninja_cmd", local.DefNinjaExc(), "Set the ninja executable")
+ ninjaTimeoutStr = flag.String("ninja_timeout", local.DefaultNinjaTimeout, "Default ninja timeout")
+ buildTimeoutStr = flag.String("build_timeout", local.DefaultNinjaBuildTimeout, "Default build timeout")
manifestPtr = flag.String("manifest", local.DefManifest(), "Set the location of the manifest file")
upstreamPtr = flag.String("upstream", "", "Upstream branch to compare files against")
repoBasePtr = flag.String("repo_base", local.DefRepoBase(), "Set the repo base directory")
workerCountPtr = flag.Int("worker_count", runtime.NumCPU(), "Number of worker routines")
buildWorkerCountPtr = flag.Int("build_worker_count", local.MaxNinjaCliWorkers, "Number of build worker routines")
+ clientServerPtr = flag.Bool("client_server", false, "Run client server mode")
buildPtr = flag.Bool("build", false, "Build targets")
jsonPtr = flag.Bool("json", false, "Print json data")
verbosePtr = flag.Bool("v", false, "Print verbose text data")
@@ -82,6 +86,12 @@
pathsFlags = flag.NewFlagSet("paths", flag.ExitOnError)
)
+// Add profiling data
+type profTime struct {
+ Description string `json:"description"`
+ DurationSecs float64 `json:"duration"`
+}
+
type commit struct {
Project app.ProjectCommit `json:"project"`
Commit *app.GitCommit `json:"commit"`
@@ -99,14 +109,32 @@
Query *app.QueryResponse `json:"query,omitempty"`
Paths []*app.BuildPath `json:"build_paths,omitempty"`
Host *app.HostReport `json:"host,omitempty"`
+
+ // Profile data
+ Profile []*profTime `json:"profile"`
}
func main() {
+ startTime := time.Now()
ctx := context.Background()
rsp := &response{}
+ var addProfileData = func(desc string) {
+ rsp.Profile = append(rsp.Profile, &profTime{Description: desc, DurationSecs: time.Since(startTime).Seconds()})
+ startTime = time.Now()
+ }
flag.Parse()
+ ninjaTimeout, err := time.ParseDuration(*ninjaTimeoutStr)
+ if err != nil {
+ log.Fatalf("Invalid ninja timeout %s", *ninjaTimeoutStr)
+ }
+
+ buildTimeout, err := time.ParseDuration(*buildTimeoutStr)
+ if err != nil {
+ log.Fatalf("Invalid build timeout %s", *buildTimeoutStr)
+ }
+
subArgs := flag.Args()
if len(subArgs) < 1 {
// Nothing to do
@@ -115,7 +143,19 @@
defBuildTarget := "droid"
log.SetFlags(log.LstdFlags | log.Llongfile)
- ninja := local.NewNinjaCli(*ninjaExcPtr, *ninjaDbPtr)
+ ninja := local.NewNinjaCli(*ninjaExcPtr, *ninjaDbPtr, ninjaTimeout, buildTimeout, *clientServerPtr)
+
+ if *clientServerPtr {
+ ninjaServ := local.NewNinjaServer(*ninjaExcPtr, *ninjaDbPtr)
+ defer ninjaServ.Kill()
+ go func() {
+
+ ninjaServ.Start(ctx)
+ }()
+ if err := ninja.WaitForServer(ctx, 60); err != nil {
+ log.Fatalf("Failed to connect to server")
+ }
+ }
rtx := &report.Context{
RepoBase: *repoBasePtr,
Repo: &report.RepoMan{},
@@ -154,8 +194,10 @@
default:
rsp.Targets = subArgs
}
-
+ addProfileData("Init")
rtx.ResolveProjectMap(ctx, *manifestPtr, *upstreamPtr)
+ addProfileData("Project Map")
+
// Resolve any commits
if len(commits) > 0 {
log.Printf("Resolving %s", commits.String())
@@ -171,6 +213,7 @@
// Add files to list of inputs
rsp.Inputs = append(rsp.Inputs, files...)
}
+ addProfileData("Commit Resolution")
}
// Run any sub tools
@@ -178,6 +221,7 @@
if err := subcommand.Run(ctx, rtx, rsp); err != nil {
log.Fatal(err)
}
+ addProfileData(subArgs[0])
}
buildErrors := 0
@@ -185,6 +229,7 @@
for _, t := range rsp.Targets {
log.Printf("Building %s\n", t)
res := ninja.Build(ctx, t)
+ addProfileData(fmt.Sprintf("Build %s", t))
log.Printf("%s\n", res.Output)
if res.Success != true {
buildErrors++
@@ -194,10 +239,10 @@
}
// Generate report
- var err error
log.Printf("Generating report for targets %s", rsp.Targets)
req := &app.ReportRequest{Targets: rsp.Targets}
rsp.Report, err = report.RunReport(ctx, rtx, req)
+ addProfileData("Report")
if err != nil {
log.Fatal(fmt.Sprintf("Report failure <%s>", err))
}
@@ -294,4 +339,9 @@
targetPrint(t)
}
+ fmt.Fprintln(w, " Run Times")
+ for _, p := range rsp.Profile {
+ fmt.Fprintf(w, " %-30s : %f secs\n", p.Description, p.DurationSecs)
+ }
+
}
diff --git a/build/treble_build/local/ninja.go b/build/treble_build/local/ninja.go
index 012bcc7..956d959 100644
--- a/build/treble_build/local/ninja.go
+++ b/build/treble_build/local/ninja.go
@@ -18,8 +18,11 @@
"bufio"
"bytes"
"context"
+ "errors"
+ "fmt"
"io"
"io/ioutil"
+ "os/exec"
"strings"
"time"
@@ -27,7 +30,11 @@
)
// Performance degrades running multiple CLIs
-const MaxNinjaCliWorkers = 4
+const (
+ MaxNinjaCliWorkers = 4
+ DefaultNinjaTimeout = "100s"
+ DefaultNinjaBuildTimeout = "30m"
+)
// Separate out the executable to allow tests to override the results
type ninjaExec interface {
@@ -174,14 +181,23 @@
cmd string
db string
+ clientMode bool
timeout time.Duration
buildTimeout time.Duration
}
func (n *ninjaCmd) runTool(ctx context.Context, tool string, targets []string) (out *bytes.Buffer, err error) {
- args := append([]string{
- "-f", n.db,
- "-t", tool}, targets...)
+
+ args := []string{"-f", n.db}
+
+ if n.clientMode {
+ args = append(args, []string{
+ "-t", "client",
+ "-c", tool}...)
+ } else {
+ args = append(args, []string{"-t", tool}...)
+ }
+ args = append(args, targets...)
data := []byte{}
err, _ = runPipe(ctx, n.timeout, n.cmd, args, func(r io.Reader) {
data, _ = ioutil.ReadAll(r)
@@ -221,6 +237,7 @@
return bytes.NewBuffer(data), err
}
+// Command line ninja
type ninjaCli struct {
n ninjaExec
}
@@ -285,7 +302,50 @@
return parseBuild(target, raw, err == nil)
}
-func NewNinjaCli(cmd string, db string) *ninjaCli {
- cli := &ninjaCli{n: &ninjaCmd{cmd: cmd, db: db, timeout: 100000 * time.Millisecond, buildTimeout: 300000 * time.Millisecond}}
+
+// Wait for server
+func (cli *ninjaCli) WaitForServer(ctx context.Context, maxTries int) error {
+ // Wait for server to response to an empty input request
+ fmt.Printf("Waiting for server.")
+ for i := 0; i < maxTries; i++ {
+ _, err := cli.Input(ctx, "")
+ if err == nil {
+ fmt.Printf("\nConnected\n")
+ return nil
+ }
+ fmt.Printf(".")
+ time.Sleep(time.Second)
+ }
+ fmt.Printf(" failed\n")
+ return errors.New("Failed to connect")
+}
+func NewNinjaCli(cmd string, db string, timeout, buildTimeout time.Duration, client bool) *ninjaCli {
+ cli := &ninjaCli{n: &ninjaCmd{cmd: cmd, db: db, timeout: timeout, buildTimeout: buildTimeout, clientMode: client}}
return cli
}
+
+type ninjaServer struct {
+ cmdName string
+ db string
+ ctx *exec.Cmd
+}
+
+// Run server
+func (srv *ninjaServer) Start(ctx context.Context) error {
+ args := []string{"-f", srv.db, "-t", "server"}
+ srv.ctx = exec.CommandContext(ctx, srv.cmdName, args[0:]...)
+ err := srv.ctx.Start()
+ if err != nil {
+ return err
+ }
+ srv.ctx.Wait()
+ return nil
+}
+func (srv *ninjaServer) Kill() {
+ if srv.ctx != nil {
+ srv.ctx.Process.Kill()
+ }
+}
+func NewNinjaServer(cmd string, db string) *ninjaServer {
+ return &ninjaServer{cmdName: cmd, db: db}
+}
