| // Copyright 2019 syzkaller project authors. All rights reserved. |
| // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. |
| |
| package bhyve |
| |
| import ( |
| "fmt" |
| "io" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "regexp" |
| "strings" |
| "time" |
| |
| "github.com/google/syzkaller/pkg/config" |
| "github.com/google/syzkaller/pkg/log" |
| "github.com/google/syzkaller/pkg/osutil" |
| "github.com/google/syzkaller/vm/vmimpl" |
| ) |
| |
| func init() { |
| vmimpl.Register("bhyve", ctor, true) |
| } |
| |
| type Config struct { |
| Bridge string `json:"bridge"` // name of network bridge device |
| Count int `json:"count"` // number of VMs to use |
| HostIP string `json:"hostip"` // VM host IP address |
| Mem string `json:"mem"` // amount of VM memory |
| Dataset string `json:"dataset"` // ZFS dataset containing VM image |
| } |
| |
| type Pool struct { |
| env *vmimpl.Env |
| cfg *Config |
| } |
| |
| type instance struct { |
| cfg *Config |
| snapshot string |
| tapdev string |
| image string |
| debug bool |
| os string |
| sshkey string |
| sshuser string |
| sshhost string |
| merger *vmimpl.OutputMerger |
| vmName string |
| bhyve *exec.Cmd |
| } |
| |
| var ipRegex = regexp.MustCompile(`bound to (([0-9]+\.){3}[0-9]+) `) |
| var tapRegex = regexp.MustCompile(`^tap[0-9]+`) |
| |
| func ctor(env *vmimpl.Env) (vmimpl.Pool, error) { |
| cfg := &Config{ |
| Count: 1, |
| Mem: "512M", |
| } |
| if err := config.LoadData(env.Config, cfg); err != nil { |
| return nil, fmt.Errorf("failed to parse bhyve vm config: %v", err) |
| } |
| if cfg.Count < 1 || cfg.Count > 128 { |
| return nil, fmt.Errorf("invalid config param count: %v, want [1-128]", cfg.Count) |
| } |
| if env.Debug && cfg.Count > 1 { |
| log.Logf(0, "limiting number of VMs from %v to 1 in debug mode", cfg.Count) |
| cfg.Count = 1 |
| } |
| pool := &Pool{ |
| cfg: cfg, |
| env: env, |
| } |
| return pool, nil |
| } |
| |
| func (pool *Pool) Count() int { |
| return pool.cfg.Count |
| } |
| |
| func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) { |
| inst := &instance{ |
| cfg: pool.cfg, |
| debug: pool.env.Debug, |
| os: pool.env.OS, |
| sshkey: pool.env.SSHKey, |
| sshuser: pool.env.SSHUser, |
| vmName: fmt.Sprintf("syzkaller-%v-%v", pool.env.Name, index), |
| } |
| |
| dataset := inst.cfg.Dataset |
| mountpoint, err := osutil.RunCmd(time.Minute, "", "zfs", "get", "-H", "-o", "value", "mountpoint", dataset) |
| if err != nil { |
| return nil, err |
| } |
| |
| snapshot := fmt.Sprintf("%v@bhyve-%v", dataset, inst.vmName) |
| clone := fmt.Sprintf("%v/bhyve-%v", dataset, inst.vmName) |
| |
| prefix := strings.TrimSuffix(string(mountpoint), "\n") + "/" |
| image := strings.TrimPrefix(pool.env.Image, prefix) |
| if image == pool.env.Image { |
| return nil, fmt.Errorf("image file %v not contained in dataset %v", image, prefix) |
| } |
| inst.image = prefix + fmt.Sprintf("bhyve-%v", inst.vmName) + "/" + image |
| |
| // Stop the instance from a previous run in case it's still running. |
| osutil.RunCmd(time.Minute, "", "bhyvectl", "--destroy", fmt.Sprintf("--vm=%v", inst.vmName)) |
| // Destroy a lingering snapshot and clone. |
| osutil.RunCmd(time.Minute, "", "zfs", "destroy", "-R", snapshot) |
| |
| // Create a snapshot of the data set containing the VM image. bhyve will |
| // use a clone of the snapshot, which gets recreated every time the VM |
| // is restarted. This is all to work around bhyve's current lack of an |
| // image snapshot facility. |
| if _, err := osutil.RunCmd(time.Minute, "", "zfs", "snapshot", snapshot); err != nil { |
| inst.Close() |
| return nil, err |
| } |
| inst.snapshot = snapshot |
| if _, err := osutil.RunCmd(time.Minute, "", "zfs", "clone", snapshot, clone); err != nil { |
| inst.Close() |
| return nil, err |
| } |
| |
| tapdev, err := osutil.RunCmd(time.Minute, "", "ifconfig", "tap", "create") |
| if err != nil { |
| inst.Close() |
| return nil, err |
| } |
| inst.tapdev = tapRegex.FindString(string(tapdev)) |
| if _, err := osutil.RunCmd(time.Minute, "", "ifconfig", inst.cfg.Bridge, "addm", inst.tapdev); err != nil { |
| inst.Close() |
| return nil, err |
| } |
| |
| if err := inst.Boot(); err != nil { |
| inst.Close() |
| return nil, err |
| } |
| |
| return inst, nil |
| } |
| |
| func (inst *instance) Boot() error { |
| loaderArgs := []string{ |
| "-c", "stdio", |
| "-m", inst.cfg.Mem, |
| "-d", inst.image, |
| "-e", "autoboot_delay=0", |
| inst.vmName, |
| } |
| |
| // Stop the instance from the previous run in case it's still running. |
| osutil.RunCmd(time.Minute, "", "bhyvectl", "--destroy", fmt.Sprintf("--vm=%v", inst.vmName)) |
| |
| _, err := osutil.RunCmd(time.Minute, "", "bhyveload", loaderArgs...) |
| if err != nil { |
| return err |
| } |
| |
| bhyveArgs := []string{ |
| "-H", "-A", "-P", |
| "-c", "1", |
| "-m", inst.cfg.Mem, |
| "-s", "0:0,hostbridge", |
| "-s", "1:0,lpc", |
| "-s", fmt.Sprintf("2:0,virtio-net,%v", inst.tapdev), |
| "-s", fmt.Sprintf("3:0,virtio-blk,%v", inst.image), |
| "-l", "com1,stdio", |
| inst.vmName, |
| } |
| |
| outr, outw, err := osutil.LongPipe() |
| if err != nil { |
| return err |
| } |
| |
| bhyve := osutil.Command("bhyve", bhyveArgs...) |
| bhyve.Stdout = outw |
| bhyve.Stderr = outw |
| if err := bhyve.Start(); err != nil { |
| outr.Close() |
| outw.Close() |
| return err |
| } |
| outw.Close() |
| outw = nil |
| inst.bhyve = bhyve |
| |
| var tee io.Writer |
| if inst.debug { |
| tee = os.Stdout |
| } |
| inst.merger = vmimpl.NewOutputMerger(tee) |
| inst.merger.Add("console", outr) |
| outr = nil |
| |
| var bootOutput []byte |
| bootOutputStop := make(chan bool) |
| ipch := make(chan string, 1) |
| go func() { |
| gotip := false |
| for { |
| select { |
| case out := <-inst.merger.Output: |
| bootOutput = append(bootOutput, out...) |
| case <-bootOutputStop: |
| close(bootOutputStop) |
| return |
| } |
| if gotip { |
| continue |
| } |
| if ip := parseIP(bootOutput); ip != "" { |
| ipch <- ip |
| gotip = true |
| } |
| } |
| }() |
| |
| select { |
| case ip := <-ipch: |
| inst.sshhost = ip |
| case <-inst.merger.Err: |
| bootOutputStop <- true |
| <-bootOutputStop |
| return vmimpl.BootError{Title: "bhyve exited", Output: bootOutput} |
| case <-time.After(10 * time.Minute): |
| bootOutputStop <- true |
| <-bootOutputStop |
| return vmimpl.BootError{Title: "no IP found", Output: bootOutput} |
| } |
| |
| if err := vmimpl.WaitForSSH(inst.debug, 10*time.Minute, inst.sshhost, |
| inst.sshkey, inst.sshuser, inst.os, 22, nil); err != nil { |
| bootOutputStop <- true |
| <-bootOutputStop |
| return vmimpl.MakeBootError(err, bootOutput) |
| } |
| bootOutputStop <- true |
| return nil |
| } |
| |
| func (inst *instance) Close() { |
| if inst.bhyve != nil { |
| inst.bhyve.Process.Kill() |
| inst.bhyve.Wait() |
| osutil.RunCmd(time.Minute, "", "bhyvectl", fmt.Sprintf("--vm=%v", inst.vmName), "--destroy") |
| inst.bhyve = nil |
| } |
| if inst.snapshot != "" { |
| osutil.RunCmd(time.Minute, "", "zfs", "destroy", "-R", inst.snapshot) |
| inst.snapshot = "" |
| } |
| if inst.tapdev != "" { |
| osutil.RunCmd(time.Minute, "", "ifconfig", inst.tapdev, "destroy") |
| inst.tapdev = "" |
| } |
| } |
| |
| func (inst *instance) Forward(port int) (string, error) { |
| return fmt.Sprintf("%v:%v", inst.cfg.HostIP, port), nil |
| } |
| |
| func (inst *instance) Copy(hostSrc string) (string, error) { |
| vmDst := filepath.Join("/root", filepath.Base(hostSrc)) |
| args := append(vmimpl.SCPArgs(inst.debug, inst.sshkey, 22), |
| hostSrc, inst.sshuser+"@"+inst.sshhost+":"+vmDst) |
| if inst.debug { |
| log.Logf(0, "running command: scp %#v", args) |
| } |
| _, err := osutil.RunCmd(10*time.Minute, "", "scp", args...) |
| if err != nil { |
| return "", err |
| } |
| return vmDst, nil |
| } |
| |
| func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) ( |
| <-chan []byte, <-chan error, error) { |
| rpipe, wpipe, err := osutil.LongPipe() |
| if err != nil { |
| return nil, nil, err |
| } |
| inst.merger.Add("ssh", rpipe) |
| |
| args := append(vmimpl.SSHArgs(inst.debug, inst.sshkey, 22), |
| inst.sshuser+"@"+inst.sshhost, command) |
| if inst.debug { |
| log.Logf(0, "running command: ssh %#v", args) |
| } |
| cmd := osutil.Command("ssh", args...) |
| cmd.Stdout = wpipe |
| cmd.Stderr = wpipe |
| if err := cmd.Start(); err != nil { |
| wpipe.Close() |
| return nil, nil, err |
| } |
| wpipe.Close() |
| errc := make(chan error, 1) |
| signal := func(err error) { |
| select { |
| case errc <- err: |
| default: |
| } |
| } |
| |
| go func() { |
| select { |
| case <-time.After(timeout): |
| signal(vmimpl.ErrTimeout) |
| case <-stop: |
| signal(vmimpl.ErrTimeout) |
| case err := <-inst.merger.Err: |
| cmd.Process.Kill() |
| if cmdErr := cmd.Wait(); cmdErr == nil { |
| // If the command exited successfully, we got EOF error from merger. |
| // But in this case no error has happened and the EOF is expected. |
| err = nil |
| } |
| signal(err) |
| return |
| } |
| cmd.Process.Kill() |
| cmd.Wait() |
| }() |
| return inst.merger.Output, errc, nil |
| } |
| |
| func (inst *instance) Diagnose() ([]byte, bool) { |
| return nil, false |
| } |
| |
| func parseIP(output []byte) string { |
| matches := ipRegex.FindSubmatch(output) |
| if len(matches) < 2 { |
| return "" |
| } |
| return string(matches[1]) |
| } |