pkg/report/linux.go

// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package report

import (
	"bufio"
	"bytes"
	"fmt"
	"net/mail"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/symbolizer"
)

type linux struct {
	kernelSrc             string
	kernelObj             string
	vmlinux               string
	symbols               map[string][]symbolizer.Symbol
	ignores               []*regexp.Regexp
	consoleOutputRe       *regexp.Regexp
	questionableRe        *regexp.Regexp
	guiltyFileBlacklist   []*regexp.Regexp
	reportStartIgnores    [][]byte
	infoMessagesWithStack [][]byte
	eoi                   []byte
}

func ctorLinux(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) {
	vmlinux := ""
	var symbols map[string][]symbolizer.Symbol
	if kernelObj != "" {
		vmlinux = filepath.Join(kernelObj, "vmlinux")
		var err error
		symbols, err = symbolizer.ReadSymbols(vmlinux)
		if err != nil {
			return nil, nil, err
		}
	}
	ctx := &linux{
		kernelSrc: kernelSrc,
		kernelObj: kernelObj,
		vmlinux:   vmlinux,
		symbols:   symbols,
		ignores:   ignores,
	}
	ctx.consoleOutputRe = regexp.MustCompile(`^(?:\*\* [0-9]+ printk messages dropped \*\* )?(?:.* login: )?(?:\<[0-9]+\>)?\[ *[0-9]+\.[0-9]+\] `)
	ctx.questionableRe = regexp.MustCompile(`(?:\[\<[0-9a-f]+\>\])? \? +[a-zA-Z0-9_.]+\+0x[0-9a-f]+/[0-9a-f]+`)
	ctx.eoi = []byte("<EOI>")
	ctx.guiltyFileBlacklist = []*regexp.Regexp{
		regexp.MustCompile(`.*\.h`),
		regexp.MustCompile(`^lib/.*`),
		regexp.MustCompile(`^virt/lib/.*`),
		regexp.MustCompile(`^mm/kasan/.*`),
		regexp.MustCompile(`^mm/kmsan/.*`),
		regexp.MustCompile(`^kernel/kcov.c`),
		regexp.MustCompile(`^mm/sl.b.c`),
		regexp.MustCompile(`^mm/percpu.*`),
		regexp.MustCompile(`^mm/vmalloc.c`),
		regexp.MustCompile(`^mm/page_alloc.c`),
		regexp.MustCompile(`^mm/util.c`),
		regexp.MustCompile(`^kernel/rcu/.*`),
		regexp.MustCompile(`^arch/.*/kernel/traps.c`),
		regexp.MustCompile(`^arch/.*/mm/fault.c`),
		regexp.MustCompile(`^kernel/locking/.*`),
		regexp.MustCompile(`^kernel/panic.c`),
		regexp.MustCompile(`^kernel/softirq.c`),
		regexp.MustCompile(`^kernel/kthread.c`),
		regexp.MustCompile(`^kernel/sched/.*.c`),
		regexp.MustCompile(`^kernel/time/timer.c`),
		regexp.MustCompile(`^kernel/workqueue.c`),
		regexp.MustCompile(`^net/core/dev.c`),
		regexp.MustCompile(`^net/core/sock.c`),
		regexp.MustCompile(`^net/core/skbuff.c`),
		regexp.MustCompile(`^fs/proc/generic.c`),
	}
	// These pattern do _not_ start a new report, i.e. can be in a middle of another report.
	ctx.reportStartIgnores = [][]byte{
		[]byte("invalid opcode: 0000"),
		[]byte("Kernel panic - not syncing: panic_on_warn set"),
		[]byte("unregister_netdevice: waiting for"),
	}
	// These pattern math kernel reports which are not bugs in itself but contain stack traces.
	// If we see them in the middle of another report, we know that the report is potentially corrupted.
	ctx.infoMessagesWithStack = [][]byte{
		[]byte("vmalloc: allocation failure:"),
		[]byte("FAULT_INJECTION: forcing a failure"),
		[]byte("FAULT_FLAG_ALLOW_RETRY missing"),
	}
	suppressions := []string{
		"fatal error: runtime: out of memory",
		"fatal error: runtime: cannot allocate memory",
		"panic: failed to start executor binary",
		"panic: executor failed: pthread_create failed",
		"panic: failed to create temp dir",
		"fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS
		"signal SIGBUS: bus error",                                // presubmably OOM turned into SIGBUS
		"Out of memory: Kill process .* \\(syz-fuzzer\\)",
		"Out of memory: Kill process .* \\(sshd\\)",
		"Killed process .* \\(syz-fuzzer\\)",
		"Killed process .* \\(sshd\\)",
		"lowmemorykiller: Killing 'syz-fuzzer'",
		"lowmemorykiller: Killing 'sshd'",
		"INIT: PANIC: segmentation violation!",
	}
	return ctx, suppressions, nil
}

func (ctx *linux) ContainsCrash(output []byte) bool {
	return containsCrash(output, linuxOopses, ctx.ignores)
}

func (ctx *linux) Parse(output []byte) *Report {
	oops, startPos, endPos, logReport, consoleReport, consoleReportReliable,
		logReportPrefix, consoleReportPrefix := ctx.parseOutput(output)
	if oops == nil {
		return nil
	}
	rep := &Report{
		Output:   output,
		StartPos: startPos,
		EndPos:   endPos,
	}
	var report []byte
	var reportPrefix [][]byte
	// Try extracting report from console output only.
	title, corrupted, format := extractDescription(consoleReportReliable, oops, linuxStackParams)
	if title != "" {
		report = consoleReport
		reportPrefix = consoleReportPrefix
	} else {
		// Failure. Try extracting report from the whole log.
		report = logReport
		reportPrefix = logReportPrefix
		title, corrupted, format = extractDescription(report, oops, linuxStackParams)
		if title == "" {
			panic(fmt.Sprintf("non matching oops for %q in:\n%s\n\nconsole:\n%s\n"+
				"output [range:%v-%v]:\n%s\n",
				oops.header, report, consoleReportReliable,
				rep.StartPos, rep.StartPos+len(report), output))
		}
	}
	rep.Title = title
	rep.Corrupted = corrupted != ""
	rep.CorruptedReason = corrupted
	// Prepend 5 lines preceding start of the report,
	// they can contain additional info related to the report.
	for _, prefix := range reportPrefix {
		rep.Report = append(rep.Report, prefix...)
		rep.Report = append(rep.Report, '\n')
	}
	rep.Report = append(rep.Report, report...)
	if !rep.Corrupted {
		rep.Corrupted, rep.CorruptedReason = ctx.isCorrupted(title, report, format)
	}
	return rep
}

// Yes, it is complex, but all state and logic are tightly coupled. It's unclear how to simplify it.
// nolint: gocyclo
func (ctx *linux) parseOutput(output []byte) (
	oops *oops, startPos, endPos int,
	logReport, consoleReport, consoleReportReliable []byte,
	logReportPrefix, consoleReportPrefix [][]byte) {
	firstReportEnd := 0
	secondReportPos := 0
	textLines := 0
	skipText := false
	for pos := 0; pos < len(output); {
		next := bytes.IndexByte(output[pos:], '\n')
		if next != -1 {
			next += pos
		} else {
			next = len(output)
		}
		line := output[pos:next]
		for _, oops1 := range linuxOopses {
			match := matchOops(line, oops1, ctx.ignores)
			if match == -1 {
				if oops != nil && secondReportPos == 0 {
					for _, pattern := range ctx.infoMessagesWithStack {
						if bytes.Contains(line, pattern) {
							secondReportPos = pos
							break
						}
					}
				}
				continue
			}
			endPos = next
			if oops == nil {
				oops = oops1
				startPos = pos
				break
			} else if secondReportPos == 0 {
				ignored := false
				for _, ignore := range ctx.reportStartIgnores {
					if bytes.Contains(line, ignore) {
						ignored = true
						break
					}
				}
				if !ignored {
					secondReportPos = pos
				}
			}
		}
		if oops == nil {
			logReportPrefix = append(logReportPrefix, append([]byte{}, line...))
			if len(logReportPrefix) > 5 {
				logReportPrefix = logReportPrefix[1:]
			}
		}
		if ctx.consoleOutputRe.Match(line) &&
			(!ctx.questionableRe.Match(line) || bytes.Contains(line, ctx.eoi)) {
			lineStart := bytes.Index(line, []byte("] ")) + pos + 2
			lineEnd := next
			if lineEnd != 0 && output[lineEnd-1] == '\r' {
				lineEnd--
			}
			if oops == nil {
				consoleReportPrefix = append(consoleReportPrefix,
					append([]byte{}, output[lineStart:lineEnd]...))
				if len(consoleReportPrefix) > 5 {
					consoleReportPrefix = consoleReportPrefix[1:]
				}
			} else {
				textLines++
				ln := output[lineStart:lineEnd]
				skipLine := skipText
				if bytes.Contains(ln, []byte("Disabling lock debugging due to kernel taint")) {
					skipLine = true
				} else if textLines > 25 &&
					bytes.Contains(ln, []byte("Kernel panic - not syncing")) {
					// If panic_on_warn set, then we frequently have 2 stacks:
					// one for the actual report (or maybe even more than one),
					// and then one for panic caused by panic_on_warn. This makes
					// reports unnecessary long and the panic (current) stack
					// is always present in the actual report. So we strip the
					// panic message. However, we check that we have enough lines
					// before the panic, because sometimes we have, for example,
					// a single WARNING line without a stack and then the panic
					// with the stack.
					skipText = true
					skipLine = true
				}
				if !skipLine {
					consoleReport = append(consoleReport, ln...)
					consoleReport = append(consoleReport, '\n')
					if secondReportPos == 0 {
						firstReportEnd = len(consoleReport)
					}
				}
			}
		}
		pos = next + 1
	}
	if oops == nil {
		return
	}
	if secondReportPos == 0 {
		secondReportPos = len(output)
	}
	logReport = output[startPos:secondReportPos]
	consoleReportReliable = consoleReport[:firstReportEnd]
	return
}

func (ctx *linux) Symbolize(rep *Report) error {
	if ctx.vmlinux == "" {
		return nil
	}
	symbolized, err := ctx.symbolize(rep.Report)
	if err != nil {
		return err
	}
	rep.Report = symbolized
	guiltyFile := ctx.extractGuiltyFile(rep.Report)
	if guiltyFile != "" {
		rep.Maintainers, err = ctx.getMaintainers(guiltyFile)
		if err != nil {
			return err
		}
	}
	return nil
}

func (ctx *linux) symbolize(text []byte) ([]byte, error) {
	symb := symbolizer.NewSymbolizer()
	defer symb.Close()
	strip := ctx.stripPrefix(symb)
	var symbolized []byte
	s := bufio.NewScanner(bytes.NewReader(text))
	for s.Scan() {
		line := append([]byte{}, s.Bytes()...)
		line = append(line, '\n')
		line = symbolizeLine(symb.Symbolize, ctx.symbols, ctx.vmlinux, strip, line)
		symbolized = append(symbolized, line...)
	}
	return symbolized, nil
}

func (ctx *linux) stripPrefix(symb *symbolizer.Symbolizer) string {
	// Vmlinux may have been moved, so check if we can find debug info
	// for some known functions and infer correct strip prefix from it.
	knownSymbols := []struct {
		symbol string
		file   string
	}{
		{"__sanitizer_cov_trace_pc", "kernel/kcov.c"},
		{"__asan_load1", "mm/kasan/kasan.c"},
		{"start_kernel", "init/main.c"},
	}
	for _, s := range knownSymbols {
		for _, covSymb := range ctx.symbols[s.symbol] {
			frames, _ := symb.Symbolize(ctx.vmlinux, covSymb.Addr)
			if len(frames) > 0 {
				file := frames[len(frames)-1].File
				if idx := strings.Index(file, s.file); idx != -1 {
					return file[:idx]
				}
			}
		}
	}
	// Strip vmlinux location from all paths.
	strip, _ := filepath.Abs(ctx.vmlinux)
	return filepath.Dir(strip) + string(filepath.Separator)
}

func symbolizeLine(symbFunc func(bin string, pc uint64) ([]symbolizer.Frame, error),
	symbols map[string][]symbolizer.Symbol, vmlinux, strip string, line []byte) []byte {
	match := linuxSymbolizeRe.FindSubmatchIndex(line)
	if match == nil {
		return line
	}
	fn := line[match[2]:match[3]]
	off, err := strconv.ParseUint(string(line[match[4]:match[5]]), 16, 64)
	if err != nil {
		return line
	}
	size, err := strconv.ParseUint(string(line[match[6]:match[7]]), 16, 64)
	if err != nil {
		return line
	}
	symb := symbols[string(fn)]
	if len(symb) == 0 {
		return line
	}
	var funcStart uint64
	for _, s := range symb {
		if funcStart == 0 || int(size) == s.Size {
			funcStart = s.Addr
		}
	}
	frames, err := symbFunc(vmlinux, funcStart+off-1)
	if err != nil || len(frames) == 0 {
		return line
	}
	var symbolized []byte
	for _, frame := range frames {
		file := frame.File
		file = strings.TrimPrefix(file, strip)
		file = strings.TrimPrefix(file, "./")
		info := fmt.Sprintf(" %v:%v", file, frame.Line)
		modified := append([]byte{}, line...)
		modified = replace(modified, match[7], match[7], []byte(info))
		if frame.Inline {
			end := match[7] + len(info)
			modified = replace(modified, end, end, []byte(" [inline]"))
			modified = replace(modified, match[2], match[7], []byte(frame.Func))
		}
		symbolized = append(symbolized, modified...)
	}
	return symbolized
}

func (ctx *linux) extractGuiltyFile(report []byte) string {
	if linuxRcuStall.Match(report) {
		// Special case for rcu stalls.
		// There are too many frames that we want to skip before actual guilty frames,
		// we would need to blacklist too many files and that would be fragile.
		// So instead we try to extract guilty file starting from the known
		// interrupt entry point first.
		if pos := bytes.Index(report, []byte(" apic_timer_interrupt+0x")); pos != -1 {
			if file := ctx.extractGuiltyFileImpl(report[pos:]); file != "" {
				return file
			}
		}
	}
	return ctx.extractGuiltyFileImpl(report)
}

func (ctx *linux) extractGuiltyFileImpl(report []byte) string {
	files := ctx.extractFiles(report)
nextFile:
	for _, file := range files {
		for _, re := range ctx.guiltyFileBlacklist {
			if re.MatchString(file) {
				continue nextFile
			}
		}
		return file
	}
	return ""
}

func (ctx *linux) getMaintainers(file string) ([]string, error) {
	mtrs, err := ctx.getMaintainersImpl(file, false)
	if err != nil {
		return nil, err
	}
	if len(mtrs) <= 1 {
		mtrs, err = ctx.getMaintainersImpl(file, true)
		if err != nil {
			return nil, err
		}
	}
	return mtrs, nil
}

func (ctx *linux) getMaintainersImpl(file string, blame bool) ([]string, error) {
	args := []string{"--no-n", "--no-rolestats"}
	if blame {
		args = append(args, "--git-blame")
	}
	args = append(args, file)
	output, err := osutil.RunCmd(time.Minute, ctx.kernelSrc, filepath.FromSlash("scripts/get_maintainer.pl"), args...)
	if err != nil {
		return nil, err
	}
	lines := strings.Split(string(output), "\n")
	var mtrs []string
	for _, line := range lines {
		addr, err := mail.ParseAddress(line)
		if err != nil {
			continue
		}
		mtrs = append(mtrs, addr.Address)
	}
	return mtrs, nil
}

func (ctx *linux) extractFiles(report []byte) []string {
	matches := filenameRe.FindAll(report, -1)
	var files []string
	for _, match := range matches {
		f := string(bytes.Split(match, []byte{':'})[0])
		files = append(files, filepath.Clean(f))
	}
	return files
}

func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (bool, string) {
	// Check if the report contains stack trace.
	if !format.noStackTrace && !bytes.Contains(report, []byte("Call Trace")) &&
		!bytes.Contains(report, []byte("backtrace")) {
		return true, "no stack trace in report"
	}
	// Check for common title corruptions.
	for _, re := range linuxCorruptedTitles {
		if re.MatchString(title) {
			return true, "title matches corrupted regexp"
		}
	}
	// When a report contains 'Call Trace', 'backtrace', 'Allocated' or 'Freed' keywords,
	// it must also contain at least a single stack frame after each of them.
	for _, key := range linuxStackKeywords {
		match := key.FindSubmatchIndex(report)
		if match == nil {
			continue
		}
		frames := bytes.Split(report[match[0]:], []byte{'\n'})
		if len(frames) < 4 {
			return true, "call trace is missed"
		}
		frames = frames[1:]
		corrupted := true
		// Check that at least one of the next few lines contains a frame.
	outer:
		for i := 0; i < 15 && i < len(frames); i++ {
			for _, key1 := range linuxStackKeywords {
				// Next stack trace starts.
				if key1.Match(frames[i]) {
					break outer
				}
			}
			if bytes.Contains(frames[i], []byte("(stack is not available)")) ||
				stackFrameRe.Match(frames[i]) {
				corrupted = false
				break
			}
		}
		if corrupted {
			return true, "no frames in a stack trace"
		}
	}
	return false, ""
}

var (
	linuxSymbolizeRe = regexp.MustCompile(`(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`)
	stackFrameRe     = regexp.MustCompile(`^ *(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`)
	linuxRcuStall    = compile("INFO: rcu_(?:preempt|sched|bh) (?:self-)?detected(?: expedited)? stall")
	linuxRipFrame    = compile(`IP: (?:(?:[0-9]+:)?(?:{{PC}} +){0,2}{{FUNC}}|[0-9]+:0x[0-9a-f]+|(?:[0-9]+:)?{{PC}} +\[< *\(null\)>\] +\(null\)|[0-9]+: +\(null\))`)
)

var linuxCorruptedTitles = []*regexp.Regexp{
	// Sometimes timestamps get merged into the middle of report description.
	regexp.MustCompile(`\[ *[0-9]+\.[0-9]+\]`),
}

var linuxStackKeywords = []*regexp.Regexp{
	regexp.MustCompile(`Call Trace`),
	regexp.MustCompile(`Allocated`),
	regexp.MustCompile(`Freed`),
	// Match 'backtrace:', but exclude 'stack backtrace:'
	regexp.MustCompile(`[^k] backtrace:`),
}

var linuxStackParams = &stackParams{
	stackStartRes: linuxStackKeywords,
	frameRes: []*regexp.Regexp{
		compile("^ +(?:{{PC}} )?{{FUNC}}"),
	},
	skipPatterns: []string{
		"__sanitizer",
		"__asan",
		"kasan",
		"check_memory_region",
		"print_address_description",
		"panic",
		"invalid_op",
		"report_bug",
		"fixup_bug",
		"do_error",
		"invalid_op",
		"_trap",
		"dump_stack",
		"warn_slowpath",
		"warn_alloc",
		"__warn",
		"debug_object",
		"work_is_static_object",
		"lockdep",
		"perf_trace",
		"lock_acquire",
		"lock_release",
		"register_lock_class",
		"spin_lock",
		"spin_unlock",
		"raw_read_lock",
		"raw_write_lock",
		"down_read",
		"down_write",
		"down_read_trylock",
		"down_write_trylock",
		"up_read",
		"up_write",
		"mutex_lock",
		"mutex_unlock",
		"memcpy",
		"memcmp",
		"memset",
		"strcmp",
		"strcpy",
		"strlen",
		"copy_to_user",
		"copy_from_user",
		"put_user",
		"get_user",
		"might_fault",
		"might_sleep",
		"list_add",
		"list_del",
		"list_replace",
		"list_move",
		"list_splice",
	},
	corruptedLines: []*regexp.Regexp{
		// Fault injection stacks are frequently intermixed with crash reports.
		compile(`^ should_fail(\.[a-z]+\.[0-9]+)?\+0x`),
		compile(`^ should_failslab(\.[a-z]+\.[0-9]+)?\+0x`),
	},
}

func warningStackFmt(skip ...string) *stackFmt {
	return &stackFmt{
		// In newer kernels WARNING traps and actual stack starts after invalid_op frame,
		// older kernels just print stack.
		parts: []*regexp.Regexp{
			linuxRipFrame,
			parseStackTrace,
		},
		parts2: []*regexp.Regexp{
			compile("Call Trace:"),
			parseStackTrace,
		},
		skip: skip,
	}
}

var linuxOopses = []*oops{
	{
		[]byte("BUG:"),
		[]oopsFormat{
			{
				title:  compile("BUG: KASAN:"),
				report: compile("BUG: KASAN: ([a-z\\-]+) in {{FUNC}}(?:.*\\n)+?.*(Read|Write) of size (?:[0-9]+)"),

				fmt: "KASAN: %[1]v %[3]v in %[4]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("BUG: KASAN: (?:[a-z\\-]+) in {{FUNC}}"),
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
			{
				title:  compile("BUG: KASAN:"),
				report: compile("BUG: KASAN: double-free or invalid-free in {{FUNC}}"),
				fmt:    "KASAN: invalid-free in %[2]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("BUG: KASAN: double-free or invalid-free in {{FUNC}}"),
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"kmem_", "slab_", "kfree", "vunmap", "vfree"},
				},
			},
			{
				title: compile("BUG: KASAN: ([a-z\\-]+) on address(?:.*\\n)+?.*(Read|Write) of size ([0-9]+)"),
				fmt:   "KASAN: %[1]v %[2]v",
			},
			{
				title:     compile("BUG: KASAN: (.*)"),
				fmt:       "KASAN: %[1]v",
				corrupted: true,
			},
			{
				title: compile("BUG: KMSAN: (.*)"),
				fmt:   "KMSAN: %[1]v",
			},
			{
				title: compile("BUG: unable to handle kernel paging request"),
				fmt:   "BUG: unable to handle kernel paging request in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						linuxRipFrame,
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
			{
				title: compile("BUG: unable to handle kernel NULL pointer dereference"),
				fmt:   "BUG: unable to handle kernel NULL pointer dereference in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						linuxRipFrame,
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
			{
				// Sometimes with such BUG failures, the second part of the header doesn't get printed
				// or gets corrupted, because kernel prints it as two separate printk() calls.
				title:     compile("BUG: unable to handle kernel"),
				fmt:       "BUG: unable to handle kernel",
				corrupted: true,
			},
			{
				title: compile("BUG: spinlock (lockup suspected|already unlocked|recursion|bad magic|wrong owner|wrong CPU)"),
				fmt:   "BUG: spinlock %[1]v in %[2]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"spin_"},
				},
			},
			{
				title: compile("BUG: soft lockup"),
				fmt:   "BUG: soft lockup in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
			{
				title:  compile("BUG: .*still has locks held!"),
				report: compile("BUG: .*still has locks held!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"),
				fmt:    "BUG: still has locks held in %[1]v",
			},
			{
				title:        compile("BUG: lock held when returning to user space"),
				report:       compile("BUG: lock held when returning to user space(?:.*\\n)+?.*leaving the kernel with locks still held(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:          "BUG: lock held when returning to user space in %[1]v",
				noStackTrace: true,
			},
			{
				title:  compile("BUG: bad unlock balance detected!"),
				report: compile("BUG: bad unlock balance detected!(?:.*\\n){0,15}?.*is trying to release lock(?:.*\\n){0,15}?.*{{PC}} +{{FUNC}}"),
				fmt:    "BUG: bad unlock balance in %[1]v",
			},
			{
				title:  compile("BUG: held lock freed!"),
				report: compile("BUG: held lock freed!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"),
				fmt:    "BUG: held lock freed in %[1]v",
			},
			{
				title:        compile("BUG: Bad rss-counter state"),
				fmt:          "BUG: Bad rss-counter state",
				noStackTrace: true,
			},
			{
				title:        compile("BUG: non-zero nr_ptes on freeing mm"),
				fmt:          "BUG: non-zero nr_ptes on freeing mm",
				noStackTrace: true,
			},
			{
				title:        compile("BUG: non-zero nr_pmds on freeing mm"),
				fmt:          "BUG: non-zero nr_pmds on freeing mm",
				noStackTrace: true,
			},
			{
				title: compile("BUG: Dentry .* still in use \\([0-9]+\\) \\[unmount of ([^\\]]+)\\]"),
				fmt:   "BUG: Dentry still in use [unmount of %[1]v]",
			},
			{
				title: compile("BUG: Bad page state"),
				fmt:   "BUG: Bad page state",
			},
			{
				title: compile("BUG: Bad page map"),
				fmt:   "BUG: Bad page map",
			},
			{
				title:        compile("BUG: workqueue lockup"),
				fmt:          "BUG: workqueue lockup",
				noStackTrace: true,
			},
			{
				title: compile("BUG: sleeping function called from invalid context (.*)"),
				fmt:   "BUG: sleeping function called from invalid context %[1]v",
			},
			{
				title: compile("BUG: using __this_cpu_([a-z_]+)\\(\\) in preemptible"),
				fmt:   "BUG: using __this_cpu_%[1]v() in preemptible code in %[2]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"dump_stack", "preemption", "preempt"},
				},
			},
			{
				title: compile("BUG: workqueue leaked lock or atomic"),
				report: compile("BUG: workqueue leaked lock or atomic(?:.*\\n)+?" +
					".*last function: ([a-zA-Z0-9_]+)\\n"),
				fmt:          "BUG: workqueue leaked lock or atomic in %[1]v",
				noStackTrace: true,
			},
			{
				title:        compile("BUG: executor-detected bug"),
				fmt:          "BUG: executor-detected bug",
				noStackTrace: true,
			},
			{
				title: compile("BUG: memory leak"),
				fmt:   "memory leak in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("backtrace:"),
						parseStackTrace,
					},
					skip: []string{"kmemleak", "kmalloc", "kcalloc", "kzalloc",
						"vmalloc", "kmem", "slab", "alloc", "create_object"},
				},
			},
		},
		[]*regexp.Regexp{
			// CONFIG_DEBUG_OBJECTS output.
			compile("ODEBUG:"),
			// Android prints this sometimes during boot.
			compile("Boot_DEBUG:"),
			// pkg/host output in debug mode.
			compile("BUG: no syscalls can create resource"),
		},
	},
	{
		[]byte("WARNING:"),
		[]oopsFormat{
			{
				title: compile("WARNING: .*lib/debugobjects\\.c.* debug_print_object"),
				fmt:   "WARNING: ODEBUG bug in %[1]v",
				// Skip all users of ODEBUG as well.
				stack: warningStackFmt("debug_", "rcu", "hrtimer_", "timer_",
					"work_", "percpu_", "kmem_", "slab_", "kfree", "vunmap", "vfree"),
			},
			{
				title: compile("WARNING: .*mm/usercopy\\.c.* usercopy_warn"),
				fmt:   "WARNING: bad usercopy in %[1]v",
				stack: warningStackFmt("usercopy", "__check"),
			},
			{
				title: compile("WARNING: .*lib/kobject\\.c.* kobject_"),
				fmt:   "WARNING: kobject bug in %[1]v",
				stack: warningStackFmt("kobject_"),
			},
			{
				title: compile("WARNING: .*fs/proc/generic\\.c.* proc_register"),
				fmt:   "WARNING: proc registration bug in %[1]v",
				stack: warningStackFmt("proc_"),
			},
			{
				title: compile("WARNING: .*lib/refcount\\.c.* refcount_"),
				fmt:   "WARNING: refcount bug in %[1]v",
				stack: warningStackFmt("refcount"),
			},
			{
				title: compile("WARNING: .*kernel/locking/lockdep\\.c.*lock_"),
				fmt:   "WARNING: locking bug in %[1]v",
				stack: warningStackFmt(),
			},
			{
				title:        compile("WARNING: lock held when returning to user space"),
				report:       compile("WARNING: lock held when returning to user space(?:.*\\n)+?.*leaving the kernel with locks still held(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:          "WARNING: lock held when returning to user space in %[1]v",
				noStackTrace: true,
			},
			{
				title: compile("WARNING: .*mm/.*\\.c.* k?.?malloc"),
				fmt:   "WARNING: kmalloc bug in %[1]v",
				stack: warningStackFmt("kmalloc", "kcalloc", "kzalloc", "krealloc",
					"vmalloc", "slab", "kmem"),
			},
			{
				title: compile("WARNING: .* at {{SRC}} {{FUNC}}"),
				fmt:   "WARNING in %[2]v",
			},
			{
				title:  compile("WARNING: possible circular locking dependency detected"),
				report: compile("WARNING: possible circular locking dependency detected(?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("WARNING: possible irq lock inversion dependency detected"),
				report: compile("WARNING: possible irq lock inversion dependency detected(?:.*\\n)+?.*just changed the state of lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detecte"),
				report: compile("WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected(?:.*\\n)+?.*is trying to acquire(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("WARNING: possible recursive locking detected"),
				report: compile("WARNING: possible recursive locking detected(?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("WARNING: inconsistent lock state"),
				report: compile("WARNING: inconsistent lock state(?:.*\\n)+?.*takes(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
				fmt:    "inconsistent lock state in %[1]v",
			},
			{
				title:  compile("WARNING: suspicious RCU usage"),
				report: compile("WARNING: suspicious RCU usage(?:.*\n)+?.*?{{SRC}}"),
				fmt:    "WARNING: suspicious RCU usage in %[2]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"rcu", "kmem", "slab", "kmalloc",
						"vmalloc", "kcalloc", "kzalloc"},
				},
			},
			{
				title:        compile("WARNING: kernel stack regs at [0-9a-f]+ in [^ ]* has bad '([^']+)' value"),
				fmt:          "WARNING: kernel stack regs has bad '%[1]v' value",
				noStackTrace: true,
			},
			{
				title:        compile("WARNING: kernel stack frame pointer at [0-9a-f]+ in [^ ]* has bad value"),
				fmt:          "WARNING: kernel stack frame pointer has bad value",
				noStackTrace: true,
			},
			{
				title:  compile("WARNING: bad unlock balance detected!"),
				report: compile("WARNING: bad unlock balance detected!(?:.*\\n){0,15}?.*is trying to release lock(?:.*\\n){0,15}?.*{{PC}} +{{FUNC}}"),
				fmt:    "WARNING: bad unlock balance in %[1]v",
			},
			{
				title:  compile("WARNING: held lock freed!"),
				report: compile("WARNING: held lock freed!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"),
				fmt:    "WARNING: held lock freed in %[1]v",
			},
			{
				title:        compile("WARNING: kernel stack regs .* has bad 'bp' value"),
				fmt:          "WARNING: kernel stack regs has bad value",
				noStackTrace: true,
			},
			{
				title:        compile("WARNING: kernel stack frame pointer .* has bad value"),
				fmt:          "WARNING: kernel stack regs has bad value",
				noStackTrace: true,
			},
		},
		[]*regexp.Regexp{
			compile("WARNING: /etc/ssh/moduli does not exist, using fixed modulus"), // printed by sshd
		},
	},
	{
		[]byte("INFO:"),
		[]oopsFormat{
			{
				title:  compile("INFO: possible circular locking dependency detected"),
				report: compile("INFO: possible circular locking dependency detected \\](?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("INFO: possible irq lock inversion dependency detected"),
				report: compile("INFO: possible irq lock inversion dependency detected \\](?:.*\\n)+?.*just changed the state of lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected"),
				report: compile("INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected \\](?:.*\\n)+?.*is trying to acquire(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("INFO: possible recursive locking detected"),
				report: compile("INFO: possible recursive locking detected \\](?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
				fmt:    "possible deadlock in %[1]v",
			},
			{
				title:  compile("INFO: inconsistent lock state"),
				report: compile("INFO: inconsistent lock state \\](?:.*\\n)+?.*takes(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
				fmt:    "inconsistent lock state in %[1]v",
			},
			{
				title: linuxRcuStall,
				fmt:   "INFO: rcu detected stall in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("apic_timer_interrupt"),
						parseStackTrace,
					},
					skip: []string{"apic_timer_interrupt", "rcu"},
				},
			},
			{
				title: linuxRcuStall,
				fmt:   "INFO: rcu detected stall in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						linuxRipFrame,
						parseStackTrace,
					},
					skip: []string{"apic_timer_interrupt", "rcu"},
				},
			},
			{
				title: compile("INFO: trying to register non-static key"),
				fmt:   "INFO: trying to register non-static key in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"stack", "lock", "IRQ"},
				},
			},
			{
				title:  compile("INFO: suspicious RCU usage"),
				report: compile("INFO: suspicious RCU usage(?:.*\n)+?.*?{{SRC}}"),
				fmt:    "INFO: suspicious RCU usage in %[2]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"rcu", "kmem", "slab", "kmalloc",
						"vmalloc", "kcalloc", "kzalloc"},
				},
			},
			{
				title: compile("INFO: task .* blocked for more than [0-9]+ seconds"),
				fmt:   "INFO: task hung in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
					skip: []string{"sched", "_lock", "down", "completion", "kthread",
						"wait", "synchronize"},
				},
			},
			{
				// This gets captured for corrupted old-style KASAN reports.
				title:     compile("INFO: (Freed|Allocated) in (.*)"),
				fmt:       "INFO: %[1]v in %[2]v",
				corrupted: true,
			},
		},
		[]*regexp.Regexp{
			compile("INFO: lockdep is turned off"),
			compile("INFO: Stall ended before state dump start"),
			compile("INFO: NMI handler"),
			compile("(handler|interrupt).*took too long"),
			compile("_INFO::"),                                       // Android can print this during boot.
			compile("INFO: sys_.* is not present in /proc/kallsyms"), // pkg/host output in debug mode
			compile("INFO: no syscalls can create resource"),         // pkg/host output in debug mode
		},
	},
	{
		[]byte("Unable to handle kernel paging request"),
		[]oopsFormat{
			{
				title:  compile("Unable to handle kernel paging request"),
				report: compile("Unable to handle kernel paging request(?:.*\\n)+?.*PC is at {{FUNC}}"),
				fmt:    "unable to handle kernel paging request in %[1]v",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("general protection fault:"),
		[]oopsFormat{
			{
				title: compile("general protection fault:"),
				fmt:   "general protection fault in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						linuxRipFrame,
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("Kernel panic"),
		[]oopsFormat{
			{
				title: compile("Kernel panic - not syncing: Attempted to kill init!"),
				fmt:   "kernel panic: Attempted to kill init!",
			},
			{
				title: compile("Kernel panic - not syncing: Couldn't open N_TTY ldisc for [^ ]+ --- error -[0-9]+"),
				fmt:   "kernel panic: Couldn't open N_TTY ldisc",
			},
			{
				// 'kernel panic: Fatal exception' is usually printed after BUG,
				// so if we captured it as a report description, that means the
				// report got truncated and we missed the actual BUG header.
				title:     compile("Kernel panic - not syncing: Fatal exception"),
				fmt:       "kernel panic: Fatal exception",
				corrupted: true,
			},
			{
				// Same, but for WARNINGs and KASAN reports.
				title:     compile("Kernel panic - not syncing: panic_on_warn set"),
				fmt:       "kernel panic: panic_on_warn set",
				corrupted: true,
			},
			{
				// Same, but for task hung reports.
				title:     compile("Kernel panic - not syncing: hung_task: blocked tasks"),
				fmt:       "kernel panic: hung_task: blocked tasks",
				corrupted: true,
			},
			{
				title: compile("Kernel panic - not syncing: (.*)"),
				fmt:   "kernel panic: %[1]v",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("kernel BUG"),
		[]oopsFormat{
			{
				title: compile("kernel BUG at mm/usercopy.c"),
				fmt:   "BUG: bad usercopy in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
			{
				title: compile("kernel BUG at lib/list_debug.c"),
				fmt:   "BUG: corrupted list in %[1]v",
				stack: &stackFmt{
					parts: []*regexp.Regexp{
						compile("Call Trace:"),
						parseStackTrace,
					},
				},
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("Kernel BUG"),
		[]oopsFormat{
			{
				title: compile("Kernel BUG (.*)"),
				fmt:   "kernel BUG %[1]v",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("BUG kmalloc-"),
		[]oopsFormat{
			{
				title: compile("BUG kmalloc-.*: Object already free"),
				fmt:   "BUG: Object already free",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("divide error:"),
		[]oopsFormat{
			{
				title:  compile("divide error: "),
				report: compile("divide error: (?:.*\\n)+?.*RIP: [0-9]+:(?:{{PC}} +{{PC}} +)?{{FUNC}}"),
				fmt:    "divide error in %[1]v",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("invalid opcode:"),
		[]oopsFormat{
			{
				title:  compile("invalid opcode: "),
				report: compile("invalid opcode: (?:.*\\n)+?.*RIP: [0-9]+:{{PC}} +{{PC}} +{{FUNC}}"),
				fmt:    "invalid opcode in %[1]v",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("UBSAN:"),
		[]oopsFormat{
			{
				title: compile("UBSAN: (.*)"),
				fmt:   "UBSAN: %[1]v",
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("Booting the kernel."),
		[]oopsFormat{
			{
				title:        compile("Booting the kernel."),
				fmt:          "unexpected kernel reboot",
				noStackTrace: true,
			},
		},
		[]*regexp.Regexp{},
	},
	{
		[]byte("unregister_netdevice: waiting for"),
		[]oopsFormat{
			{
				title:        compile("unregister_netdevice: waiting for (?:.*) to become free"),
				fmt:          "unregister_netdevice: waiting for DEV to become free",
				noStackTrace: true,
			},
		},
		[]*regexp.Regexp{},
	},
}