| // Copyright 2017 syzkaller project authors. All rights reserved. |
| // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. |
| |
| package main |
| |
| import ( |
| "context" |
| "fmt" |
| "regexp" |
| "strconv" |
| "strings" |
| "time" |
| |
| "github.com/google/syzkaller/dashboard/dashapi" |
| "github.com/google/syzkaller/pkg/hash" |
| "github.com/google/syzkaller/pkg/subsystem" |
| db "google.golang.org/appengine/v2/datastore" |
| ) |
| |
| // This file contains definitions of entities stored in datastore. |
| |
| const ( |
| maxTextLen = 200 |
| MaxStringLen = 1024 |
| |
| maxBugHistoryDays = 365 * 5 |
| ) |
| |
| type Manager struct { |
| Namespace string |
| Name string |
| Link string |
| CurrentBuild string |
| FailedBuildBug string |
| FailedSyzBuildBug string |
| LastAlive time.Time |
| CurrentUpTime time.Duration |
| LastGeneratedJob time.Time |
| } |
| |
| // ManagerStats holds per-day manager runtime stats. |
| // Has Manager as parent entity. Keyed by Date. |
| type ManagerStats struct { |
| Date int // YYYYMMDD |
| MaxCorpus int64 |
| MaxPCs int64 // coverage |
| MaxCover int64 // what we call feedback signal everywhere else |
| TotalFuzzingTime time.Duration |
| TotalCrashes int64 |
| CrashTypes int64 // unique crash types |
| SuppressedCrashes int64 |
| TotalExecs int64 |
| // These are only recorded once right after corpus is triaged. |
| TriagedCoverage int64 |
| TriagedPCs int64 |
| } |
| |
| type Asset struct { |
| Type dashapi.AssetType |
| DownloadURL string |
| CreateDate time.Time |
| } |
| |
| type Build struct { |
| Namespace string |
| Manager string |
| ID string // unique ID generated by syz-ci |
| Type BuildType |
| Time time.Time |
| OS string |
| Arch string |
| VMArch string |
| SyzkallerCommit string |
| SyzkallerCommitDate time.Time |
| CompilerID string |
| KernelRepo string |
| KernelBranch string |
| KernelCommit string |
| KernelCommitTitle string `datastore:",noindex"` |
| KernelCommitDate time.Time `datastore:",noindex"` |
| KernelConfig int64 // reference to KernelConfig text entity |
| Assets []Asset // build-related assets |
| AssetsLastCheck time.Time // the last time we checked the assets for deprecation |
| } |
| |
| type Bug struct { |
| Namespace string |
| Seq int64 // sequences of the bug with the same title |
| Title string |
| MergedTitles []string // crash titles that we already merged into this bug |
| AltTitles []string // alternative crash titles that we may merge into this bug |
| Status int |
| StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why |
| DupOf string |
| NumCrashes int64 |
| NumRepro int64 |
| // ReproLevel is the best ever found repro level for this bug. |
| // HeadReproLevel is best known repro level that still works on the HEAD commit. |
| ReproLevel dashapi.ReproLevel |
| HeadReproLevel dashapi.ReproLevel `datastore:"HeadReproLevel"` |
| BisectCause BisectStatus |
| BisectFix BisectStatus |
| HasReport bool |
| NeedCommitInfo bool |
| FirstTime time.Time |
| LastTime time.Time |
| LastSavedCrash time.Time |
| LastReproTime time.Time |
| LastCauseBisect time.Time |
| FixTime time.Time // when we become aware of the fixing commit |
| LastActivity time.Time // last time we observed any activity related to the bug |
| Closed time.Time |
| SubsystemsTime time.Time // when we have updated subsystems last time |
| SubsystemsRev int |
| Reporting []BugReporting |
| Commits []string // titles of fixing commmits |
| CommitInfo []Commit // additional info for commits (for historical reasons parallel array to Commits) |
| HappenedOn []string // list of managers |
| PatchedOn []string `datastore:",noindex"` // list of managers |
| UNCC []string // don't CC these emails on this bug |
| // Kcidb publishing status bitmask: |
| // bit 0 - the bug is published |
| // bit 1 - don't want to publish it (syzkaller build/test errors) |
| KcidbStatus int64 |
| DailyStats []BugDailyStats |
| Labels []BugLabel |
| DiscussionInfo []BugDiscussionInfo |
| TreeTests BugTreeTestInfo |
| // FixCandidateJob holds the key of the latest successful cross-tree fix bisection job. |
| FixCandidateJob string |
| ReproAttempts []BugReproAttempt |
| } |
| |
| type BugTreeTestInfo struct { |
| // NeedPoll is set to true if this bug needs to be considered ASAP. |
| NeedPoll bool |
| // NextPoll can be used to delay the next inspection of the bug. |
| NextPoll time.Time |
| // List contains latest data about cross-tree patch tests. |
| List []BugTreeTest |
| } |
| |
| type BugTreeTest struct { |
| CrashID int64 |
| Repo string |
| Branch string // May be also equal to a commit. |
| // If the values below are set, the testing was done on a merge base. |
| MergeBaseRepo string |
| MergeBaseBranch string |
| // Below are job keys. |
| First string // The first job that finished successfully. |
| FirstOK string |
| FirstCrash string |
| Last string |
| Error string // If some job succeeds afterwards, it should be cleared. |
| Pending string |
| } |
| |
| type BugLabelType string |
| |
| type BugLabel struct { |
| Label BugLabelType |
| // Either empty (for flags) or contains the value. |
| Value string |
| // The email of the user who manually set this subsystem tag. |
| // If empty, the label was set automatically. |
| SetBy string |
| // Link to the message. |
| Link string |
| } |
| |
| func (label BugLabel) String() string { |
| if label.Value == "" { |
| return string(label.Label) |
| } |
| return string(label.Label) + ":" + label.Value |
| } |
| |
| // BugReproAttempt describes a single attempt to generate a repro for a bug. |
| type BugReproAttempt struct { |
| Time time.Time |
| Manager string |
| Log int64 |
| } |
| |
| func (bug *Bug) SetAutoSubsystems(c context.Context, list []*subsystem.Subsystem, now time.Time, rev int) { |
| bug.SubsystemsRev = rev |
| bug.SubsystemsTime = now |
| var objects []BugLabel |
| for _, item := range list { |
| objects = append(objects, BugLabel{Label: SubsystemLabel, Value: item.Name}) |
| } |
| bug.SetLabels(makeLabelSet(c, bug.Namespace), objects) |
| } |
| |
| func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) error) error { |
| tx := func(c context.Context) error { |
| bug := new(Bug) |
| if err := db.Get(c, bugKey, bug); err != nil { |
| return fmt.Errorf("failed to get bug: %w", err) |
| } |
| err := transform(bug) |
| if err != nil { |
| return err |
| } |
| if _, err := db.Put(c, bugKey, bug); err != nil { |
| return fmt.Errorf("failed to put bug: %w", err) |
| } |
| return nil |
| } |
| return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) |
| } |
| |
| func (bug *Bug) hasUserSubsystems() bool { |
| return bug.HasUserLabel(SubsystemLabel) |
| } |
| |
| // Initially, subsystem labels were stored as Tags.Subsystems, but over time |
| // it turned out that we'd better store all labels together. |
| // Let's keep this conversion code until "Tags" are removed from all bugs. |
| // Then it can be removed. |
| |
| type Bug202304 struct { |
| Tags BugTags202304 |
| } |
| |
| type BugTags202304 struct { |
| Subsystems []BugTag202304 |
| } |
| |
| type BugTag202304 struct { |
| Name string |
| SetBy string |
| } |
| |
| func (bug *Bug) Load(origProps []db.Property) error { |
| // First filer out Tag properties. |
| var tags, ps []db.Property |
| for _, p := range origProps { |
| if strings.HasPrefix(p.Name, "Tags.") { |
| tags = append(tags, p) |
| } else { |
| ps = append(ps, p) |
| } |
| } |
| if err := db.LoadStruct(bug, ps); err != nil { |
| return err |
| } |
| if len(tags) > 0 { |
| old := Bug202304{} |
| if err := db.LoadStruct(&old, tags); err != nil { |
| return err |
| } |
| for _, entry := range old.Tags.Subsystems { |
| bug.Labels = append(bug.Labels, BugLabel{ |
| Label: SubsystemLabel, |
| SetBy: entry.SetBy, |
| Value: entry.Name, |
| }) |
| } |
| } |
| headReproFound := false |
| for _, p := range ps { |
| if p.Name == "HeadReproLevel" { |
| headReproFound = true |
| break |
| } |
| } |
| if !headReproFound { |
| // The field is new, so it won't be set in all entities. |
| // Assume it to be equal to the best found repro for the bug. |
| bug.HeadReproLevel = bug.ReproLevel |
| } |
| return nil |
| } |
| |
| func (bug *Bug) Save() ([]db.Property, error) { |
| return db.SaveStruct(bug) |
| } |
| |
| type BugDailyStats struct { |
| Date int // YYYYMMDD |
| CrashCount int |
| } |
| |
| type Commit struct { |
| Hash string |
| Title string |
| Author string |
| AuthorName string |
| CC string `datastore:",noindex"` // (|-delimited list) |
| Date time.Time |
| } |
| |
| func (com Commit) toDashapi() *dashapi.Commit { |
| return &dashapi.Commit{ |
| Hash: com.Hash, |
| Title: com.Title, |
| Author: com.Author, |
| AuthorName: com.AuthorName, |
| Date: com.Date, |
| } |
| } |
| |
| type BugDiscussionInfo struct { |
| Source string |
| Summary DiscussionSummary |
| } |
| |
| type DiscussionSummary struct { |
| AllMessages int |
| ExternalMessages int |
| LastMessage time.Time |
| LastPatchMessage time.Time |
| } |
| |
| type BugReporting struct { |
| Name string // refers to Reporting.Name |
| ID string // unique ID per BUG/BugReporting used in commucation with external systems |
| ExtID string // arbitrary reporting ID that is passed back in dashapi.BugReport |
| Link string |
| CC string // additional emails added to CC list (|-delimited list) |
| CrashID int64 // crash that we've last reported in this reporting |
| Auto bool // was it auto-upstreamed/obsoleted? |
| // If Dummy is true, the corresponding Reporting stage was introduced later and the object was just |
| // inserted to preserve consistency across the system. Even though it's indicated as Closed and Reported, |
| // it never actually was. |
| Dummy bool |
| ReproLevel dashapi.ReproLevel // may be less then bug.ReproLevel if repro arrived but we didn't report it yet |
| Labels string // a comma-separated string of already reported labels |
| OnHold time.Time // if set, the bug must not be upstreamed |
| Reported time.Time |
| Closed time.Time |
| } |
| |
| func (r *BugReporting) GetLabels() []string { |
| return strings.Split(r.Labels, ",") |
| } |
| |
| func (r *BugReporting) AddLabel(label string) { |
| newList := unique(append(r.GetLabels(), label)) |
| r.Labels = strings.Join(newList, ",") |
| } |
| |
| type Crash struct { |
| // May be different from bug.Title due to AltTitles. |
| // May be empty for old bugs, in such case bug.Title is the right title. |
| Title string |
| Manager string |
| BuildID string |
| Time time.Time |
| Reported time.Time // set if this crash was ever reported |
| References []CrashReference |
| Maintainers []string `datastore:",noindex"` |
| Log int64 // reference to CrashLog text entity |
| Flags int64 // properties of the Crash |
| Report int64 // reference to CrashReport text entity |
| ReportElements CrashReportElements // parsed parts of the crash report |
| ReproOpts []byte `datastore:",noindex"` |
| ReproSyz int64 // reference to ReproSyz text entity |
| ReproC int64 // reference to ReproC text entity |
| ReproIsRevoked bool // the repro no longer triggers the bug on HEAD |
| LastReproRetest time.Time // the last time when the repro was re-checked |
| MachineInfo int64 // Reference to MachineInfo text entity. |
| // Custom crash priority for reporting (greater values are higher priority). |
| // For example, a crash in mainline kernel has higher priority than a crash in a side branch. |
| // For historical reasons this is called ReportLen. |
| ReportLen int64 |
| Assets []Asset // crash-related assets |
| AssetsLastCheck time.Time // the last time we checked the assets for deprecation |
| } |
| |
| type CrashReportElements struct { |
| GuiltyFiles []string // guilty files as determined during the crash report parsing |
| } |
| |
| type CrashReferenceType string |
| |
| const ( |
| CrashReferenceReporting = "reporting" |
| CrashReferenceJob = "job" |
| // This one is needed for backward compatibility. |
| crashReferenceUnknown = "unknown" |
| ) |
| |
| type CrashReference struct { |
| Type CrashReferenceType |
| // For CrashReferenceReporting, it refers to Reporting.Name |
| // For CrashReferenceJob, it refers to extJobID(jobKey) |
| Key string |
| Time time.Time |
| } |
| |
| func (crash *Crash) AddReference(newRef CrashReference) { |
| crash.Reported = newRef.Time |
| for i, ref := range crash.References { |
| if ref.Type != newRef.Type || ref.Key != newRef.Key { |
| continue |
| } |
| crash.References[i].Time = newRef.Time |
| return |
| } |
| crash.References = append(crash.References, newRef) |
| } |
| |
| func (crash *Crash) ClearReference(t CrashReferenceType, key string) { |
| newRefs := []CrashReference{} |
| crash.Reported = time.Time{} |
| for _, ref := range crash.References { |
| if ref.Type == t && ref.Key == key { |
| continue |
| } |
| if ref.Time.After(crash.Reported) { |
| crash.Reported = ref.Time |
| } |
| newRefs = append(newRefs, ref) |
| } |
| crash.References = newRefs |
| } |
| |
| func (crash *Crash) Load(ps []db.Property) error { |
| if err := db.LoadStruct(crash, ps); err != nil { |
| return err |
| } |
| // Earlier we only relied on Reported, which does not let us reliably unreport a crash. |
| // We need some means of ref counting, so let's create a dummy reference to keep the |
| // crash from being purged. |
| if !crash.Reported.IsZero() && len(crash.References) == 0 { |
| crash.References = append(crash.References, CrashReference{ |
| Type: crashReferenceUnknown, |
| Time: crash.Reported, |
| }) |
| } |
| return nil |
| } |
| |
| func (crash *Crash) Save() ([]db.Property, error) { |
| return db.SaveStruct(crash) |
| } |
| |
| type Discussion struct { |
| ID string // the base message ID |
| Source string |
| Type string |
| Subject string |
| BugKeys []string |
| // Message contains last N messages. |
| // N is supposed to be big enough, so that in almost all cases |
| // AllMessages == len(Messages) holds true. |
| Messages []DiscussionMessage |
| // Since Messages could be trimmed, we have to keep aggregate stats. |
| Summary DiscussionSummary |
| } |
| |
| func discussionKey(c context.Context, source, id string) *db.Key { |
| return db.NewKey(c, "Discussion", fmt.Sprintf("%v-%v", source, id), 0, nil) |
| } |
| |
| func (d *Discussion) key(c context.Context) *db.Key { |
| return discussionKey(c, d.Source, d.ID) |
| } |
| |
| type DiscussionMessage struct { |
| ID string |
| // External is true if the message is not from the bot itself. |
| // Let's use a shorter name to save space. |
| External bool `datastore:"e"` |
| Time time.Time `datastore:",noindex"` |
| } |
| |
| // ReportingState holds dynamic info associated with reporting. |
| type ReportingState struct { |
| Entries []ReportingStateEntry |
| } |
| |
| type ReportingStateEntry struct { |
| Namespace string |
| Name string |
| // Current reporting quota consumption. |
| Sent int |
| Date int // YYYYMMDD |
| } |
| |
| // Subsystem holds the history of grouped per-subsystem open bug reminders. |
| type Subsystem struct { |
| Namespace string |
| Name string |
| // ListsQueried is the last time bug lists were queried for the subsystem. |
| ListsQueried time.Time |
| // LastBugList is the last time we have actually managed to generate a bug list. |
| LastBugList time.Time |
| } |
| |
| // SubsystemReport holds a single report about open bugs in a subsystem. |
| // There'll be one record for moderation (if it's needed) and one for actual reporting. |
| type SubsystemReport struct { |
| Created time.Time |
| BugKeys []string `datastore:",noindex"` |
| TotalStats SubsystemReportStats |
| PeriodStats SubsystemReportStats |
| Stages []SubsystemReportStage |
| } |
| |
| func (r *SubsystemReport) getBugKeys() ([]*db.Key, error) { |
| ret := []*db.Key{} |
| for _, encoded := range r.BugKeys { |
| key, err := db.DecodeKey(encoded) |
| if err != nil { |
| return nil, fmt.Errorf("failed to parse %#v: %w", encoded, err) |
| } |
| ret = append(ret, key) |
| } |
| return ret, nil |
| } |
| |
| func (r *SubsystemReport) findStage(id string) *SubsystemReportStage { |
| for j := range r.Stages { |
| stage := &r.Stages[j] |
| if stage.ID == id { |
| return stage |
| } |
| } |
| return nil |
| } |
| |
| type SubsystemReportStats struct { |
| Reported int |
| LowPrio int |
| Fixed int |
| } |
| |
| func (s *SubsystemReportStats) toDashapi() dashapi.BugListReportStats { |
| return dashapi.BugListReportStats{ |
| Reported: s.Reported, |
| LowPrio: s.LowPrio, |
| Fixed: s.Fixed, |
| } |
| } |
| |
| // There can be at most two stages. |
| // One has Moderation=true, the other one has Moderation=false. |
| type SubsystemReportStage struct { |
| ID string |
| ExtID string |
| Link string |
| Reported time.Time |
| Closed time.Time |
| Moderation bool |
| } |
| |
| // Job represent a single patch testing or bisection job for syz-ci. |
| // Later we may want to extend this to other types of jobs: |
| // - test of a committed fix |
| // - reproduce crash |
| // - test that crash still happens on HEAD |
| // |
| // Job has Bug as parent entity. |
| type Job struct { |
| Type JobType |
| Created time.Time |
| User string |
| CC []string |
| Reporting string |
| ExtID string // email Message-ID |
| Link string // web link for the job (e.g. email in the group) |
| Namespace string |
| Manager string |
| BugTitle string |
| CrashID int64 |
| |
| // Provided by user: |
| KernelRepo string |
| KernelBranch string |
| Patch int64 // reference to Patch text entity |
| KernelConfig int64 // reference to the kernel config entity |
| |
| Attempts int // number of times we tried to execute this job |
| IsRunning bool // the job might have been started, but never finished |
| LastStarted time.Time `datastore:"Started"` |
| Finished time.Time // if set, job is finished |
| TreeOrigin bool // whether the job is related to tree origin detection |
| |
| // If patch test should be done on the merge base between two branches. |
| MergeBaseRepo string |
| MergeBaseBranch string |
| |
| // By default, bisection starts from the revision of the associated crash. |
| // The BisectFrom field can override this. |
| BisectFrom string |
| |
| // Result of execution: |
| CrashTitle string // if empty, we did not hit crash during testing |
| CrashLog int64 // reference to CrashLog text entity |
| CrashReport int64 // reference to CrashReport text entity |
| Commits []Commit |
| BuildID string |
| Log int64 // reference to Log text entity |
| Error int64 // reference to Error text entity, if set job failed |
| Flags dashapi.JobDoneFlags |
| |
| Reported bool // have we reported result back to user? |
| InvalidatedBy string // user who marked this bug as invalid, empty by default |
| BackportedCommit Commit |
| } |
| |
| func (job *Job) IsBisection() bool { |
| return job.Type == JobBisectCause || job.Type == JobBisectFix |
| } |
| |
| func (job *Job) IsFinished() bool { |
| return !job.Finished.IsZero() |
| } |
| |
| type JobType int |
| |
| const ( |
| JobTestPatch JobType = iota |
| JobBisectCause |
| JobBisectFix |
| ) |
| |
| func (typ JobType) toDashapiReportType() dashapi.ReportType { |
| switch typ { |
| case JobTestPatch: |
| return dashapi.ReportTestPatch |
| case JobBisectCause: |
| return dashapi.ReportBisectCause |
| case JobBisectFix: |
| return dashapi.ReportBisectFix |
| default: |
| panic(fmt.Sprintf("unknown job type %v", typ)) |
| } |
| } |
| |
| func (job *Job) isUnreliableBisect() bool { |
| if job.Type != JobBisectCause && job.Type != JobBisectFix { |
| panic(fmt.Sprintf("bad job type %v", job.Type)) |
| } |
| // If a bisection points to a merge or a commit that does not affect the kernel binary, |
| // it is considered an unreliable/wrong result and should not be reported in emails. |
| return job.Flags&dashapi.BisectResultMerge != 0 || |
| job.Flags&dashapi.BisectResultNoop != 0 || |
| job.Flags&dashapi.BisectResultRelease != 0 || |
| job.Flags&dashapi.BisectResultIgnore != 0 |
| } |
| |
| func (job *Job) IsCrossTree() bool { |
| return job.MergeBaseRepo != "" && job.IsBisection() |
| } |
| |
| // Text holds text blobs (crash logs, reports, reproducers, etc). |
| type Text struct { |
| Namespace string |
| Text []byte `datastore:",noindex"` // gzip-compressed text |
| } |
| |
| const ( |
| textCrashLog = "CrashLog" |
| textCrashReport = "CrashReport" |
| textReproSyz = "ReproSyz" |
| textReproC = "ReproC" |
| textMachineInfo = "MachineInfo" |
| textKernelConfig = "KernelConfig" |
| textPatch = "Patch" |
| textLog = "Log" |
| textError = "Error" |
| textReproLog = "ReproLog" |
| ) |
| |
| const ( |
| BugStatusOpen = iota |
| ) |
| |
| const ( |
| BugStatusFixed = 1000 + iota |
| BugStatusInvalid |
| BugStatusDup |
| ) |
| |
| const ( |
| ReproLevelNone = dashapi.ReproLevelNone |
| ReproLevelSyz = dashapi.ReproLevelSyz |
| ReproLevelC = dashapi.ReproLevelC |
| ) |
| |
| type BuildType int |
| |
| const ( |
| BuildNormal BuildType = iota |
| BuildFailed |
| BuildJob |
| ) |
| |
| type BisectStatus int |
| |
| const ( |
| BisectNot BisectStatus = iota |
| BisectPending |
| BisectError |
| BisectYes // have 1 commit |
| BisectUnreliable // have 1 commit, but suspect it's wrong |
| BisectInconclusive // multiple commits due to skips |
| BisectHorizont // happens on the oldest commit we can test (or HEAD for fix bisection) |
| bisectStatusLast // this value can be changed (not stored in datastore) |
| ) |
| |
| func (status BisectStatus) String() string { |
| switch status { |
| case BisectError: |
| return "error" |
| case BisectYes: |
| return "done" |
| case BisectUnreliable: |
| return "unreliable" |
| case BisectInconclusive: |
| return "inconclusive" |
| case BisectHorizont: |
| return "inconclusive" |
| default: |
| return "" |
| } |
| } |
| |
| func mgrKey(c context.Context, ns, name string) *db.Key { |
| return db.NewKey(c, "Manager", fmt.Sprintf("%v-%v", ns, name), 0, nil) |
| } |
| |
| func (mgr *Manager) key(c context.Context) *db.Key { |
| return mgrKey(c, mgr.Namespace, mgr.Name) |
| } |
| |
| func loadManager(c context.Context, ns, name string) (*Manager, error) { |
| mgr := new(Manager) |
| if err := db.Get(c, mgrKey(c, ns, name), mgr); err != nil { |
| if err != db.ErrNoSuchEntity { |
| return nil, fmt.Errorf("failed to get manager %v/%v: %w", ns, name, err) |
| } |
| mgr = &Manager{ |
| Namespace: ns, |
| Name: name, |
| } |
| } |
| return mgr, nil |
| } |
| |
| // updateManager does transactional compare-and-swap on the manager and its current stats. |
| func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, stats *ManagerStats) error) error { |
| date := timeDate(timeNow(c)) |
| tx := func(c context.Context) error { |
| mgr, err := loadManager(c, ns, name) |
| if err != nil { |
| return err |
| } |
| mgrKey := mgr.key(c) |
| stats := new(ManagerStats) |
| statsKey := db.NewKey(c, "ManagerStats", "", int64(date), mgrKey) |
| if err := db.Get(c, statsKey, stats); err != nil { |
| if err != db.ErrNoSuchEntity { |
| return fmt.Errorf("failed to get stats %v/%v/%v: %w", ns, name, date, err) |
| } |
| stats = &ManagerStats{ |
| Date: date, |
| } |
| } |
| |
| if err := fn(mgr, stats); err != nil { |
| return err |
| } |
| |
| if _, err := db.Put(c, mgrKey, mgr); err != nil { |
| return fmt.Errorf("failed to put manager: %w", err) |
| } |
| if _, err := db.Put(c, statsKey, stats); err != nil { |
| return fmt.Errorf("failed to put manager stats: %w", err) |
| } |
| return nil |
| } |
| return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10}) |
| } |
| |
| func loadAllManagers(c context.Context, ns string) ([]*Manager, []*db.Key, error) { |
| var managers []*Manager |
| query := db.NewQuery("Manager") |
| if ns != "" { |
| query = query.Filter("Namespace=", ns) |
| } |
| keys, err := query.GetAll(c, &managers) |
| if err != nil { |
| return nil, nil, fmt.Errorf("failed to query managers: %w", err) |
| } |
| var result []*Manager |
| var resultKeys []*db.Key |
| for i, mgr := range managers { |
| if getNsConfig(c, mgr.Namespace).Managers[mgr.Name].Decommissioned { |
| continue |
| } |
| result = append(result, mgr) |
| resultKeys = append(resultKeys, keys[i]) |
| } |
| return result, resultKeys, nil |
| } |
| |
| func buildKey(c context.Context, ns, id string) *db.Key { |
| if ns == "" { |
| panic("requesting build key outside of namespace") |
| } |
| h := hash.String([]byte(fmt.Sprintf("%v-%v", ns, id))) |
| return db.NewKey(c, "Build", h, 0, nil) |
| } |
| |
| func loadBuild(c context.Context, ns, id string) (*Build, error) { |
| build := new(Build) |
| if err := db.Get(c, buildKey(c, ns, id), build); err != nil { |
| if err == db.ErrNoSuchEntity { |
| return nil, fmt.Errorf("unknown build %v/%v", ns, id) |
| } |
| return nil, fmt.Errorf("failed to get build %v/%v: %w", ns, id, err) |
| } |
| return build, nil |
| } |
| |
| func lastManagerBuild(c context.Context, ns, manager string) (*Build, error) { |
| mgr, err := loadManager(c, ns, manager) |
| if err != nil { |
| return nil, err |
| } |
| if mgr.CurrentBuild == "" { |
| return nil, fmt.Errorf("failed to fetch manager build: no builds") |
| } |
| return loadBuild(c, ns, mgr.CurrentBuild) |
| } |
| |
| func loadBuilds(c context.Context, ns, manager string, typ BuildType) ([]*Build, error) { |
| const limit = 500 |
| var builds []*Build |
| _, err := db.NewQuery("Build"). |
| Filter("Namespace=", ns). |
| Filter("Manager=", manager). |
| Filter("Type=", typ). |
| Order("-Time"). |
| Limit(limit). |
| GetAll(c, &builds) |
| if err != nil { |
| return nil, err |
| } |
| return builds, nil |
| } |
| |
| func (bug *Bug) displayTitle() string { |
| if bug.Seq == 0 { |
| return bug.Title |
| } |
| return fmt.Sprintf("%v (%v)", bug.Title, bug.Seq+1) |
| } |
| |
| var displayTitleRe = regexp.MustCompile(`^(.*) \(([0-9]+)\)$`) |
| |
| func splitDisplayTitle(display string) (string, int64, error) { |
| match := displayTitleRe.FindStringSubmatchIndex(display) |
| if match == nil { |
| return display, 0, nil |
| } |
| title := display[match[2]:match[3]] |
| seqStr := display[match[4]:match[5]] |
| seq, err := strconv.ParseInt(seqStr, 10, 64) |
| if err != nil { |
| return "", 0, fmt.Errorf("failed to parse bug title: %w", err) |
| } |
| if seq <= 0 || seq > 1e6 { |
| return "", 0, fmt.Errorf("failed to parse bug title: seq=%v", seq) |
| } |
| return title, seq - 1, nil |
| } |
| |
| func canonicalBug(c context.Context, bug *Bug) (*Bug, error) { |
| for { |
| if bug.Status != BugStatusDup { |
| return bug, nil |
| } |
| canon := new(Bug) |
| bugKey := db.NewKey(c, "Bug", bug.DupOf, 0, nil) |
| if err := db.Get(c, bugKey, canon); err != nil { |
| return nil, fmt.Errorf("failed to get dup bug %q for %q: %w", |
| bug.DupOf, bug.keyHash(c), err) |
| } |
| bug = canon |
| } |
| } |
| |
| func (bug *Bug) key(c context.Context) *db.Key { |
| return db.NewKey(c, "Bug", bug.keyHash(c), 0, nil) |
| } |
| |
| func (bug *Bug) keyHash(c context.Context) string { |
| return bugKeyHash(c, bug.Namespace, bug.Title, bug.Seq) |
| } |
| |
| func bugKeyHash(c context.Context, ns, title string, seq int64) string { |
| return hash.String([]byte(fmt.Sprintf("%v-%v-%v-%v", getNsConfig(c, ns).Key, ns, title, seq))) |
| } |
| |
| func loadSimilarBugs(c context.Context, bug *Bug) ([]*Bug, error) { |
| domain := getNsConfig(c, bug.Namespace).SimilarityDomain |
| dedup := make(map[string]bool) |
| dedup[bug.keyHash(c)] = true |
| |
| ret := []*Bug{} |
| for _, title := range bug.AltTitles { |
| var similar []*Bug |
| _, err := db.NewQuery("Bug"). |
| Filter("AltTitles=", title). |
| GetAll(c, &similar) |
| if err != nil { |
| return nil, err |
| } |
| for _, bug := range similar { |
| if getNsConfig(c, bug.Namespace).SimilarityDomain != domain || |
| dedup[bug.keyHash(c)] { |
| continue |
| } |
| dedup[bug.keyHash(c)] = true |
| ret = append(ret, bug) |
| } |
| } |
| return ret, nil |
| } |
| |
| // Since these IDs appear in Reported-by tags in commit, we slightly limit their size. |
| const reportingHashLen = 20 |
| |
| func bugReportingHash(bugHash, reporting string) string { |
| return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:reportingHashLen] |
| } |
| |
| func looksLikeReportingHash(id string) bool { |
| // This is only used as best-effort check. |
| // Now we produce 20-chars ids, but we used to use full sha1 hash. |
| return len(id) == reportingHashLen || len(id) == 2*len(hash.Sig{}) |
| } |
| |
| func (bug *Bug) updateCommits(commits []string, now time.Time) { |
| bug.Commits = commits |
| bug.CommitInfo = nil |
| bug.NeedCommitInfo = true |
| bug.FixTime = now |
| bug.PatchedOn = nil |
| } |
| |
| func (bug *Bug) getCommitInfo(i int) Commit { |
| if i < len(bug.CommitInfo) { |
| return bug.CommitInfo[i] |
| } |
| return Commit{} |
| } |
| |
| func (bug *Bug) increaseCrashStats(now time.Time) { |
| bug.NumCrashes++ |
| date := timeDate(now) |
| if len(bug.DailyStats) == 0 || bug.DailyStats[len(bug.DailyStats)-1].Date < date { |
| bug.DailyStats = append(bug.DailyStats, BugDailyStats{date, 1}) |
| } else { |
| // It is theoretically possible that this method might get into a situation, when |
| // the latest saved date is later than now. But we assume that this can only happen |
| // in a small window around the start of the day and it is better to attribute a |
| // crash to the next day than to get a mismatch between NumCrashes and the sum of |
| // CrashCount. |
| bug.DailyStats[len(bug.DailyStats)-1].CrashCount++ |
| } |
| |
| if len(bug.DailyStats) > maxBugHistoryDays { |
| bug.DailyStats = bug.DailyStats[len(bug.DailyStats)-maxBugHistoryDays:] |
| } |
| } |
| |
| func (bug *Bug) dailyStatsTail(from time.Time) []BugDailyStats { |
| startDate := timeDate(from) |
| startPos := len(bug.DailyStats) |
| for ; startPos > 0; startPos-- { |
| if bug.DailyStats[startPos-1].Date < startDate { |
| break |
| } |
| } |
| return bug.DailyStats[startPos:] |
| } |
| |
| func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) { |
| var status dashapi.BugStatus |
| switch bug.Status { |
| case BugStatusOpen: |
| status = dashapi.BugStatusOpen |
| case BugStatusFixed: |
| status = dashapi.BugStatusFixed |
| case BugStatusInvalid: |
| status = dashapi.BugStatusInvalid |
| case BugStatusDup: |
| status = dashapi.BugStatusDup |
| default: |
| return status, fmt.Errorf("unknown bugs status %v", bug.Status) |
| } |
| return status, nil |
| } |
| |
| // If an entity of type EmergencyStop exists, syzbot's operation is paused until |
| // a support engineer deletes it from the DB. |
| type EmergencyStop struct { |
| Time time.Time |
| User string |
| } |
| |
| func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error { |
| crash := new(Crash) |
| crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) |
| if err := db.Get(c, crashKey, crash); err != nil { |
| return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) |
| } |
| crash.AddReference(ref) |
| if _, err := db.Put(c, crashKey, crash); err != nil { |
| return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) |
| } |
| return nil |
| } |
| |
| func removeCrashReference(c context.Context, crashID int64, bugKey *db.Key, |
| t CrashReferenceType, key string) error { |
| crash := new(Crash) |
| crashKey := db.NewKey(c, "Crash", "", crashID, bugKey) |
| if err := db.Get(c, crashKey, crash); err != nil { |
| return fmt.Errorf("failed to get reported crash %v: %w", crashID, err) |
| } |
| crash.ClearReference(t, key) |
| if _, err := db.Put(c, crashKey, crash); err != nil { |
| return fmt.Errorf("failed to put reported crash %v: %w", crashID, err) |
| } |
| return nil |
| } |
| |
| func kernelRepoInfo(c context.Context, build *Build) KernelRepo { |
| return kernelRepoInfoRaw(c, build.Namespace, build.KernelRepo, build.KernelBranch) |
| } |
| |
| func kernelRepoInfoRaw(c context.Context, ns, url, branch string) KernelRepo { |
| var info KernelRepo |
| for _, repo := range getNsConfig(c, ns).Repos { |
| if repo.URL == url && repo.Branch == branch { |
| info = repo |
| break |
| } |
| } |
| if info.Alias == "" { |
| info.Alias = url |
| if branch != "" { |
| info.Alias += " " + branch |
| } |
| } |
| return info |
| } |
| |
| func textLink(tag string, id int64) string { |
| if id == 0 { |
| return "" |
| } |
| return fmt.Sprintf("/text?tag=%v&x=%v", tag, strconv.FormatUint(uint64(id), 16)) |
| } |
| |
| // timeDate returns t's date as a single int YYYYMMDD. |
| func timeDate(t time.Time) int { |
| year, month, day := t.Date() |
| return year*10000 + int(month)*100 + day |
| } |
| |
| func stringInList(list []string, str string) bool { |
| for _, s := range list { |
| if s == str { |
| return true |
| } |
| } |
| return false |
| } |
| |
| func stringListsIntersect(a, b []string) bool { |
| m := map[string]bool{} |
| for _, strA := range a { |
| m[strA] = true |
| } |
| for _, strB := range b { |
| if m[strB] { |
| return true |
| } |
| } |
| return false |
| } |
| |
| func mergeString(list []string, str string) []string { |
| if !stringInList(list, str) { |
| list = append(list, str) |
| } |
| return list |
| } |
| |
| func mergeStringList(list, add []string) []string { |
| for _, str := range add { |
| list = mergeString(list, str) |
| } |
| return list |
| } |
| |
| // dateTime converts date in YYYYMMDD format back to Time. |
| func dateTime(date int) time.Time { |
| return time.Date(date/10000, time.Month(date/100%100), date%100, 0, 0, 0, 0, time.UTC) |
| } |