datastore/app: switch more bug queries to paged queries
Update #1448
diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index b488bb5..1baf0af 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -177,13 +177,12 @@
if err := json.Unmarshal(payload, req); err != nil {
return nil, fmt.Errorf("failed to unmarshal request: %v", err)
}
- var bugs []*Bug
- _, err := db.NewQuery("Bug").
- Filter("Namespace=", ns).
- Filter("Status<", BugStatusFixed).
- GetAll(c, &bugs)
+ bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+ return query.Filter("Namespace=", ns).
+ Filter("Status<", BugStatusFixed)
+ })
if err != nil {
- return nil, fmt.Errorf("failed to query bugs: %v", err)
+ return nil, err
}
m := make(map[string]bool)
loop:
@@ -521,13 +520,12 @@
func addCommitsToBugsInStatus(c context.Context, status int, ns, manager string, managers []string,
presentCommits map[string]bool, bugFixedBy map[string][]string) error {
- var bugs []*Bug
- _, err := db.NewQuery("Bug").
- Filter("Namespace=", ns).
- Filter("Status=", status).
- GetAll(c, &bugs)
+ bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+ return query.Filter("Namespace=", ns).
+ Filter("Status=", status)
+ })
if err != nil {
- return fmt.Errorf("failed to query bugs: %v", err)
+ return err
}
for _, bug := range bugs {
var fixCommits []string
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index d43f9d1..64fb6ff 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -527,7 +527,14 @@
func fetchNamespaceBugs(c context.Context, accessLevel AccessLevel,
ns, manager string) ([]*uiBugGroup, int, error) {
- bugs, err := loadAllBugs(c, ns, manager)
+ filter := func(query *db.Query) *db.Query {
+ query = query.Filter("Namespace=", ns)
+ if manager != "" {
+ query = query.Filter("HappenedOn=", manager)
+ }
+ return query
+ }
+ bugs, err := loadAllBugs(c, filter)
if err != nil {
return nil, 0, err
}
@@ -616,14 +623,15 @@
func fetchTerminalBugs(c context.Context, accessLevel AccessLevel,
ns, manager string, typ *TerminalBug) (*uiBugGroup, error) {
- var bugs []*Bug
- query := db.NewQuery("Bug").
- Filter("Namespace=", ns).
- Filter("Status=", typ.Status)
- if manager != "" {
- query = query.Filter("HappenedOn=", manager)
- }
- if _, err := query.GetAll(c, &bugs); err != nil {
+ bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+ query = query.Filter("Namespace=", ns).
+ Filter("Status=", typ.Status)
+ if manager != "" {
+ query = query.Filter("HappenedOn=", manager)
+ }
+ return query
+ })
+ if err != nil {
return nil, err
}
state, err := loadReportingState(c)
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 43067fe..417f3a9 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -44,10 +44,9 @@
log.Errorf(c, "%v", err)
return nil
}
- var bugs []*Bug
- _, err = db.NewQuery("Bug").
- Filter("Status<", BugStatusFixed).
- GetAll(c, &bugs)
+ bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+ return query.Filter("Status<", BugStatusFixed)
+ })
if err != nil {
log.Errorf(c, "%v", err)
return nil
@@ -152,10 +151,9 @@
}
func reportingPollNotifications(c context.Context, typ string) []*dashapi.BugNotification {
- var bugs []*Bug
- _, err := db.NewQuery("Bug").
- Filter("Status<", BugStatusFixed).
- GetAll(c, &bugs)
+ bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+ return query.Filter("Status<", BugStatusFixed)
+ })
if err != nil {
log.Errorf(c, "%v", err)
return nil
@@ -471,9 +469,9 @@
return repos
}
-func loadAllBugs(c context.Context, ns, manager string) ([]*Bug, error) {
+func loadAllBugs(c context.Context, filter func(*db.Query) *db.Query) ([]*Bug, error) {
var bugs []*Bug
- err := foreachBug(c, ns, manager, func(bug *Bug) error {
+ err := foreachBug(c, filter, func(bug *Bug, _ *db.Key) error {
bugs = append(bugs, bug)
return nil
})
@@ -483,41 +481,39 @@
return bugs, nil
}
-func foreachBug(c context.Context, ns, manager string, fn func(bug *Bug) error) error {
+func foreachBug(c context.Context, filter func(*db.Query) *db.Query, fn func(bug *Bug, key *db.Key) error) error {
const batchSize = 1000
- var cursor db.Cursor
- for first := true; ; first = false {
+ var cursor *db.Cursor
+ for {
query := db.NewQuery("Bug").Limit(batchSize)
- if ns != "" {
- query = query.Filter("Namespace=", ns)
- if manager != "" {
- query = query.Filter("HappenedOn=", manager)
- }
+ if filter != nil {
+ query = filter(query)
}
- if !first {
- query = query.Start(cursor)
+ if cursor != nil {
+ query = query.Start(*cursor)
}
iter := query.Run(c)
for i := 0; ; i++ {
bug := new(Bug)
- _, err := iter.Next(bug)
+ key, err := iter.Next(bug)
if err == db.Done {
if i < batchSize {
return nil
}
- cursor, err = iter.Cursor()
- if err != nil {
- return fmt.Errorf("cursor failed while fetching bugs: %v", err)
- }
break
}
if err != nil {
return fmt.Errorf("failed to fetch bugs: %v", err)
}
- if err := fn(bug); err != nil {
+ if err := fn(bug, key); err != nil {
return err
}
}
+ cur, err := iter.Cursor()
+ if err != nil {
+ return fmt.Errorf("cursor failed while fetching bugs: %v", err)
+ }
+ cursor = &cur
}
}
@@ -528,7 +524,7 @@
idMap[id] = true
}
var closed []string
- err := foreachBug(c, "", "", func(bug *Bug) error {
+ err := foreachBug(c, nil, func(bug *Bug, _ *db.Key) error {
for i := range bug.Reporting {
bugReporting := &bug.Reporting[i]
if !idMap[bugReporting.ID] {
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 3b3df67..b88953b 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -111,7 +111,7 @@
reports := reportingPollBugs(c, emailType)
for _, rep := range reports {
if err := emailSendBugReport(c, rep); err != nil {
- log.Errorf(c, "%v", err)
+ log.Errorf(c, "emailPollBugs: %v", err)
}
}
return nil
@@ -147,7 +147,7 @@
notifs := reportingPollNotifications(c, emailType)
for _, notif := range notifs {
if err := emailSendBugNotif(c, notif); err != nil {
- log.Errorf(c, "%v", err)
+ log.Errorf(c, "emailPollNotifications: %v", err)
}
}
return nil
@@ -256,7 +256,7 @@
func handleIncomingMail(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
if err := incomingMail(c, r); err != nil {
- log.Errorf(c, "%v", err)
+ log.Errorf(c, "handleIncomingMail: %v", err)
}
}
