Cap concurrency when parsing blueprint files
Darwin has a default limit of 256 open files per process. Parsing
too many blueprint files in parallel can hit the limit. Cap the
concurrency at 200.
Test: manual testing with limit set to 32
Change-Id: Ic64d21d2c0ffd7c86bf3f02fb51216ee5684a80c
diff --git a/context.go b/context.go
index 6793ac3..9dd68b4 100644
--- a/context.go
+++ b/context.go
@@ -719,10 +719,14 @@
// Number of outstanding goroutines to wait for
count := 0
- startParseBlueprintsFile := func(filename string, scope *parser.Scope) {
+ startParseBlueprintsFile := func(blueprint stringAndScope) {
+ if blueprintsSet[blueprint.string] {
+ return
+ }
+ blueprintsSet[blueprint.string] = true
count++
go func() {
- c.parseBlueprintsFile(filename, scope, rootDir,
+ c.parseBlueprintsFile(blueprint.string, blueprint.Scope, rootDir,
errsCh, fileCh, blueprintsCh, depsCh)
doneCh <- struct{}{}
}()
@@ -730,7 +734,9 @@
tooManyErrors := false
- startParseBlueprintsFile(rootFile, nil)
+ startParseBlueprintsFile(stringAndScope{rootFile, nil})
+
+ var pending []stringAndScope
loop:
for {
@@ -749,14 +755,19 @@
if tooManyErrors {
continue
}
- if blueprintsSet[blueprint.string] {
+ // Limit concurrent calls to parseBlueprintFiles to 200
+ // Darwin has a default limit of 256 open files
+ if count >= 200 {
+ pending = append(pending, blueprint)
continue
}
-
- blueprintsSet[blueprint.string] = true
- startParseBlueprintsFile(blueprint.string, blueprint.Scope)
+ startParseBlueprintsFile(blueprint)
case <-doneCh:
count--
+ if len(pending) > 0 {
+ startParseBlueprintsFile(pending[len(pending)-1])
+ pending = pending[:len(pending)-1]
+ }
if count == 0 {
break loop
}
