Snap for 8512216 from c25454a0d5ee92d6d882ae9774063d4c7123cb5e to tm-frc-adbd-release
Change-Id: Iaa40aa1a816b22b01c3c3a243fb723a52df8e4d3
diff --git a/Android.bp b/Android.bp
index 74c1421..c84d04a 100644
--- a/Android.bp
+++ b/Android.bp
@@ -31,6 +31,7 @@
bootstrap_go_package {
name: "blueprint",
deps: [
+ "blueprint-metrics",
"blueprint-parser",
"blueprint-pathtools",
"blueprint-proptools",
diff --git a/bootstrap/command.go b/bootstrap/command.go
index e475709..8c045b4 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -81,7 +81,9 @@
} else {
fatalf("-l <moduleListFile> is required and must be nonempty")
}
+ ctx.BeginEvent("list_modules")
filesToParse, err := ctx.ListModulePaths(srcDir)
+ ctx.EndEvent("list_modules")
if err != nil {
fatalf("could not enumerate files: %v\n", err.Error())
}
@@ -91,10 +93,12 @@
ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory())
ctx.RegisterSingletonType("bootstrap", newSingletonFactory())
+ ctx.BeginEvent("parse_bp")
blueprintFiles, errs := ctx.ParseFileList(".", filesToParse, config)
if len(errs) > 0 {
fatalErrors(errs)
}
+ ctx.EndEvent("parse_bp")
// Add extra ninja file dependencies
ninjaDeps = append(ninjaDeps, blueprintFiles...)
@@ -124,6 +128,8 @@
var f *os.File
var buf *bufio.Writer
+ ctx.BeginEvent("write_files")
+ defer ctx.EndEvent("write_files")
if args.EmptyNinjaFile {
if err := ioutil.WriteFile(joinPath(ctx.SrcDir(), args.OutFile), []byte(nil), outFilePermissions); err != nil {
fatalf("error writing empty Ninja file: %s", err)
diff --git a/bpmodify/bpmodify.go b/bpmodify/bpmodify.go
index 29d28f0..431eb83 100644
--- a/bpmodify/bpmodify.go
+++ b/bpmodify/bpmodify.go
@@ -30,8 +30,9 @@
targetedProperty = new(qualifiedProperty)
addIdents = new(identSet)
removeIdents = new(identSet)
-
- setString *string
+ removeProperty = flag.Bool("remove-property", false, "remove the property")
+ setString *string
+ addLiteral *string
)
func init() {
@@ -39,6 +40,7 @@
flag.Var(targetedProperty, "parameter", "alias to -property=`name`")
flag.Var(targetedProperty, "property", "fully qualified `name` of property to modify (default \"deps\")")
flag.Var(addIdents, "a", "comma or whitespace separated list of identifiers to add")
+ flag.Var(stringPtrFlag{&addLiteral}, "add-literal", "a literal to add")
flag.Var(removeIdents, "r", "comma or whitespace separated list of identifiers to remove")
flag.Var(stringPtrFlag{&setString}, "str", "set a string property")
flag.Usage = usage
@@ -145,12 +147,12 @@
func processModule(module *parser.Module, moduleName string,
file *parser.File) (modified bool, errs []error) {
- prop, err := getRecursiveProperty(module, targetedProperty.name(), targetedProperty.prefixes())
+ prop, parent, err := getRecursiveProperty(module, targetedProperty.name(), targetedProperty.prefixes())
if err != nil {
return false, []error{err}
}
if prop == nil {
- if len(addIdents.idents) > 0 {
+ if len(addIdents.idents) > 0 || addLiteral != nil {
// We are adding something to a non-existing list prop, so we need to create it first.
prop, modified, err = createRecursiveProperty(module, targetedProperty.name(), targetedProperty.prefixes(), &parser.List{})
} else if setString != nil {
@@ -166,25 +168,28 @@
// Here should be unreachable, but still handle it for completeness.
return false, []error{err}
}
+ } else if *removeProperty {
+ // remove-property is used solely, so return here.
+ return parent.RemoveProperty(prop.Name), nil
}
m, errs := processParameter(prop.Value, targetedProperty.String(), moduleName, file)
modified = modified || m
return modified, errs
}
-func getRecursiveProperty(module *parser.Module, name string, prefixes []string) (prop *parser.Property, err error) {
- prop, _, err = getOrCreateRecursiveProperty(module, name, prefixes, nil)
- return prop, err
+func getRecursiveProperty(module *parser.Module, name string, prefixes []string) (prop *parser.Property, parent *parser.Map, err error) {
+ prop, parent, _, err = getOrCreateRecursiveProperty(module, name, prefixes, nil)
+ return prop, parent, err
}
func createRecursiveProperty(module *parser.Module, name string, prefixes []string,
empty parser.Expression) (prop *parser.Property, modified bool, err error) {
-
- return getOrCreateRecursiveProperty(module, name, prefixes, empty)
+ prop, _, modified, err = getOrCreateRecursiveProperty(module, name, prefixes, empty)
+ return prop, modified, err
}
func getOrCreateRecursiveProperty(module *parser.Module, name string, prefixes []string,
- empty parser.Expression) (prop *parser.Property, modified bool, err error) {
+ empty parser.Expression) (prop *parser.Property, parent *parser.Map, modified bool, err error) {
m := &module.Map
for i, prefix := range prefixes {
if prop, found := m.GetProperty(prefix); found {
@@ -193,7 +198,7 @@
} else {
// We've found a property in the AST and such property is not of type
// *parser.Map, which must mean we didn't modify the AST.
- return nil, false, fmt.Errorf("Expected property %q to be a map, found %s",
+ return nil, nil, false, fmt.Errorf("Expected property %q to be a map, found %s",
strings.Join(prefixes[:i+1], "."), prop.Value.Type())
}
} else if empty != nil {
@@ -204,18 +209,18 @@
// check after this for loop must fail, because the node we inserted is an
// empty parser.Map, thus this function will return |modified| is true.
} else {
- return nil, false, nil
+ return nil, nil, false, nil
}
}
if prop, found := m.GetProperty(name); found {
// We've found a property in the AST, which must mean we didn't modify the AST.
- return prop, false, nil
+ return prop, m, false, nil
} else if empty != nil {
prop = &parser.Property{Name: name, Value: empty}
m.Properties = append(m.Properties, prop)
- return prop, true, nil
+ return prop, m, true, nil
} else {
- return nil, false, nil
+ return nil, nil, false, nil
}
}
@@ -253,6 +258,21 @@
if (wasSorted || *sortLists) && modified {
parser.SortList(file, list)
}
+ } else if addLiteral != nil {
+ if *sortLists {
+ return false, []error{fmt.Errorf("sorting not supported when adding a literal")}
+ }
+ list, ok := value.(*parser.List)
+ if !ok {
+ return false, []error{fmt.Errorf("expected parameter %s in module %s to be list, found %s",
+ paramName, moduleName, value.Type().String())}
+ }
+ value, errs := parser.ParseExpression(strings.NewReader(*addLiteral))
+ if errs != nil {
+ return false, errs
+ }
+ list.Values = append(list.Values, value)
+ modified = true
} else if setString != nil {
str, ok := value.(*parser.String)
if !ok {
@@ -324,8 +344,13 @@
return
}
- if len(addIdents.idents) == 0 && len(removeIdents.idents) == 0 && setString == nil {
- report(fmt.Errorf("-a, -r or -str parameter is required"))
+ if len(addIdents.idents) == 0 && len(removeIdents.idents) == 0 && setString == nil && addLiteral == nil && !*removeProperty {
+ report(fmt.Errorf("-a, -add-literal, -r, -remove-property or -str parameter is required"))
+ return
+ }
+
+ if *removeProperty && (len(addIdents.idents) > 0 || len(removeIdents.idents) > 0 || setString != nil || addLiteral != nil) {
+ report(fmt.Errorf("-remove-property cannot be used with other parameter(s)"))
return
}
diff --git a/bpmodify/bpmodify_test.go b/bpmodify/bpmodify_test.go
index a92d439..4340edb 100644
--- a/bpmodify/bpmodify_test.go
+++ b/bpmodify/bpmodify_test.go
@@ -23,13 +23,15 @@
)
var testCases = []struct {
- name string
- input string
- output string
- property string
- addSet string
- removeSet string
- setString *string
+ name string
+ input string
+ output string
+ property string
+ addSet string
+ removeSet string
+ addLiteral *string
+ setString *string
+ removeProperty bool
}{
{
name: "add",
@@ -252,6 +254,25 @@
addSet: "bar-v10-bar",
},
{
+ name: "add a struct with literal",
+ input: `cc_foo {name: "foo"}`,
+ output: `cc_foo {
+ name: "foo",
+ structs: [
+ {
+ version: "1",
+ imports: [
+ "bar1",
+ "bar2",
+ ],
+ },
+ ],
+}
+`,
+ property: "structs",
+ addLiteral: proptools.StringPtr(`{version: "1", imports: ["bar1", "bar2"]}`),
+ },
+ {
name: "set string",
input: `
cc_foo {
@@ -284,6 +305,56 @@
property: "foo",
setString: proptools.StringPtr("bar"),
},
+ {
+ name: "remove existing property",
+ input: `
+ cc_foo {
+ name: "foo",
+ foo: "baz",
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ }
+ `,
+ property: "foo",
+ removeProperty: true,
+ }, {
+ name: "remove nested property",
+ input: `
+ cc_foo {
+ name: "foo",
+ foo: {
+ bar: "baz",
+ },
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ foo: {},
+ }
+ `,
+ property: "foo.bar",
+ removeProperty: true,
+ }, {
+ name: "remove non-existing property",
+ input: `
+ cc_foo {
+ name: "foo",
+ foo: "baz",
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ foo: "baz",
+ }
+ `,
+ property: "bar",
+ removeProperty: true,
+ },
}
func simplifyModuleDefinition(def string) string {
@@ -300,7 +371,9 @@
targetedProperty.Set(testCase.property)
addIdents.Set(testCase.addSet)
removeIdents.Set(testCase.removeSet)
+ removeProperty = &testCase.removeProperty
setString = testCase.setString
+ addLiteral = testCase.addLiteral
inAst, errs := parser.ParseAndEval("", strings.NewReader(testCase.input), parser.NewScope(nil))
if len(errs) > 0 {
diff --git a/context.go b/context.go
index e50df90..6496948 100644
--- a/context.go
+++ b/context.go
@@ -34,6 +34,7 @@
"text/scanner"
"text/template"
+ "github.com/google/blueprint/metrics"
"github.com/google/blueprint/parser"
"github.com/google/blueprint/pathtools"
"github.com/google/blueprint/proptools"
@@ -71,7 +72,9 @@
type Context struct {
context.Context
- // set at instantiation
+ // Used for metrics-related event logging.
+ EventHandler *metrics.EventHandler
+
moduleFactories map[string]ModuleFactory
nameInterface NameInterface
moduleGroups []*moduleGroup
@@ -80,7 +83,6 @@
preSingletonInfo []*singletonInfo
singletonInfo []*singletonInfo
mutatorInfo []*mutatorInfo
- earlyMutatorInfo []*mutatorInfo
variantMutatorNames []string
depsModified uint32 // positive if a mutator modified the dependencies
@@ -380,8 +382,10 @@
}
func newContext() *Context {
+ eventHandler := metrics.EventHandler{}
return &Context{
Context: context.Background(),
+ EventHandler: &eventHandler,
moduleFactories: make(map[string]ModuleFactory),
nameInterface: NewSimpleNameInterface(),
moduleInfo: make(map[Module]*moduleInfo),
@@ -625,38 +629,6 @@
return mutator
}
-// RegisterEarlyMutator registers a mutator that will be invoked to split
-// Modules into multiple variant Modules before any dependencies have been
-// created. Each registered mutator is invoked in registration order once
-// per Module (including each variant from previous early mutators). Module
-// order is unpredictable.
-//
-// In order for dependencies to be satisifed in a later pass, all dependencies
-// of a module either must have an identical variant or must have no variations.
-//
-// The mutator type names given here must be unique to all bottom up or early
-// mutators in the Context.
-//
-// Deprecated, use a BottomUpMutator instead. The only difference between
-// EarlyMutator and BottomUpMutator is that EarlyMutator runs before the
-// deprecated DynamicDependencies.
-func (c *Context) RegisterEarlyMutator(name string, mutator EarlyMutator) {
- for _, m := range c.variantMutatorNames {
- if m == name {
- panic(fmt.Errorf("mutator name %s is already registered", name))
- }
- }
-
- c.earlyMutatorInfo = append(c.earlyMutatorInfo, &mutatorInfo{
- bottomUpMutator: func(mctx BottomUpMutatorContext) {
- mutator(mctx)
- },
- name: name,
- })
-
- c.variantMutatorNames = append(c.variantMutatorNames, name)
-}
-
// SetIgnoreUnknownModuleTypes sets the behavior of the context in the case
// where it encounters an unknown module type while parsing Blueprints files. By
// default, the context will report unknown module types as an error. If this
@@ -1538,6 +1510,8 @@
// the modules depended upon are defined and that no circular dependencies
// exist.
func (c *Context) ResolveDependencies(config interface{}) (deps []string, errs []error) {
+ c.BeginEvent("resolve_deps")
+ defer c.EndEvent("resolve_deps")
return c.resolveDependencies(c.Context, config)
}
@@ -2261,7 +2235,7 @@
return
}
-type jsonVariationMap map[string]string
+type jsonVariationMap []Variation
type jsonModuleName struct {
Name string
@@ -2283,7 +2257,17 @@
}
func toJsonVariationMap(vm variationMap) jsonVariationMap {
- return jsonVariationMap(vm)
+ m := make(jsonVariationMap, 0, len(vm))
+ for k, v := range vm {
+ m = append(m, Variation{k, v})
+ }
+ sort.Slice(m, func(i, j int) bool {
+ if m[i].Mutator != m[j].Mutator {
+ return m[i].Mutator < m[j].Mutator
+ }
+ return m[i].Variation < m[j].Variation
+ })
+ return m
}
func jsonModuleNameFromModuleInfo(m *moduleInfo) *jsonModuleName {
@@ -2404,6 +2388,8 @@
// methods.
func (c *Context) PrepareBuildActions(config interface{}) (deps []string, errs []error) {
+ c.BeginEvent("prepare_build_actions")
+ defer c.EndEvent("prepare_build_actions")
pprof.Do(c.Context, pprof.Labels("blueprint", "PrepareBuildActions"), func(ctx context.Context) {
c.buildActionsReady = false
@@ -2464,13 +2450,8 @@
}
func (c *Context) runMutators(ctx context.Context, config interface{}) (deps []string, errs []error) {
- var mutators []*mutatorInfo
-
pprof.Do(ctx, pprof.Labels("blueprint", "runMutators"), func(ctx context.Context) {
- mutators = append(mutators, c.earlyMutatorInfo...)
- mutators = append(mutators, c.mutatorInfo...)
-
- for _, mutator := range mutators {
+ for _, mutator := range c.mutatorInfo {
pprof.Do(ctx, pprof.Labels("mutator", mutator.name), func(context.Context) {
var newDeps []string
if mutator.topDownMutator != nil {
@@ -4106,6 +4087,14 @@
return nil
}
+func (c *Context) BeginEvent(name string) {
+ c.EventHandler.Begin(name)
+}
+
+func (c *Context) EndEvent(name string) {
+ c.EventHandler.End(name)
+}
+
func (c *Context) writeLocalBuildActions(nw *ninjaWriter,
defs *localBuildActions) error {
diff --git a/metrics/Android.bp b/metrics/Android.bp
new file mode 100644
index 0000000..3668668
--- /dev/null
+++ b/metrics/Android.bp
@@ -0,0 +1,27 @@
+//
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package {
+ default_applicable_licenses: ["build_blueprint_license"],
+}
+
+bootstrap_go_package {
+ name: "blueprint-metrics",
+ pkgPath: "github.com/google/blueprint/metrics",
+ srcs: [
+ "event_handler.go",
+ ],
+}
diff --git a/metrics/event_handler.go b/metrics/event_handler.go
new file mode 100644
index 0000000..c19d039
--- /dev/null
+++ b/metrics/event_handler.go
@@ -0,0 +1,104 @@
+// Copyright 2022 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+ "fmt"
+ "strings"
+ "time"
+)
+
+// EventHandler tracks nested events and their start/stop times in a single
+// thread.
+type EventHandler struct {
+ completedEvents []Event
+
+ // These fields handle event scoping. When starting a new event, a new entry
+ // is pushed onto these fields. When ending an event, these fields are popped.
+ scopeIds []string
+ scopeStartTimes []time.Time
+}
+
+// _now wraps the time.Now() function. _now is declared for unit testing purpose.
+var _now = func() time.Time {
+ return time.Now()
+}
+
+// Event holds the performance metrics data of a single build event.
+type Event struct {
+ // A unique human-readable identifier / "name" for the build event. Event
+ // names use period-delimited scoping. For example, if an event alpha starts,
+ // then an event bravo starts, then an event charlie starts and ends, the
+ // unique identifier for charlie will be 'alpha.bravo.charlie'.
+ Id string
+
+ Start time.Time
+ end time.Time
+}
+
+// RuntimeNanoseconds returns the number of nanoseconds between the start
+// and end times of the event.
+func (e Event) RuntimeNanoseconds() uint64 {
+ return uint64(e.end.Sub(e.Start).Nanoseconds())
+}
+
+// Begin logs the start of an event. This must be followed by a corresponding
+// call to End (though other events may begin and end before this event ends).
+// Events within the same scope must have unique names.
+func (h *EventHandler) Begin(name string) {
+ h.scopeIds = append(h.scopeIds, name)
+ h.scopeStartTimes = append(h.scopeStartTimes, _now())
+}
+
+// End logs the end of an event. All events nested within this event must have
+// themselves been marked completed.
+func (h *EventHandler) End(name string) {
+ if len(h.scopeIds) == 0 || name != h.scopeIds[len(h.scopeIds)-1] {
+ panic(fmt.Errorf("Unexpected scope end '%s'. Current scope: (%s)",
+ name, h.scopeIds))
+ }
+ event := Event{
+ // The event Id is formed from the period-delimited scope names of all
+ // active events (e.g. `alpha.beta.charlie`). See Event.Id documentation
+ // for more detail.
+ Id: strings.Join(h.scopeIds, "."),
+ Start: h.scopeStartTimes[len(h.scopeStartTimes)-1],
+ end: _now(),
+ }
+ h.completedEvents = append(h.completedEvents, event)
+ h.scopeIds = h.scopeIds[:len(h.scopeIds)-1]
+ h.scopeStartTimes = h.scopeStartTimes[:len(h.scopeStartTimes)-1]
+}
+
+// CompletedEvents returns all events which have been completed, after
+// validation.
+// It is an error to call this method if there are still ongoing events, or
+// if two events were completed with the same scope and name.
+func (h *EventHandler) CompletedEvents() []Event {
+ if len(h.scopeIds) > 0 {
+ panic(fmt.Errorf(
+ "Retrieving events before all events have been closed. Current scope: (%s)",
+ h.scopeIds))
+ }
+ // Validate no two events have the same full id.
+ ids := map[string]bool{}
+ for _, event := range h.completedEvents {
+ if _, containsId := ids[event.Id]; containsId {
+ panic(fmt.Errorf("Duplicate event registered: %s", event.Id))
+ }
+ ids[event.Id] = true
+ }
+ return h.completedEvents
+}
diff --git a/module_ctx.go b/module_ctx.go
index 5d2b39b..53ee405 100644
--- a/module_ctx.go
+++ b/module_ctx.go
@@ -39,7 +39,7 @@
// modified as necessary by the Mutator.
//
// The Module implementation can access the build configuration as well as any
-// modules on which on which it depends (as defined by the "deps" property
+// modules on which it depends (as defined by the "deps" property
// specified in the Blueprints file, dynamically added by implementing the
// (deprecated) DynamicDependerModule interface, or dynamically added by a
// BottomUpMutator) using the ModuleContext passed to GenerateBuildActions.
@@ -831,32 +831,6 @@
MutatorName() string
}
-type EarlyMutatorContext interface {
- BaseMutatorContext
-
- // CreateVariations splits a module into multiple variants, one for each name in the variationNames
- // parameter. It returns a list of new modules in the same order as the variationNames
- // list.
- //
- // If any of the dependencies of the module being operated on were already split
- // by calling CreateVariations with the same name, the dependency will automatically
- // be updated to point the matching variant.
- //
- // If a module is split, and then a module depending on the first module is not split
- // when the Mutator is later called on it, the dependency of the depending module will
- // automatically be updated to point to the first variant.
- CreateVariations(...string) []Module
-
- // CreateLocalVariations splits a module into multiple variants, one for each name in the variantNames
- // parameter. It returns a list of new modules in the same order as the variantNames
- // list.
- //
- // Local variations do not affect automatic dependency resolution - dependencies added
- // to the split module via deps or DynamicDependerModule must exactly match a variant
- // that contains all the non-local variations.
- CreateLocalVariations(...string) []Module
-}
-
type TopDownMutatorContext interface {
BaseMutatorContext
@@ -995,7 +969,6 @@
// if a second Mutator chooses to split the module a second time.
type TopDownMutator func(mctx TopDownMutatorContext)
type BottomUpMutator func(mctx BottomUpMutatorContext)
-type EarlyMutator func(mctx EarlyMutatorContext)
// DependencyTag is an interface to an arbitrary object that embeds BaseDependencyTag. It can be
// used to transfer information on a dependency between the mutator that called AddDependency
@@ -1288,20 +1261,21 @@
// CreateModule creates a new module by calling the factory method for the specified moduleType, and applies
// the specified property structs to it as if the properties were set in a blueprint file.
- CreateModule(ModuleFactory, ...interface{}) Module
+ CreateModule(ModuleFactory, string, ...interface{}) Module
// RegisterScopedModuleType creates a new module type that is scoped to the current Blueprints
// file.
RegisterScopedModuleType(name string, factory ModuleFactory)
}
-func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module {
+func (l *loadHookContext) CreateModule(factory ModuleFactory, typeName string, props ...interface{}) Module {
module := newModule(factory)
module.relBlueprintsFile = l.module.relBlueprintsFile
module.pos = l.module.pos
module.propertyPos = l.module.propertyPos
module.createdBy = l.module
+ module.typeName = typeName
for _, p := range props {
err := proptools.AppendMatchingProperties(module.properties, p, nil)
diff --git a/parser/parser.go b/parser/parser.go
index 3e26b21..bb8817e 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -98,6 +98,14 @@
return parse(p)
}
+func ParseExpression(r io.Reader) (value Expression, errs []error) {
+ p := newParser(r, NewScope(nil))
+ value = p.parseExpression()
+ p.accept(scanner.EOF)
+ errs = p.errors
+ return
+}
+
type parser struct {
scanner scanner.Scanner
tok rune
diff --git a/parser/printer.go b/parser/printer.go
index ac7ffe1..f377505 100644
--- a/parser/printer.go
+++ b/parser/printer.go
@@ -139,7 +139,7 @@
func (p *printer) printList(list []Expression, pos, endPos scanner.Position) {
p.requestSpace()
p.printToken("[", pos)
- if len(list) > 1 || pos.Line != endPos.Line {
+ if len(list) > 1 || pos.Line != endPos.Line || listHasMap(list) {
p.requestNewline()
p.indent(p.curIndent() + 4)
for _, value := range list {
@@ -392,3 +392,12 @@
return b
}
}
+
+func listHasMap(list []Expression) bool {
+ for _, value := range list {
+ if _, ok := value.(*Map); ok {
+ return true
+ }
+ }
+ return false
+}
diff --git a/parser/printer_test.go b/parser/printer_test.go
index 077a782..c889b2a 100644
--- a/parser/printer_test.go
+++ b/parser/printer_test.go
@@ -428,6 +428,27 @@
}
`,
},
+ {
+ input: `
+// test
+stuff {
+ namespace: "google",
+ list_of_structs: [{ key1: "a", key2: "b" }],
+}
+`,
+ output: `
+// test
+stuff {
+ namespace: "google",
+ list_of_structs: [
+ {
+ key1: "a",
+ key2: "b",
+ },
+ ],
+}
+`,
+ },
}
func TestPrinter(t *testing.T) {
diff --git a/pathtools/fs.go b/pathtools/fs.go
index ed1251b..b959289 100644
--- a/pathtools/fs.go
+++ b/pathtools/fs.go
@@ -89,7 +89,7 @@
}
type FileSystem interface {
- // Open opens a file for reading. Follows symlinks.
+ // Open opens a file for reading. Follows symlinks.
Open(name string) (ReaderAtSeekerCloser, error)
// Exists returns whether the file exists and whether it is a directory. Follows symlinks.
@@ -124,11 +124,29 @@
// osFs implements FileSystem using the local disk.
type osFs struct {
- srcDir string
+ srcDir string
+ openFilesChan chan bool
}
func NewOsFs(path string) FileSystem {
- return &osFs{srcDir: path}
+ // Darwin has a default limit of 256 open files, rate limit open files to 200
+ limit := 200
+ return &osFs{
+ srcDir: path,
+ openFilesChan: make(chan bool, limit),
+ }
+}
+
+func (fs *osFs) acquire() {
+ if fs.openFilesChan != nil {
+ fs.openFilesChan <- true
+ }
+}
+
+func (fs *osFs) release() {
+ if fs.openFilesChan != nil {
+ <-fs.openFilesChan
+ }
}
func (fs *osFs) toAbs(path string) string {
@@ -163,11 +181,31 @@
return paths
}
+// OsFile wraps an os.File to also release open file descriptors semaphore on close
+type OsFile struct {
+ *os.File
+ fs *osFs
+}
+
+// Close closes file and releases the open file descriptor semaphore
+func (f *OsFile) Close() error {
+ err := f.File.Close()
+ f.fs.release()
+ return err
+}
+
func (fs *osFs) Open(name string) (ReaderAtSeekerCloser, error) {
- return os.Open(fs.toAbs(name))
+ fs.acquire()
+ f, err := os.Open(fs.toAbs(name))
+ if err != nil {
+ return nil, err
+ }
+ return &OsFile{f, fs}, nil
}
func (fs *osFs) Exists(name string) (bool, bool, error) {
+ fs.acquire()
+ defer fs.release()
stat, err := os.Stat(fs.toAbs(name))
if err == nil {
return true, stat.IsDir(), nil
@@ -179,6 +217,8 @@
}
func (fs *osFs) IsDir(name string) (bool, error) {
+ fs.acquire()
+ defer fs.release()
info, err := os.Stat(fs.toAbs(name))
if err != nil {
return false, err
@@ -187,6 +227,8 @@
}
func (fs *osFs) IsSymlink(name string) (bool, error) {
+ fs.acquire()
+ defer fs.release()
if info, err := os.Lstat(fs.toAbs(name)); err != nil {
return false, err
} else {
@@ -199,16 +241,22 @@
}
func (fs *osFs) glob(pattern string) ([]string, error) {
+ fs.acquire()
+ defer fs.release()
paths, err := filepath.Glob(fs.toAbs(pattern))
fs.removeSrcDirPrefixes(paths)
return paths, err
}
func (fs *osFs) Lstat(path string) (stats os.FileInfo, err error) {
+ fs.acquire()
+ defer fs.release()
return os.Lstat(fs.toAbs(path))
}
func (fs *osFs) Stat(path string) (stats os.FileInfo, err error) {
+ fs.acquire()
+ defer fs.release()
return os.Stat(fs.toAbs(path))
}
@@ -218,6 +266,8 @@
}
func (fs *osFs) ReadDirNames(name string) ([]string, error) {
+ fs.acquire()
+ defer fs.release()
dir, err := os.Open(fs.toAbs(name))
if err != nil {
return nil, err
@@ -234,6 +284,8 @@
}
func (fs *osFs) Readlink(name string) (string, error) {
+ fs.acquire()
+ defer fs.release()
return os.Readlink(fs.toAbs(name))
}