blob: 96a8b09738106f72ce69066e9c1a2d1716900063 [file] [log] [blame]
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bp2build
import (
"android/soong/android"
"android/soong/bazel"
"fmt"
"reflect"
"sort"
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
)
type BazelAttributes struct {
Attrs map[string]string
}
type BazelTarget struct {
name string
packageName string
content string
ruleClass string
bzlLoadLocation string
handcrafted bool
}
// IsLoadedFromStarlark determines if the BazelTarget's rule class is loaded from a .bzl file,
// as opposed to a native rule built into Bazel.
func (t BazelTarget) IsLoadedFromStarlark() bool {
return t.bzlLoadLocation != ""
}
// Label is the fully qualified Bazel label constructed from the BazelTarget's
// package name and target name.
func (t BazelTarget) Label() string {
if t.packageName == "." {
return "//:" + t.name
} else {
return "//" + t.packageName + ":" + t.name
}
}
// BazelTargets is a typedef for a slice of BazelTarget objects.
type BazelTargets []BazelTarget
// HasHandcraftedTargetsreturns true if a set of bazel targets contain
// handcrafted ones.
func (targets BazelTargets) hasHandcraftedTargets() bool {
for _, target := range targets {
if target.handcrafted {
return true
}
}
return false
}
// sort a list of BazelTargets in-place, by name, and by generated/handcrafted types.
func (targets BazelTargets) sort() {
sort.Slice(targets, func(i, j int) bool {
if targets[i].handcrafted != targets[j].handcrafted {
// Handcrafted targets will be generated after the bp2build generated targets.
return targets[j].handcrafted
}
// This will cover all bp2build generated targets.
return targets[i].name < targets[j].name
})
}
// String returns the string representation of BazelTargets, without load
// statements (use LoadStatements for that), since the targets are usually not
// adjacent to the load statements at the top of the BUILD file.
func (targets BazelTargets) String() string {
var res string
for i, target := range targets {
// There is only at most 1 handcrafted "target", because its contents
// represent the entire BUILD file content from the tree. See
// build_conversion.go#getHandcraftedBuildContent for more information.
//
// Add a header to make it easy to debug where the handcrafted targets
// are in a generated BUILD file.
if target.handcrafted {
res += "# -----------------------------\n"
res += "# Section: Handcrafted targets. \n"
res += "# -----------------------------\n\n"
}
res += target.content
if i != len(targets)-1 {
res += "\n\n"
}
}
return res
}
// LoadStatements return the string representation of the sorted and deduplicated
// Starlark rule load statements needed by a group of BazelTargets.
func (targets BazelTargets) LoadStatements() string {
bzlToLoadedSymbols := map[string][]string{}
for _, target := range targets {
if target.IsLoadedFromStarlark() {
bzlToLoadedSymbols[target.bzlLoadLocation] =
append(bzlToLoadedSymbols[target.bzlLoadLocation], target.ruleClass)
}
}
var loadStatements []string
for bzl, ruleClasses := range bzlToLoadedSymbols {
loadStatement := "load(\""
loadStatement += bzl
loadStatement += "\", "
ruleClasses = android.SortedUniqueStrings(ruleClasses)
for i, ruleClass := range ruleClasses {
loadStatement += "\"" + ruleClass + "\""
if i != len(ruleClasses)-1 {
loadStatement += ", "
}
}
loadStatement += ")"
loadStatements = append(loadStatements, loadStatement)
}
return strings.Join(android.SortedUniqueStrings(loadStatements), "\n")
}
type bpToBuildContext interface {
ModuleName(module blueprint.Module) string
ModuleDir(module blueprint.Module) string
ModuleSubDir(module blueprint.Module) string
ModuleType(module blueprint.Module) string
VisitAllModules(visit func(blueprint.Module))
VisitDirectDeps(module blueprint.Module, visit func(blueprint.Module))
}
type CodegenContext struct {
config android.Config
context android.Context
mode CodegenMode
additionalDeps []string
}
func (c *CodegenContext) Mode() CodegenMode {
return c.mode
}
// CodegenMode is an enum to differentiate code-generation modes.
type CodegenMode int
const (
// Bp2Build: generate BUILD files with targets buildable by Bazel directly.
//
// This mode is used for the Soong->Bazel build definition conversion.
Bp2Build CodegenMode = iota
// QueryView: generate BUILD files with targets representing fully mutated
// Soong modules, representing the fully configured Soong module graph with
// variants and dependency endges.
//
// This mode is used for discovering and introspecting the existing Soong
// module graph.
QueryView
)
func (mode CodegenMode) String() string {
switch mode {
case Bp2Build:
return "Bp2Build"
case QueryView:
return "QueryView"
default:
return fmt.Sprintf("%d", mode)
}
}
// AddNinjaFileDeps adds dependencies on the specified files to be added to the ninja manifest. The
// primary builder will be rerun whenever the specified files are modified. Allows us to fulfill the
// PathContext interface in order to add dependencies on hand-crafted BUILD files. Note: must also
// call AdditionalNinjaDeps and add them manually to the ninja file.
func (ctx *CodegenContext) AddNinjaFileDeps(deps ...string) {
ctx.additionalDeps = append(ctx.additionalDeps, deps...)
}
// AdditionalNinjaDeps returns additional ninja deps added by CodegenContext
func (ctx *CodegenContext) AdditionalNinjaDeps() []string {
return ctx.additionalDeps
}
func (ctx *CodegenContext) Config() android.Config { return ctx.config }
func (ctx *CodegenContext) Context() android.Context { return ctx.context }
// NewCodegenContext creates a wrapper context that conforms to PathContext for
// writing BUILD files in the output directory.
func NewCodegenContext(config android.Config, context android.Context, mode CodegenMode) *CodegenContext {
return &CodegenContext{
context: context,
config: config,
mode: mode,
}
}
// props is an unsorted map. This function ensures that
// the generated attributes are sorted to ensure determinism.
func propsToAttributes(props map[string]string) string {
var attributes string
for _, propName := range android.SortedStringKeys(props) {
if shouldGenerateAttribute(propName) {
attributes += fmt.Sprintf(" %s = %s,\n", propName, props[propName])
}
}
return attributes
}
func GenerateBazelTargets(ctx *CodegenContext, generateFilegroups bool) (map[string]BazelTargets, CodegenMetrics, CodegenCompatLayer) {
buildFileToTargets := make(map[string]BazelTargets)
buildFileToAppend := make(map[string]bool)
// Simple metrics tracking for bp2build
metrics := CodegenMetrics{
RuleClassCount: make(map[string]int),
}
compatLayer := CodegenCompatLayer{
NameToLabelMap: make(map[string]string),
}
dirs := make(map[string]bool)
bpCtx := ctx.Context()
bpCtx.VisitAllModules(func(m blueprint.Module) {
dir := bpCtx.ModuleDir(m)
dirs[dir] = true
var t BazelTarget
switch ctx.Mode() {
case Bp2Build:
if b, ok := m.(android.Bazelable); ok && b.HasHandcraftedLabel() {
metrics.handCraftedTargetCount += 1
metrics.TotalModuleCount += 1
compatLayer.AddNameToLabelEntry(m.Name(), b.HandcraftedLabel())
pathToBuildFile := getBazelPackagePath(b)
// We are using the entire contents of handcrafted build file, so if multiple targets within
// a package have handcrafted targets, we only want to include the contents one time.
if _, exists := buildFileToAppend[pathToBuildFile]; exists {
return
}
var err error
t, err = getHandcraftedBuildContent(ctx, b, pathToBuildFile)
if err != nil {
panic(fmt.Errorf("Error converting %s: %s", bpCtx.ModuleName(m), err))
}
// TODO(b/181575318): currently we append the whole BUILD file, let's change that to do
// something more targeted based on the rule type and target
buildFileToAppend[pathToBuildFile] = true
} else if btm, ok := m.(android.BazelTargetModule); ok {
t = generateBazelTarget(bpCtx, m, btm)
metrics.RuleClassCount[t.ruleClass] += 1
compatLayer.AddNameToLabelEntry(m.Name(), t.Label())
} else {
metrics.TotalModuleCount += 1
return
}
case QueryView:
// Blocklist certain module types from being generated.
if canonicalizeModuleType(bpCtx.ModuleType(m)) == "package" {
// package module name contain slashes, and thus cannot
// be mapped cleanly to a bazel label.
return
}
t = generateSoongModuleTarget(bpCtx, m)
default:
panic(fmt.Errorf("Unknown code-generation mode: %s", ctx.Mode()))
}
buildFileToTargets[dir] = append(buildFileToTargets[dir], t)
})
if generateFilegroups {
// Add a filegroup target that exposes all sources in the subtree of this package
// NOTE: This also means we generate a BUILD file for every Android.bp file (as long as it has at least one module)
for dir, _ := range dirs {
buildFileToTargets[dir] = append(buildFileToTargets[dir], BazelTarget{
name: "bp2build_all_srcs",
content: `filegroup(name = "bp2build_all_srcs", srcs = glob(["**/*"]))`,
ruleClass: "filegroup",
})
}
}
return buildFileToTargets, metrics, compatLayer
}
func getBazelPackagePath(b android.Bazelable) string {
label := b.HandcraftedLabel()
pathToBuildFile := strings.TrimPrefix(label, "//")
pathToBuildFile = strings.Split(pathToBuildFile, ":")[0]
return pathToBuildFile
}
func getHandcraftedBuildContent(ctx *CodegenContext, b android.Bazelable, pathToBuildFile string) (BazelTarget, error) {
p := android.ExistentPathForSource(ctx, pathToBuildFile, HandcraftedBuildFileName)
if !p.Valid() {
return BazelTarget{}, fmt.Errorf("Could not find file %q for handcrafted target.", pathToBuildFile)
}
c, err := b.GetBazelBuildFileContents(ctx.Config(), pathToBuildFile, HandcraftedBuildFileName)
if err != nil {
return BazelTarget{}, err
}
// TODO(b/181575318): once this is more targeted, we need to include name, rule class, etc
return BazelTarget{
content: c,
handcrafted: true,
}, nil
}
func generateBazelTarget(ctx bpToBuildContext, m blueprint.Module, btm android.BazelTargetModule) BazelTarget {
ruleClass := btm.RuleClass()
bzlLoadLocation := btm.BzlLoadLocation()
// extract the bazel attributes from the module.
props := getBuildProperties(ctx, m)
delete(props.Attrs, "bp2build_available")
// Return the Bazel target with rule class and attributes, ready to be
// code-generated.
attributes := propsToAttributes(props.Attrs)
targetName := targetNameForBp2Build(ctx, m)
return BazelTarget{
name: targetName,
packageName: ctx.ModuleDir(m),
ruleClass: ruleClass,
bzlLoadLocation: bzlLoadLocation,
content: fmt.Sprintf(
bazelTarget,
ruleClass,
targetName,
attributes,
),
handcrafted: false,
}
}
// Convert a module and its deps and props into a Bazel macro/rule
// representation in the BUILD file.
func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) BazelTarget {
props := getBuildProperties(ctx, m)
// TODO(b/163018919): DirectDeps can have duplicate (module, variant)
// items, if the modules are added using different DependencyTag. Figure
// out the implications of that.
depLabels := map[string]bool{}
if aModule, ok := m.(android.Module); ok {
ctx.VisitDirectDeps(aModule, func(depModule blueprint.Module) {
depLabels[qualifiedTargetLabel(ctx, depModule)] = true
})
}
attributes := propsToAttributes(props.Attrs)
depLabelList := "[\n"
for depLabel, _ := range depLabels {
depLabelList += fmt.Sprintf(" %q,\n", depLabel)
}
depLabelList += " ]"
targetName := targetNameWithVariant(ctx, m)
return BazelTarget{
name: targetName,
content: fmt.Sprintf(
soongModuleTarget,
targetName,
ctx.ModuleName(m),
canonicalizeModuleType(ctx.ModuleType(m)),
ctx.ModuleSubDir(m),
depLabelList,
attributes),
}
}
func getBuildProperties(ctx bpToBuildContext, m blueprint.Module) BazelAttributes {
var allProps map[string]string
// TODO: this omits properties for blueprint modules (blueprint_go_binary,
// bootstrap_go_binary, bootstrap_go_package), which will have to be handled separately.
if aModule, ok := m.(android.Module); ok {
allProps = ExtractModuleProperties(aModule)
}
return BazelAttributes{
Attrs: allProps,
}
}
// Generically extract module properties and types into a map, keyed by the module property name.
func ExtractModuleProperties(aModule android.Module) map[string]string {
ret := map[string]string{}
// Iterate over this android.Module's property structs.
for _, properties := range aModule.GetProperties() {
propertiesValue := reflect.ValueOf(properties)
// Check that propertiesValue is a pointer to the Properties struct, like
// *cc.BaseLinkerProperties or *java.CompilerProperties.
//
// propertiesValue can also be type-asserted to the structs to
// manipulate internal props, if needed.
if isStructPtr(propertiesValue.Type()) {
structValue := propertiesValue.Elem()
for k, v := range extractStructProperties(structValue, 0) {
ret[k] = v
}
} else {
panic(fmt.Errorf(
"properties must be a pointer to a struct, got %T",
propertiesValue.Interface()))
}
}
return ret
}
func isStructPtr(t reflect.Type) bool {
return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct
}
// prettyPrint a property value into the equivalent Starlark representation
// recursively.
func prettyPrint(propertyValue reflect.Value, indent int) (string, error) {
if isZero(propertyValue) {
// A property value being set or unset actually matters -- Soong does set default
// values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at
// https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480
//
// In Bazel-parlance, we would use "attr.<type>(default = <default
// value>)" to set the default value of unset attributes. In the cases
// where the bp2build converter didn't set the default value within the
// mutator when creating the BazelTargetModule, this would be a zero
// value. For those cases, we return an empty string so we don't
// unnecessarily generate empty values.
return "", nil
}
var ret string
switch propertyValue.Kind() {
case reflect.String:
ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String()))
case reflect.Bool:
ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface()))
case reflect.Int, reflect.Uint, reflect.Int64:
ret = fmt.Sprintf("%v", propertyValue.Interface())
case reflect.Ptr:
return prettyPrint(propertyValue.Elem(), indent)
case reflect.Slice:
if propertyValue.Len() == 0 {
return "[]", nil
}
if propertyValue.Len() == 1 {
// Single-line list for list with only 1 element
ret += "["
indexedValue, err := prettyPrint(propertyValue.Index(0), indent)
if err != nil {
return "", err
}
ret += indexedValue
ret += "]"
} else {
// otherwise, use a multiline list.
ret += "[\n"
for i := 0; i < propertyValue.Len(); i++ {
indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1)
if err != nil {
return "", err
}
if indexedValue != "" {
ret += makeIndent(indent + 1)
ret += indexedValue
ret += ",\n"
}
}
ret += makeIndent(indent)
ret += "]"
}
case reflect.Struct:
// Special cases where the bp2build sends additional information to the codegenerator
// by wrapping the attributes in a custom struct type.
if attr, ok := propertyValue.Interface().(bazel.Attribute); ok {
return prettyPrintAttribute(attr, indent)
} else if label, ok := propertyValue.Interface().(bazel.Label); ok {
return fmt.Sprintf("%q", label.Label), nil
}
ret = "{\n"
// Sort and print the struct props by the key.
structProps := extractStructProperties(propertyValue, indent)
if len(structProps) == 0 {
return "", nil
}
for _, k := range android.SortedStringKeys(structProps) {
ret += makeIndent(indent + 1)
ret += fmt.Sprintf("%q: %s,\n", k, structProps[k])
}
ret += makeIndent(indent)
ret += "}"
case reflect.Interface:
// TODO(b/164227191): implement pretty print for interfaces.
// Interfaces are used for for arch, multilib and target properties.
return "", nil
default:
return "", fmt.Errorf(
"unexpected kind for property struct field: %s", propertyValue.Kind())
}
return ret, nil
}
// Converts a reflected property struct value into a map of property names and property values,
// which each property value correctly pretty-printed and indented at the right nest level,
// since property structs can be nested. In Starlark, nested structs are represented as nested
// dicts: https://docs.bazel.build/skylark/lib/dict.html
func extractStructProperties(structValue reflect.Value, indent int) map[string]string {
if structValue.Kind() != reflect.Struct {
panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind()))
}
ret := map[string]string{}
structType := structValue.Type()
for i := 0; i < structValue.NumField(); i++ {
field := structType.Field(i)
if shouldSkipStructField(field) {
continue
}
fieldValue := structValue.Field(i)
if isZero(fieldValue) {
// Ignore zero-valued fields
continue
}
propertyName := proptools.PropertyNameForField(field.Name)
prettyPrintedValue, err := prettyPrint(fieldValue, indent+1)
if err != nil {
panic(
fmt.Errorf(
"Error while parsing property: %q. %s",
propertyName,
err))
}
if prettyPrintedValue != "" {
ret[propertyName] = prettyPrintedValue
}
}
return ret
}
func isZero(value reflect.Value) bool {
switch value.Kind() {
case reflect.Func, reflect.Map, reflect.Slice:
return value.IsNil()
case reflect.Array:
valueIsZero := true
for i := 0; i < value.Len(); i++ {
valueIsZero = valueIsZero && isZero(value.Index(i))
}
return valueIsZero
case reflect.Struct:
valueIsZero := true
for i := 0; i < value.NumField(); i++ {
valueIsZero = valueIsZero && isZero(value.Field(i))
}
return valueIsZero
case reflect.Ptr:
if !value.IsNil() {
return isZero(reflect.Indirect(value))
} else {
return true
}
// Always print bools, if you want a bool attribute to be able to take the default value, use a
// bool pointer instead
case reflect.Bool:
return false
default:
if !value.IsValid() {
return true
}
zeroValue := reflect.Zero(value.Type())
result := value.Interface() == zeroValue.Interface()
return result
}
}
func escapeString(s string) string {
s = strings.ReplaceAll(s, "\\", "\\\\")
// b/184026959: Reverse the application of some common control sequences.
// These must be generated literally in the BUILD file.
s = strings.ReplaceAll(s, "\t", "\\t")
s = strings.ReplaceAll(s, "\n", "\\n")
s = strings.ReplaceAll(s, "\r", "\\r")
return strings.ReplaceAll(s, "\"", "\\\"")
}
func makeIndent(indent int) string {
if indent < 0 {
panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent))
}
return strings.Repeat(" ", indent)
}
func targetNameForBp2Build(c bpToBuildContext, logicModule blueprint.Module) string {
return strings.Replace(c.ModuleName(logicModule), bazel.BazelTargetModuleNamePrefix, "", 1)
}
func targetNameWithVariant(c bpToBuildContext, logicModule blueprint.Module) string {
name := ""
if c.ModuleSubDir(logicModule) != "" {
// TODO(b/162720883): Figure out a way to drop the "--" variant suffixes.
name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule)
} else {
name = c.ModuleName(logicModule)
}
return strings.Replace(name, "//", "", 1)
}
func qualifiedTargetLabel(c bpToBuildContext, logicModule blueprint.Module) string {
return fmt.Sprintf("//%s:%s", c.ModuleDir(logicModule), targetNameWithVariant(c, logicModule))
}