Generate GenericConfig objects from MakeConfig objects.
Applies heuristics to the sequence of Blocks to do so.
Test: rm -rf out/config/ && m product-config-test product-config && java -jar out/host/linux-x86/testcases/product-config-test/product-config-test.jar && time ( out/host/linux-x86/bin/product-config --ckati_bin /source/kati/ckati > ~/Desktop/out.txt )
Change-Id: Id6763781bc876e2b2e0be320a7259c1ed41c2334
diff --git a/tools/product_config/src/com/android/build/config/ConfigBase.java b/tools/product_config/src/com/android/build/config/ConfigBase.java
index 5ac1fc2..0c67d16 100644
--- a/tools/product_config/src/com/android/build/config/ConfigBase.java
+++ b/tools/product_config/src/com/android/build/config/ConfigBase.java
@@ -63,6 +63,10 @@
mProductVars.put(name, type);
}
+ public TreeMap<String, VarType> getProductVars() {
+ return mProductVars;
+ }
+
public VarType getVarType(String name) {
final VarType t = mProductVars.get(name);
if (t != null) {
@@ -75,4 +79,15 @@
public boolean isProductVar(String name) {
return mProductVars.get(name) != null;
}
+
+ /**
+ * Copy common base class fields from that to this.
+ */
+ public void copyFrom(ConfigBase that) {
+ setPhase(that.getPhase());
+ setRootNodes(that.getRootNodes());
+ for (Map.Entry<String, ConfigBase.VarType> entry: that.getProductVars().entrySet()) {
+ addProductVar(entry.getKey(), entry.getValue());
+ }
+ }
}
diff --git a/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java b/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java
new file mode 100644
index 0000000..369d4d6
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/ConvertMakeToGenericConfig.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Converts a MakeConfig into a Generic config by applying heuristics about
+ * the types of variable assignments that we do.
+ */
+public class ConvertMakeToGenericConfig {
+ private final Errors mErrors;
+
+ public ConvertMakeToGenericConfig(Errors errors) {
+ mErrors = errors;
+ }
+
+ public GenericConfig convert(MakeConfig make) {
+ final GenericConfig result = new GenericConfig();
+
+ // Base class fields
+ result.copyFrom(make);
+
+ // Each file
+ for (MakeConfig.ConfigFile f: make.getConfigFiles()) {
+ final GenericConfig.ConfigFile genericFile
+ = new GenericConfig.ConfigFile(f.getFilename());
+ result.addConfigFile(genericFile);
+
+ final List<MakeConfig.Block> blocks = f.getBlocks();
+
+ // Some assertions:
+ // TODO: Include better context for these errors.
+ // There should always be at least a BEGIN and an AFTER, so assert this.
+ if (blocks.size() < 2) {
+ throw new RuntimeException("expected at least blocks.size() >= 2. Actcual size: "
+ + blocks.size());
+ }
+ if (blocks.get(0).getBlockType() != MakeConfig.BlockType.BEFORE) {
+ throw new RuntimeException("expected first block to be BEFORE");
+ }
+ if (blocks.get(blocks.size() - 1).getBlockType() != MakeConfig.BlockType.AFTER) {
+ throw new RuntimeException("expected first block to be AFTER");
+ }
+ // Everything in between should be an INHERIT block.
+ for (int index = 1; index < blocks.size() - 1; index++) {
+ if (blocks.get(index).getBlockType() != MakeConfig.BlockType.INHERIT) {
+ throw new RuntimeException("expected INHERIT at block " + index);
+ }
+ }
+
+ // Each block represents a snapshot of the interpreter variable state (minus a few big
+ // sets of variables which we don't export because they're used in the internals
+ // of node_fns.mk, so we know they're not necessary here). The first (BEFORE) one
+ // is everything that is set before the file is included, so it forms the base
+ // for everything else.
+ MakeConfig.Block prevBlock = blocks.get(0);
+
+ for (int index = 1; index < blocks.size(); index++) {
+ final MakeConfig.Block block = blocks.get(index);
+ for (final Map.Entry<String, Str> entry: block.getVars().entrySet()) {
+ final String varName = entry.getKey();
+ final GenericConfig.Assign assign = convertAssignment(block.getBlockType(),
+ block.getInheritedFile(), make.getVarType(varName), varName,
+ entry.getValue(), prevBlock.getVar(varName));
+ if (assign != null) {
+ genericFile.addStatement(assign);
+ }
+ }
+ // Handle variables that are in prevBlock but not block -- they were
+ // deleted. Is this even possible, or do they show up as ""? We will
+ // treat them as positive assigments to empty string
+ for (String prevName: prevBlock.getVars().keySet()) {
+ if (!block.getVars().containsKey(prevName)) {
+ genericFile.addStatement(
+ new GenericConfig.Assign(prevName, new Str("")));
+ }
+ }
+ if (block.getBlockType() == MakeConfig.BlockType.INHERIT) {
+ genericFile.addStatement(
+ new GenericConfig.Inherit(block.getInheritedFile()));
+ }
+ // For next iteration
+ prevBlock = block;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Converts one variable from a MakeConfig Block into a GenericConfig Assignment.
+ */
+ GenericConfig.Assign convertAssignment(MakeConfig.BlockType blockType, Str inheritedFile,
+ ConfigBase.VarType varType, String varName, Str varVal, Str prevVal) {
+ if (prevVal == null) {
+ // New variable.
+ return new GenericConfig.Assign(varName, varVal);
+ } else if (!varVal.equals(prevVal)) {
+ // The value changed from the last block.
+ if (varVal.equals("")) {
+ // It was set to empty
+ return new GenericConfig.Assign(varName, varVal);
+ } else {
+ // Product vars have the @inherit processing. Other vars we
+ // will just ignore and put in one section at the end, based
+ // on the difference between the BEFORE and AFTER blocks.
+ if (varType == ConfigBase.VarType.UNKNOWN) {
+ if (blockType == MakeConfig.BlockType.AFTER) {
+ // For UNKNOWN variables, we don't worry about the
+ // intermediate steps, just take the final value.
+ return new GenericConfig.Assign(varName, varVal);
+ } else {
+ return null;
+ }
+ } else {
+ return convertInheritedVar(blockType, inheritedFile,
+ varName, varVal, prevVal);
+ }
+ }
+ } else {
+ // Variable not touched
+ return null;
+ }
+ }
+
+ /**
+ * Handle the special inherited values, where the inherit-product puts in the
+ * @inherit:... markers, adding Statements to the ConfigFile.
+ */
+ GenericConfig.Assign convertInheritedVar(MakeConfig.BlockType blockType, Str inheritedFile,
+ String varName, Str varVal, Str prevVal) {
+ String varText = varVal.toString();
+ String prevText = prevVal.toString().trim();
+ if (blockType == MakeConfig.BlockType.INHERIT) {
+ // inherit-product appends @inherit:... so drop that.
+ final String marker = "@inherit:" + inheritedFile;
+ if (varText.endsWith(marker)) {
+ varText = varText.substring(0, varText.length() - marker.length()).trim();
+ } else {
+ mErrors.ERROR_IMPROPER_PRODUCT_VAR_MARKER.add(varVal.getPosition(),
+ "Variable didn't end with marker \"" + marker + "\": " + varText);
+ }
+ }
+
+ if (!varText.equals(prevText)) {
+ // If the variable value was actually changed.
+ final ArrayList<String> words = split(varText, prevText);
+ if (words.size() == 0) {
+ // Pure Assignment, none of the previous value is present.
+ return new GenericConfig.Assign(varName, new Str(varVal.getPosition(), varText));
+ } else {
+ // Self referential value (prepend, append, both).
+ if (words.size() > 2) {
+ // This is indicative of a construction that might not be quite
+ // what we want. The above code will do something that works if it was
+ // of the form "VAR := a $(VAR) b $(VAR) c", but if the original code
+ // something else this won't work. This doesn't happen in AOSP, but
+ // it's a theoretically possibility, so someone might do it.
+ mErrors.WARNING_VARIABLE_RECURSION.add(varVal.getPosition(),
+ "Possible unsupported variable recursion: "
+ + varName + " = " + varVal + " (prev=" + prevVal + ")");
+ }
+ return new GenericConfig.Assign(varName, Str.toList(varVal.getPosition(), words));
+ }
+ } else {
+ // Variable not touched
+ return null;
+ }
+ }
+
+ /**
+ * Split 'haystack' on occurrences of 'needle'. Trims each string of whitespace
+ * to preserve make list semantics.
+ */
+ private static ArrayList<String> split(String haystack, String needle) {
+ final ArrayList<String> result = new ArrayList();
+ final int needleLen = needle.length();
+ if (needleLen == 0) {
+ return result;
+ }
+ int start = 0;
+ int end;
+ while ((end = haystack.indexOf(needle, start)) >= 0) {
+ result.add(haystack.substring(start, end).trim());
+ start = end + needleLen;
+ }
+ result.add(haystack.substring(start).trim());
+ return result;
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/DumpConfigParser.java b/tools/product_config/src/com/android/build/config/DumpConfigParser.java
index b954f32..94bf205 100644
--- a/tools/product_config/src/com/android/build/config/DumpConfigParser.java
+++ b/tools/product_config/src/com/android/build/config/DumpConfigParser.java
@@ -1,4 +1,3 @@
-
/*
* Copyright (C) 2020 The Android Open Source Project
*
@@ -183,7 +182,8 @@
// There is already a file in progress, so add another var block to that.
block = new MakeConfig.Block(MakeConfig.BlockType.INHERIT);
- block.setInheritedFile(inheritedFile);
+ // TODO: Make dumpconfig.mk also output a Position for inherit-product
+ block.setInheritedFile(new Str(inheritedFile));
configFile.addBlock(block);
if (DEBUG) {
@@ -240,8 +240,8 @@
+ " Saw: " + blockType);
}
- // Add the value to the block in progress
- block.addValue(varName, new Str(pos, varValue));
+ // Add the variable to the block in progress
+ block.addVar(varName, new Str(pos, varValue));
} else {
if (DEBUG) {
System.out.print("# ");
diff --git a/tools/product_config/src/com/android/build/config/Errors.java b/tools/product_config/src/com/android/build/config/Errors.java
index 9290b72..92a4b30 100644
--- a/tools/product_config/src/com/android/build/config/Errors.java
+++ b/tools/product_config/src/com/android/build/config/Errors.java
@@ -52,4 +52,11 @@
public final Category ERROR_DUMPCONFIG = new Category(5, false, Level.ERROR,
"Error parsing the output of kati and dumpconfig.mk.");
+ public final Category WARNING_VARIABLE_RECURSION = new Category(6, true, Level.WARNING,
+ "Possible unsupported variable recursion.");
+
+ // This could be a warning, but it's very likely that the data is corrupted somehow
+ // if we're seeing this.
+ public final Category ERROR_IMPROPER_PRODUCT_VAR_MARKER = new Category(7, true, Level.ERROR,
+ "Bad input from dumpvars causing corrupted product variables.");
}
diff --git a/tools/product_config/src/com/android/build/config/GenericConfig.java b/tools/product_config/src/com/android/build/config/GenericConfig.java
new file mode 100644
index 0000000..2ee2735
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/GenericConfig.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Language-agnostic representation of a configuration statement.
+ */
+public class GenericConfig extends ConfigBase {
+ /**
+ * The config files that were imported in this config pass.
+ */
+ protected final TreeMap<String, ConfigFile> mConfigFiles = new TreeMap();
+
+ /**
+ * A configuration file.
+ */
+ public static class ConfigFile {
+ /**
+ * The name of the file, relative to the tree root.
+ */
+ private final String mFilename;
+
+ /**
+ * Sections of variable definitions and import statements. Product config
+ * files will always have at least one block.
+ */
+ private final ArrayList<Statement> mStatements = new ArrayList();
+
+ public ConfigFile(String filename) {
+ mFilename = filename;
+ }
+
+ public String getFilename() {
+ return mFilename;
+ }
+
+ public void addStatement(Statement statement) {
+ mStatements.add(statement);
+ }
+
+ public ArrayList<Statement> getStatements() {
+ return mStatements;
+ }
+ }
+
+ /**
+ * Base class for statements that appear in config files.
+ */
+ public static class Statement {
+ }
+
+ /**
+ * A variable assignment.
+ */
+ public static class Assign extends Statement {
+ private final String mVarName;
+ private final List<Str> mValue;
+
+ /**
+ * Assignment of a single value
+ */
+ public Assign(String varName, Str value) {
+ mVarName = varName;
+ mValue = new ArrayList();
+ mValue.add(value);
+ }
+
+ /**
+ * Assignment referencing a previous value.
+ * VAR := $(1) $(VAR) $(2) $(VAR) $(3)
+ */
+ public Assign(String varName, List<Str> value) {
+ mVarName = varName;
+ mValue = value;
+ }
+
+ public String getName() {
+ return mVarName;
+ }
+
+ public List<Str> getValue() {
+ return mValue;
+ }
+ }
+
+ /**
+ * An $(inherit-product FILENAME) statement
+ */
+ public static class Inherit extends Statement {
+ private final Str mFilename;
+
+ public Inherit(Str filename) {
+ mFilename = filename;
+ }
+
+ public Str getFilename() {
+ return mFilename;
+ }
+ }
+
+ /**
+ * Adds the given config file. Returns any one previously added, or null.
+ */
+ public ConfigFile addConfigFile(ConfigFile file) {
+ return mConfigFiles.put(file.getFilename(), file);
+ }
+
+ public TreeMap<String, ConfigFile> getFiles() {
+ return mConfigFiles;
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/Main.java b/tools/product_config/src/com/android/build/config/Main.java
index 81d9e7b..7417fc7 100644
--- a/tools/product_config/src/com/android/build/config/Main.java
+++ b/tools/product_config/src/com/android/build/config/Main.java
@@ -18,6 +18,7 @@
import java.util.List;
import java.util.Map;
+import java.util.TreeSet;
public class Main {
private final Errors mErrors;
@@ -46,6 +47,14 @@
System.out.println("====================");
makeConfig.printToStream(System.out);
+ ConvertMakeToGenericConfig m2g = new ConvertMakeToGenericConfig(mErrors);
+ GenericConfig generic = m2g.convert(makeConfig);
+
+ System.out.println("======================");
+ System.out.println("REGENERATED MAKE FILES");
+ System.out.println("======================");
+ MakeWriter.write(System.out, generic, 0);
+
// TODO: Run kati and extract the variables and convert all that into starlark files.
// TODO: Run starlark with all the generated ones and the hand written ones.
diff --git a/tools/product_config/src/com/android/build/config/MakeConfig.java b/tools/product_config/src/com/android/build/config/MakeConfig.java
index 300b655..dda0db9 100644
--- a/tools/product_config/src/com/android/build/config/MakeConfig.java
+++ b/tools/product_config/src/com/android/build/config/MakeConfig.java
@@ -70,7 +70,7 @@
public static class Block {
private final BlockType mBlockType;
private final TreeMap<String, Str> mValues = new TreeMap();
- private String mInheritedFile;
+ private Str mInheritedFile;
public Block(BlockType blockType) {
mBlockType = blockType;
@@ -80,19 +80,23 @@
return mBlockType;
}
- public void addValue(String varName, Str varValue) {
+ public void addVar(String varName, Str varValue) {
mValues.put(varName, varValue);
}
- public TreeMap<String, Str> getValues() {
+ public Str getVar(String varName) {
+ return mValues.get(varName);
+ }
+
+ public TreeMap<String, Str> getVars() {
return mValues;
}
- public void setInheritedFile(String filename) {
+ public void setInheritedFile(Str filename) {
mInheritedFile = filename;
}
- public String getInheritedFile() {
+ public Str getInheritedFile() {
return mInheritedFile;
}
}
@@ -148,7 +152,7 @@
out.println(" inherited: " + block.getInheritedFile());
}
out.println(" values: {");
- for (Map.Entry<String,Str> var: block.getValues().entrySet()) {
+ for (Map.Entry<String,Str> var: block.getVars().entrySet()) {
if (!var.getKey().equals("PRODUCT_PACKAGES")) {
continue;
}
diff --git a/tools/product_config/src/com/android/build/config/MakeWriter.java b/tools/product_config/src/com/android/build/config/MakeWriter.java
new file mode 100644
index 0000000..8c79c46
--- /dev/null
+++ b/tools/product_config/src/com/android/build/config/MakeWriter.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.config;
+
+import java.io.PrintStream;
+import java.util.List;
+
+public class MakeWriter {
+ public static final int FLAG_WRITE_HEADER = 1;
+ public static final int FLAG_WRITE_ANNOTATIONS = 1 << 1;
+
+ private final boolean mWriteHeader;
+ private final boolean mWriteAnnotations;
+
+ public static void write(PrintStream out, GenericConfig config, int flags) {
+ (new MakeWriter(flags)).write(out, config);
+ }
+
+ private MakeWriter(int flags) {
+ mWriteHeader = (flags & FLAG_WRITE_HEADER) != 0;
+ mWriteAnnotations = (flags & FLAG_WRITE_ANNOTATIONS) != 0;
+ }
+
+ private void write(PrintStream out, GenericConfig config) {
+ for (GenericConfig.ConfigFile file: config.getFiles().values()) {
+ out.println("---------------------------------------------------------");
+ out.println("FILE: " + file.getFilename());
+ out.println("---------------------------------------------------------");
+ writeFile(out, config, file);
+ out.println();
+ }
+ }
+
+ private void writeFile(PrintStream out, GenericConfig config, GenericConfig.ConfigFile file) {
+ if (mWriteHeader) {
+ out.println("# This file is generated by the product_config tool");
+ }
+ for (GenericConfig.Statement statement: file.getStatements()) {
+ if (statement instanceof GenericConfig.Assign) {
+ writeAssign(out, config, (GenericConfig.Assign)statement);
+ } else if (statement instanceof GenericConfig.Inherit) {
+ writeInherit(out, (GenericConfig.Inherit)statement);
+ } else {
+ throw new RuntimeException("Unexpected Statement: " + statement);
+ }
+ }
+ }
+
+ private void writeAssign(PrintStream out, GenericConfig config,
+ GenericConfig.Assign statement) {
+ final List<Str> values = statement.getValue();
+ final int size = values.size();
+ final String varName = statement.getName();
+ Position pos = null;
+ if (size == 0) {
+ return;
+ } else if (size == 1) {
+ // Plain :=
+ final Str value = values.get(0);
+ out.print(varName + " := " + value);
+ pos = value.getPosition();
+ } else if (size == 2 && values.get(0).toString().length() == 0) {
+ // Plain +=
+ final Str value = values.get(1);
+ out.print(varName + " += " + value);
+ pos = value.getPosition();
+ } else {
+ // Write it out the long way
+ out.print(varName + " := " + values.get(0));
+ for (int i = 1; i < size; i++) {
+ out.print("$(" + varName + ") " + values.get(i));
+ pos = values.get(i).getPosition();
+ }
+ }
+ if (mWriteAnnotations) {
+ out.print(" # " + config.getVarType(varName) + " " + pos);
+ }
+ out.println();
+ }
+
+ private void writeInherit(PrintStream out, GenericConfig.Inherit statement) {
+ final Str filename = statement.getFilename();
+ out.print("$(call inherit-product " + filename + ")");
+ if (mWriteAnnotations) {
+ out.print(" # " + filename.getPosition());
+ }
+ out.println();
+ }
+}
diff --git a/tools/product_config/src/com/android/build/config/Str.java b/tools/product_config/src/com/android/build/config/Str.java
index 7dbe2e5..9c345a6 100644
--- a/tools/product_config/src/com/android/build/config/Str.java
+++ b/tools/product_config/src/com/android/build/config/Str.java
@@ -16,6 +16,9 @@
package com.android.build.config;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* A String and a Position, where it came from in source code.
*/
@@ -64,4 +67,12 @@
public int hashCode() {
return mValue.hashCode();
}
+
+ public static ArrayList<Str> toList(Position pos, List<String> list) {
+ final ArrayList<Str> result = new ArrayList(list.size());
+ for (String s: list) {
+ result.add(new Str(pos, s));
+ }
+ return result;
+ }
}
