Generate separate VTS models for each HAL version
Prior to this change, the VTS test generator produced the same code
for all HAL versions. This made it it difficult to introduce any structural
changes in newer versions. For example, it was difficult to rename
1.1@Operand.type to 1.2@Operand.extendedType (see b/121193452).
This change makes the VTS test generator aware of the output HAL version,
eliminating the problem.
Further, the generated VTS code is included in different test suites using
the following pattern:
namespace V1_X {
#include "all_generated_V1_Y_vts_tests.cpp"
}
This can lead to an awkward situation when a 1.0 generated test exposes
1.2 features (see b/122740334). This change transforms the pattern to
namespace V1_X {
#include "V1_X/all_generated_V1_Y_vts_tests.cpp"
}
which is the first step to removing the bad pattern altogether.
Bug: 124040454
Fix: 121193452
Bug: 122740334
Test: HAL_VERSION=all ./runtime/test/specs/generate_vts_test.sh
Test: ./runtime/test/specs/generate_test.sh --force
Change-Id: If3bfbdfa016cd32a39b8a63028481da7318d6e02
diff --git a/runtime/test/specs/generate_vts_test.sh b/runtime/test/specs/generate_vts_test.sh
index c20d966..dc223c2 100755
--- a/runtime/test/specs/generate_vts_test.sh
+++ b/runtime/test/specs/generate_vts_test.sh
@@ -14,14 +14,29 @@
# limitations under the License.
# usage: generate_vts_test.sh <tests>
+# usage: HAL_VERSION=V1_0 generate_vts_test.sh <tests>
+# usage: HAL_VERSION=all generate_vts_test.sh <tests>
set -Eeuo pipefail
-NNAPI_VERSION="
-V1_0
-V1_1
-V1_2
-"
+NNAPI_VERSIONS="V1_0 V1_1 V1_2"
+LATEST_NNAPI_VERSION="V1_2"
+
+# TODO: Refactor the script to make this a command-line flag.
+HAL_VERSION="${HAL_VERSION:-$LATEST_NNAPI_VERSION}"
+
+if [[ "$HAL_VERSION" == "all" ]]; then
+ # Re-call this script with each version explicitly.
+ for ver in $NNAPI_VERSIONS; do
+ HAL_VERSION="$ver" "$0" "$@"
+ done
+ exit
+elif [[ " $NNAPI_VERSIONS " != *" $HAL_VERSION "* || "$HAL_VERSION" == *" "* ]]; then
+ echo "Unknown HAL version: $HAL_VERSION; expected one of: $NNAPI_VERSIONS all"
+ exit 1
+fi
+
+echo "Generating VTS tests for HAL $HAL_VERSION"
cd "$(dirname $0)"
@@ -29,18 +44,22 @@
function generate_one_testcase {
# Generate one testcase
BASENAME=`basename -s .mod.py $1`
- $VTS_PATH/../../tools/test_generator/vts_generator.py ./`basename $1`\
- -m $VTS_PATH/generated/vts_models/$BASENAME.model.cpp \
- -e $VTS_PATH/generated/examples/$BASENAME.example.cpp
+ $VTS_PATH/../../tools/test_generator/vts_generator.py ./`basename $1` \
+ --model $VTS_PATH/generated/vts/$HAL_VERSION/models/$BASENAME.model.cpp \
+ --example $VTS_PATH/generated/examples/$BASENAME.example.cpp \
+ --target_hal_version $HAL_VERSION
}
function generate_wrapper {
- for ver in $NNAPI_VERSION;
+ for ver in $NNAPI_VERSIONS;
do
+ if [[ "$ver" > "$HAL_VERSION" ]]; then
+ break
+ fi
VER_DIR=$VTS_PATH/specs/$ver
[ ! -d $VER_DIR ] && continue
pushd $VER_DIR > /dev/null
- OUTFILE=$VTS_PATH/generated/all_generated_${ver}_vts_tests.cpp
+ OUTFILE=$VTS_PATH/generated/vts/$HAL_VERSION/all_generated_${ver}_vts_tests.cpp
echo "// clang-format off" > $OUTFILE
echo "// DO NOT EDIT;" >> $OUTFILE
echo "// Generated by ml/nn/runtime/test/specs/generate_vts_test.sh" >> $OUTFILE
@@ -59,16 +78,21 @@
}
function generate_spec_dirs {
- for ver in $NNAPI_VERSION;
+ for ver in $NNAPI_VERSIONS;
do
+ if [[ "$ver" > "$HAL_VERSION" ]]; then
+ break
+ fi
VER_DIR=$VTS_PATH/specs/$ver
[ ! -d $VER_DIR ] && continue
- OUTFILE=$VTS_PATH/generated/all_generated_${ver}_vts_tests.cpp
+ OUTFILE=$VTS_PATH/generated/vts/$HAL_VERSION/all_generated_${ver}_vts_tests.cpp
echo "// clang-format off" > $OUTFILE
echo "// DO NOT EDIT;" >> $OUTFILE
echo "// Generated by ml/nn/runtime/test/specs/generate_vts_test.sh" >> $OUTFILE
$VTS_PATH/../../tools/test_generator/vts_generator.py $VER_DIR \
- -m $VTS_PATH/generated/vts_models -e $VTS_PATH/generated/examples >> $OUTFILE
+ --model $VTS_PATH/generated/vts/$HAL_VERSION/models \
+ --example $VTS_PATH/generated/examples \
+ --target_hal_version $HAL_VERSION >> $OUTFILE
echo "Generated file in $VTS_PATH/generated/"`basename $OUTFILE`
done
}
diff --git a/tools/test_generator/cts_generator.py b/tools/test_generator/cts_generator.py
index 3214462..e8f5e7f 100755
--- a/tools/test_generator/cts_generator.py
+++ b/tools/test_generator/cts_generator.py
@@ -293,7 +293,7 @@
print("Generating test(s) from spec: %s" % tg.FileNames.specFile, file=sys.stderr)
exec(open(tg.FileNames.specFile, "r").read())
print("Output CTS model: %s" % tg.FileNames.modelFile, file=sys.stderr)
- print("Output example:%s" % tg.FileNames.exampleFile, file=sys.stderr)
+ print("Output example: %s" % tg.FileNames.exampleFile, file=sys.stderr)
print("Output CTS test: %s" % tg.FileNames.testFile, file=sys.stderr)
with SmartOpen(tg.FileNames.modelFile) as model_fd, \
SmartOpen(tg.FileNames.exampleFile) as example_fd, \
diff --git a/tools/test_generator/vts_generator.py b/tools/test_generator/vts_generator.py
index 109b632..09c689d 100755
--- a/tools/test_generator/vts_generator.py
+++ b/tools/test_generator/vts_generator.py
@@ -64,6 +64,11 @@
from cts_generator import DumpCtsExample
from cts_generator import DumpCtsIsIgnored
+
+# TODO: Make this part of tg.Configuration?
+target_hal_version = None
+
+
# Take a model from command line
def ParseCmdLine():
parser = argparse.ArgumentParser()
@@ -74,9 +79,16 @@
"-e", "--example", help="the output example file", default="-")
parser.add_argument(
"-t", "--test", help="the output test file", default="-")
+ parser.add_argument(
+ "--target_hal_version",
+ help="the HAL version of the output",
+ required=True,
+ choices=["V1_0", "V1_1", "V1_2"])
args = parser.parse_args()
tg.FileNames.InitializeFileLists(
args.spec, args.model, args.example, args.test)
+ global target_hal_version
+ target_hal_version = args.target_hal_version
# Generate operands in VTS format
def generate_vts_operands(model):
@@ -251,6 +263,7 @@
}}
"""
model_dict = {
+ "hal_version": target_hal_version,
"create_test_model_name": str(model.createTestFunctionName),
"operations": generate_vts_operations(model),
"operand_decls": generate_vts_operands(model),
@@ -323,7 +336,7 @@
print("Generating test(s) from spec: %s" % tg.FileNames.specFile, file=sys.stderr)
exec (open(tg.FileNames.specFile, "r").read())
print("Output VTS model: %s" % tg.FileNames.modelFile, file=sys.stderr)
- print("Output example:" + tg.FileNames.exampleFile, file=sys.stderr)
+ print("Output example: " + tg.FileNames.exampleFile, file=sys.stderr)
with SmartOpen(tg.FileNames.modelFile) as model_fd, \
SmartOpen(tg.FileNames.exampleFile) as example_fd, \
SmartOpen(tg.FileNames.testFile, mode="a") as test_fd:
