[infra] Make infra tests idempotent
Change-Id: I1a75b211753ea5e097234dd4104d0906cfd04673
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/223916
Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go
index 12d85c5..fd8a7eb 100644
--- a/infra/bots/gen_tasks.go
+++ b/infra/bots/gen_tasks.go
@@ -1057,9 +1057,13 @@
fmt.Sprintf("pool:%s", CONFIG.Pool),
}
}
- task := kitchenTask(name, "infra", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, dims, EXTRA_PROPS, OUTPUT_NONE)
+ extraProps := map[string]string{
+ "repository": specs.PLACEHOLDER_REPO,
+ }
+ task := kitchenTask(name, "infra", "infra_tests.isolate", SERVICE_ACCOUNT_COMPILE, dims, extraProps, OUTPUT_NONE)
task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GSUTIL...)
- usesGit(task, name)
+ task.Idempotent = true
+ usesGit(task, name) // We don't run bot_update, but Go needs a git repo.
usesGo(b, task, name)
b.MustAddTask(name, task)
return name
diff --git a/infra/bots/infra_tests.isolate b/infra/bots/infra_tests.isolate
new file mode 100644
index 0000000..ea52292
--- /dev/null
+++ b/infra/bots/infra_tests.isolate
@@ -0,0 +1,11 @@
+{
+ 'includes': [
+ 'swarm_recipe.isolate',
+ ],
+ 'variables': {
+ 'files': [
+ '../../go.mod',
+ '../../go.sum',
+ ],
+ },
+}
diff --git a/infra/bots/recipes/infra.expected/infra_tests.json b/infra/bots/recipes/infra.expected/infra_tests.json
index c4f74ec..377e02b 100644
--- a/infra/bots/recipes/infra.expected/infra_tests.json
+++ b/infra/bots/recipes/infra.expected/infra_tests.json
@@ -1,105 +1,60 @@
[
{
"cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[START_DIR]/cache/work"
+ "git",
+ "init"
],
- "infra_step": true,
- "name": "makedirs checkout_path"
- },
- {
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "remove",
- "[START_DIR]/cache/work/.gclient_entries"
- ],
- "infra_step": true,
- "name": "remove [START_DIR]/cache/work/.gclient_entries"
- },
- {
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
- "--spec-path",
- "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
- "--patch_root",
- "skia",
- "--revision_mapping_file",
- "{\"got_revision\": \"skia\"}",
- "--git-cache-dir",
- "[START_DIR]/cache/git",
- "--cleanup-dir",
- "[CLEANUP]/bot_update",
- "--output_json",
- "/path/to/tmp/json",
- "--revision",
- "skia@abc123"
- ],
- "cwd": "[START_DIR]/cache/work",
- "env_prefixes": {
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "GOCACHE": "[START_DIR]/cache/go_cache",
+ "GOPATH": "[START_DIR]/cache/gopath",
+ "GOROOT": "[START_DIR]/go/go",
+ "PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
},
- "infra_step": true,
- "name": "bot_update",
- "~followup_annotations": [
- "@@@STEP_TEXT@Some step text@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@",
- "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
- "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
- ]
+ "name": "git init"
+ },
+ {
+ "cmd": [
+ "git",
+ "add",
+ "."
+ ],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "GOCACHE": "[START_DIR]/cache/go_cache",
+ "GOPATH": "[START_DIR]/cache/gopath",
+ "GOROOT": "[START_DIR]/go/go",
+ "PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "git add ."
+ },
+ {
+ "cmd": [
+ "git",
+ "commit",
+ "-a",
+ "-m",
+ "initial commit"
+ ],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "GOCACHE": "[START_DIR]/cache/go_cache",
+ "GOPATH": "[START_DIR]/cache/gopath",
+ "GOROOT": "[START_DIR]/go/go",
+ "PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "git commit"
},
{
"cmd": [
"python",
"-u",
- "[START_DIR]/cache/work/skia/infra/bots/infra_tests.py"
+ "[START_DIR]/skia/infra/bots/infra_tests.py"
],
- "cwd": "[START_DIR]/cache/work/skia",
+ "cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOCACHE": "[START_DIR]/cache/go_cache",
diff --git a/infra/bots/recipes/infra.py b/infra/bots/recipes/infra.py
index 5a90dcc..cfed268 100644
--- a/infra/bots/recipes/infra.py
+++ b/infra/bots/recipes/infra.py
@@ -7,9 +7,9 @@
DEPS = [
- 'checkout',
'infra',
'recipe_engine/context',
+ 'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/step',
'vars',
@@ -18,14 +18,12 @@
def RunSteps(api):
api.vars.setup()
- checkout_root = api.checkout.default_checkout_root
- api.checkout.bot_update(checkout_root=checkout_root)
# Run the infra tests.
repo_name = api.properties['repository'].split('/')[-1]
if repo_name.endswith('.git'):
repo_name = repo_name[:-len('.git')]
- repo_root = checkout_root.join(repo_name)
+ repo_root = api.path['start_dir'].join(repo_name)
infra_tests = repo_root.join('infra', 'bots', 'infra_tests.py')
# Merge the default environment with the Go environment.
@@ -40,7 +38,12 @@
else:
env[k] = v
- with api.context(cwd=checkout_root.join(repo_name), env=env):
+ with api.context(cwd=repo_root, env=env):
+ # Some tests assume that they're being run inside a git repo.
+ api.step('git init', cmd=['git', 'init'])
+ api.step('git add .', cmd=['git', 'add', '.'])
+ api.step('git commit', cmd=['git', 'commit', '-a', '-m', 'initial commit'])
+
# Unfortunately, the recipe tests are flaky due to file removal on Windows.
# Run multiple attempts.
last_exc = None
@@ -58,7 +61,6 @@
api.test('infra_tests') +
api.properties(buildername='Housekeeper-PerCommit-InfraTests_Win',
repository='https://skia.googlesource.com/skia.git',
- revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]')
)
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 8fa7b8f..5507230 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -17485,7 +17485,7 @@
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"infra",
- "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Housekeeper-PerCommit-InfraTests_Linux\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}",
+ "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Housekeeper-PerCommit-InfraTests_Linux\",\"repository\":\"<(REPO)\",\"swarm_out_dir\":\"output_ignored\"}",
"skia"
],
"dependencies": [
@@ -17511,8 +17511,9 @@
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
+ "idempotent": true,
"io_timeout_ns": 3600000000000,
- "isolate": "swarm_recipe.isolate",
+ "isolate": "infra_tests.isolate",
"max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
@@ -17595,7 +17596,7 @@
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"infra",
- "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Housekeeper-PerCommit-InfraTests_Win\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}",
+ "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Housekeeper-PerCommit-InfraTests_Win\",\"repository\":\"<(REPO)\",\"swarm_out_dir\":\"output_ignored\"}",
"skia"
],
"dependencies": [
@@ -17621,8 +17622,9 @@
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
+ "idempotent": true,
"io_timeout_ns": 3600000000000,
- "isolate": "swarm_recipe.isolate",
+ "isolate": "infra_tests.isolate",
"max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
