ci: Generate MinIO credentials within LAVA jobs
As these credentials are valid only for 15 minutes, generate them closer
to when they are going to be used.
Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Daniel Stone <daniels@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6124>
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index 0055ef5..7501d09 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -43,7 +43,7 @@
variables:
GIT_STRATEGY: none # testing doesn't build anything from source
ENV_VARS: "DEQP_PARALLEL=6"
- FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1"
+ FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1"
DEQP_VERSION: gles2
ARTIFACTS_PREFIX: "https://minio-packet.freedesktop.org/mesa-lava/"
MESA_URL: "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index a378b99..85b0e5b 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -38,10 +38,6 @@
cp -Rp .gitlab-ci/deqp-*-fails.txt install/
cp -Rp .gitlab-ci/deqp-*-skips.txt install/
-ci-fairy minio login $CI_JOB_JWT
-# These credentials will be used for uploading artifacts from test jobs
-cp .minio_credentials install/
-
# Tar up the install dir so that symlinks and hardlinks aren't each
# packed separately in the zip file.
mkdir -p artifacts/
@@ -55,5 +51,6 @@
gzip -c artifacts/install.tar > mesa-${DEBIAN_ARCH}.tar.gz
MINIO_PATH=minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
+ ci-fairy minio login $CI_JOB_JWT
ci-fairy minio cp mesa-${DEBIAN_ARCH}.tar.gz minio://${MINIO_PATH}/mesa-${DEBIAN_ARCH}.tar.gz
fi
diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh
index 5c864b9..246e79c 100755
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@@ -31,9 +31,6 @@
RESULTS=`pwd`/results
mkdir -p $RESULTS
-# For artifact uploads to MinIO
-cp install/.minio_credentials .
-
# Perform a self-test to ensure tracie is working properly.
if [ -z "$TRACIE_NO_UNIT_TESTS" ]; then
TRACIE_UPLOAD_TO_MINIO=0 python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh
index da34bd4..f046b7c 100755
--- a/.gitlab-ci/tracie-runner-vk.sh
+++ b/.gitlab-ci/tracie-runner-vk.sh
@@ -20,9 +20,6 @@
export DXVK_LOG_LEVEL="none"
export DXVK_STATE_CACHE=0
-# For artifact uploads to MinIO
-cp install/.minio_credentials .
-
# Perform a self-test to ensure tracie is working properly.
python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py
index 0a2c123..2c96bf4 100644
--- a/.gitlab-ci/tracie/tracie.py
+++ b/.gitlab-ci/tracie/tracie.py
@@ -13,6 +13,7 @@
import time
import yaml
import shutil
+import xml.etree.ElementTree as ET
from email.utils import formatdate
from pathlib import Path
@@ -26,6 +27,8 @@
MINIO_HOST = "minio-packet.freedesktop.org"
DASHBOARD_URL = "https://tracie.freedesktop.org/dashboard"
+minio_credentials = None
+
def replay(trace_path, device_name):
success = dump_trace_images.dump_from_trace(trace_path, [], device_name)
@@ -70,12 +73,38 @@
return base64.encodebytes(signature).strip().decode()
+def ensure_minio_credentials():
+ global minio_credentials
+
+ if minio_credentials is None:
+ minio_credentials = {}
+
+ params = {'Action': 'AssumeRoleWithWebIdentity',
+ 'Version': '2011-06-15',
+ 'RoleArn': 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole',
+ 'RoleSessionName': '%s:%s' % (os.environ['CI_PROJECT_PATH'], os.environ['CI_JOB_ID']),
+ 'DurationSeconds': 900,
+ 'WebIdentityToken': os.environ['CI_JOB_JWT']}
+ r = requests.post('https://%s' % (MINIO_HOST), params=params)
+ if r.status_code >= 400:
+ print(r.text)
+ r.raise_for_status()
+
+ root = ET.fromstring(r.text)
+ for attr in root.iter():
+ if attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}AccessKeyId':
+ minio_credentials['AccessKeyId'] = attr.text
+ elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SecretAccessKey':
+ minio_credentials['SecretAccessKey'] = attr.text
+ elif attr.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}SessionToken':
+ minio_credentials['SessionToken'] = attr.text
+
def upload_to_minio(file_name, resource, content_type):
- with open('.minio_credentials', 'r') as f:
- credentials = json.load(f)[MINIO_HOST]
- minio_key = credentials["AccessKeyId"]
- minio_secret = credentials["SecretAccessKey"]
- minio_token = credentials["SessionToken"]
+ ensure_minio_credentials()
+
+ minio_key = minio_credentials['AccessKeyId']
+ minio_secret = minio_credentials['SecretAccessKey']
+ minio_token = minio_credentials['SessionToken']
date = formatdate(timeval=None, localtime=False, usegmt=True)
url = 'https://%s%s' % (MINIO_HOST, resource)
