Refactor docker image publishing
This adds support to push docker images to quay.io, like other projects in
the iovisor org.
It separates docker image builds into a separate github workflow, and
refactors the package building process slightly, to be generic, in order to
create builds for both ubuntu 16.04 and ubuntu 18.04.
This provides a means to distribute intermediate apt packages between releases,
and also enables uploading these as CI artifacts.
As recent releases have not annotated their tags, it drops the requirement for
tags to be annotated in selecting the version to use.
diff --git a/.github/workflows/bcc-test.yml b/.github/workflows/bcc-test.yml
index 1a72ff5..4907e29 100644
--- a/.github/workflows/bcc-test.yml
+++ b/.github/workflows/bcc-test.yml
@@ -92,38 +92,3 @@
# https://github.com/marketplace/actions/debugging-with-tmate
# - name: Setup tmate session
# uses: mxschmitt/action-tmate@v1
-
- # Optionally publish container images, guarded by the GitHub secret
- # DOCKER_PUBLISH.
- # GitHub secrets can be configured as follows:
- # - DOCKER_PUBLISH = 1
- # - DOCKER_IMAGE = docker.io/myorg/bcc
- # - DOCKER_USERNAME = username
- # - DOCKER_PASSWORD = password
- publish:
- name: Publish
- runs-on: ubuntu-latest
- steps:
-
- - uses: actions/checkout@v1
-
- - name: Initialize workflow variables
- id: vars
- shell: bash
- run: |
- echo ::set-output name=DOCKER_PUBLISH::${DOCKER_PUBLISH}
- env:
- DOCKER_PUBLISH: "${{ secrets.DOCKER_PUBLISH }}"
-
- - name: Build container image and publish to registry
- id: publish-registry
- uses: elgohr/Publish-Docker-Github-Action@2.8
- if: ${{ steps.vars.outputs.DOCKER_PUBLISH }}
- with:
- name: ${{ secrets.DOCKER_IMAGE }}
- username: ${{ secrets.DOCKER_USERNAME }}
- password: ${{ secrets.DOCKER_PASSWORD }}
- workdir: .
- dockerfile: Dockerfile.ubuntu
- snapshot: true
- cache: ${{ github.event_name != 'schedule' }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 0000000..3021a95
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,101 @@
+name: Publish Build Artifacts
+
+on: push
+
+jobs:
+ publish_images:
+ # Optionally publish container images, guarded by the GitHub secret
+ # QUAY_PUBLISH.
+ # To set this up, sign up for quay.io (you can connect it to your github)
+ # then create a robot user with write access user called "bcc_buildbot",
+ # and add the secret token for it to GitHub secrets as:
+ # - QUAY_TOKEN = <token from quay.io>
+ name: Publish to quay.io
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ env:
+ - NAME: xenial-release
+ OS_RELEASE: 16.04
+ - NAME: bionic-release
+ OS_RELEASE: 18.04
+ steps:
+
+ - uses: actions/checkout@v1
+
+ - name: Initialize workflow variables
+ id: vars
+ shell: bash
+ run: |
+ if [ -n "${QUAY_TOKEN}" ];then
+ echo "Quay token is set, will push an image"
+ echo ::set-output name=QUAY_PUBLISH::true
+ else
+ echo "Quay token not set, skipping"
+ fi
+
+ env:
+ QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
+
+ - name: Authenticate with quay.io docker registry
+ if: >
+ steps.vars.outputs.QUAY_PUBLISH
+ env:
+ QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}
+ run: ./scripts/docker/auth.sh ${{ github.repository }}
+
+ - name: Package docker image and push to quay.io
+ if: >
+ steps.vars.outputs.QUAY_PUBLISH
+ run: >
+ ./scripts/docker/push.sh
+ ${{ github.repository }}
+ ${{ github.ref }}
+ ${{ github.sha }}
+ ${{ matrix.env['NAME'] }}
+ ${{ matrix.env['OS_RELEASE'] }}
+
+ # Uploads the packages built in docker to the github build as an artifact for convenience
+ - uses: actions/upload-artifact@v1
+ with:
+ name: ${{ matrix.env['NAME'] }}
+ path: output
+
+ # Optionally publish container images to custom docker repository,
+ # guarded by presence of all required github secrets.
+ # GitHub secrets can be configured as follows:
+ # - DOCKER_IMAGE = docker.io/myorg/bcc
+ # - DOCKER_USERNAME = username
+ # - DOCKER_PASSWORD = password
+ publish_dockerhub:
+ name: Publish To Dockerhub
+ runs-on: ubuntu-latest
+ steps:
+
+ - uses: actions/checkout@v1
+
+ - name: Initialize workflow variables
+ id: vars
+ shell: bash
+ run: |
+ if [ -n "${DOCKER_IMAGE}" ] && \
+ [ -n "${DOCKER_USERNAME}" ] && \
+ [ -n "${DOCKER_PASSWORD}" ];then
+ echo "Custom docker credentials set, will push an image"
+ echo ::set-output name=DOCKER_PUBLISH::true
+ else
+ echo "Custom docker credentials not, skipping"
+ fi
+
+ - name: Build container image and publish to registry
+ id: publish-registry
+ uses: elgohr/Publish-Docker-Github-Action@2.8
+ if: ${{ steps.vars.outputs.DOCKER_PUBLISH }}
+ with:
+ name: ${{ secrets.DOCKER_IMAGE }}
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_PASSWORD }}
+ workdir: .
+ dockerfile: Dockerfile.ubuntu
+ snapshot: true
+ cache: ${{ github.event_name != 'schedule' }}
diff --git a/.gitignore b/.gitignore
index 14994d7..18a6f5a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,7 @@
*critical.log
obj-x86_64-linux-gnu
examples/cgroupid/cgroupid
+
+# Output from docker builds
+scripts/docker/output/
+/output/
diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu
index a2582cd..1aeb841 100644
--- a/Dockerfile.ubuntu
+++ b/Dockerfile.ubuntu
@@ -1,4 +1,9 @@
-FROM ubuntu:bionic as builder
+ARG OS_TAG=18.04
+FROM ubuntu:${OS_TAG} as builder
+
+ARG OS_TAG
+ARG BUILD_TYPE=release
+ARG DEBIAN_FRONTEND=noninteractive
MAINTAINER Brenden Blanco <bblanco@gmail.com>
@@ -10,10 +15,9 @@
WORKDIR /root/bcc
RUN /usr/lib/pbuilder/pbuilder-satisfydepends && \
- ./scripts/build-deb.sh
+ ./scripts/build-deb.sh ${BUILD_TYPE}
-
-FROM ubuntu:bionic
+FROM ubuntu:${OS_TAG}
COPY --from=builder /root/bcc/*.deb /root/bcc/
diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh
index 1e450b6..2fe7e3b 100755
--- a/scripts/build-deb.sh
+++ b/scripts/build-deb.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# helper script to be invoked by jenkins/buildbot
+# helper script to be invoked by jenkins/buildbot or github actions
# $1 [optional]: the build type - release | nightly | test
buildtype=${1:-test}
diff --git a/scripts/docker/auth.sh b/scripts/docker/auth.sh
new file mode 100755
index 0000000..a4166be
--- /dev/null
+++ b/scripts/docker/auth.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+# For now only quay.io is supported, but this could be portable to dockerhub
+# and other image repositories.
+
+# Forks can push using this approach if they create a quay.io bot user
+# with name matching of ORGNAME+bcc_buildbot, or by setting QUAY_BOT_NAME
+
+git_repo=$1 # github.repository format: ORGNAME/REPONAME
+
+# Set this value as QUAY_TOKEN in the github repository settings "Secrets" tab
+[[ -z "${QUAY_TOKEN}" ]] && echo "QUAY_TOKEN not set" && exit 0
+
+# Set this to match the name of the bot user on quay.io
+[[ -z "${QUAY_BOT_NAME}" ]] && QUAY_BOT_NAME="bcc_buildbot"
+
+quay_user="$(dirname ${git_repo})+${QUAY_BOT_NAME}"
+echo "${QUAY_TOKEN}" | docker login -u="${quay_user}" --password-stdin quay.io
diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh
new file mode 100755
index 0000000..728ef6d
--- /dev/null
+++ b/scripts/docker/build.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Builds debian packages using docker wrapper
+
+function help() {
+ message=$1
+ echo "USAGE: build.sh DOCKER_REPO DOCKER_TAG OS_TAG [DISTRO]"
+ echo "hint: ${message}"
+}
+
+docker_repo=$1
+docker_tag=$2
+os_tag=$3
+distro=${4:-ubuntu}
+
+[ -z "${docker_repo}" ] && help "You must specify repo, eg: quay.io/iovisoc/bcc" && exit 1
+[ -z "${docker_tag}" ] && help "You must specify tag, eg: bionic-release-master, latest, SHA, git tag, etc " && exit 1
+[ -z "${os_tag}" ] && help "You must specify os tag, eg: 18.04, bionic, etc " && exit 1
+
+
+# The main docker image build,
+echo "Building ${distro} ${os_tag} release docker image for ${docker_repo}:${docker_tag}"
+docker build -t ${docker_repo}:${docker_tag} --build-arg OS_TAG=${os_tag} -f Dockerfile.${distro} .
+
+echo "Copying build artifacts to $(pwd)/output"
+mkdir output
+docker run -v $(pwd)/output:/output ${docker_repo}:${docker_tag} /bin/bash -c "cp /root/bcc/* /output"
diff --git a/scripts/docker/push.sh b/scripts/docker/push.sh
new file mode 100755
index 0000000..8a48526
--- /dev/null
+++ b/scripts/docker/push.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+# Push docker tags to a configured docker repo, defaulting to quay.io
+# You must run login.sh before running this script.
+
+DEFAULT_DOCKER_REPO="quay.io"
+DEFAULT_RELEASE_TARGET="bionic-release" # will allow unprefixed tags
+
+# Currently only support pushing to quay.io
+DOCKER_REPO=${DEFAULT_DOCKER_REPO}
+
+git_repo=$1 # github.repository format: ORGNAME/REPONAME
+git_ref=$2 # github.ref format: refs/REMOTE/REF
+ # eg, refs/heads/BRANCH
+ # refs/tags/v0.9.6-pre
+git_sha=$3 # github.sha GIT_SHA
+type_name=$4 # build name, s/+/_/g eg, bionic-release
+os_tag=${5:-18.04} # numeric docker tag eg, 18.04
+
+# refname will be either a branch like "master" or "some-branch",
+# or a tag, like "v1.17.0-pre".
+# When a tag is pushed, a build is done for both the branch and the tag, as
+# separate builds.
+# This is a feature specific to github actions based on the `github.ref` object
+refname=$(basename ${git_ref})
+
+# The build type needs to be sanitized into a valid tag, replacing + with _
+type_tag="$(echo ${type_name} | sed 's/+/_/g')"
+
+
+echo "Triggering image build"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+${SCRIPT_DIR}/build.sh ${DOCKER_REPO}/${git_repo} ${git_sha}-${type_tag} ${os_tag}
+
+echo "Upload image for git sha ${git_sha} to ${DOCKER_REPO}/${git_repo}"
+docker push ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag}
+
+echo "Push tags to branch or git tag HEAD refs"
+docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:${refname}-${type_tag}
+docker push ${DOCKER_REPO}/${git_repo}:${refname}-${type_tag}
+
+# Only push to un-suffixed tags for the default release target build type
+if [[ "${type_name}" == "${DEFAULT_RELEASE_TARGET}"* ]];then
+
+ # Update branch / git tag ref
+ echo "Pushing tags for ${DOCKER_REPO}/${git_repo}:${refname}"
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:${refname}
+ docker push ${DOCKER_REPO}/${git_repo}:${refname}
+
+ if [[ "${refname}" == "master" ]];then
+ if [[ "${edge}" == "ON" ]];then
+ echo "This is an edge build on master, pushing ${DOCKER_REPO}/${git_repo}:edge"
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:edge
+ docker push ${DOCKER_REPO}/${git_repo}:edge
+ else
+ echo "This is a build on master, pushing ${DOCKER_REPO}/${git_repo}:latest :SHA as well"
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:latest
+ docker tag ${DOCKER_REPO}/${git_repo}:${git_sha}-${type_tag} ${DOCKER_REPO}/${git_repo}:${git_sha}
+ docker push ${DOCKER_REPO}/${git_repo}:latest
+ docker push ${DOCKER_REPO}/${git_repo}:${git_sha}
+ fi
+ fi
+fi
diff --git a/scripts/git-tag.sh b/scripts/git-tag.sh
index 073b4a1..5aef730 100644
--- a/scripts/git-tag.sh
+++ b/scripts/git-tag.sh
@@ -1,4 +1,4 @@
-git_tag_latest=$(git describe --abbrev=0)
+git_tag_latest=$(git describe --tags --abbrev=0)
git_rev_count=$(git rev-list $git_tag_latest.. --count)
git_rev_count=$[$git_rev_count+1]
git_subject=$(git log --pretty="%s" -n 1)
