blob: 8b95490f808b3882256e3eab341768929ea593aa [file] [log] [blame]
# -*- python -*-
# ex: set filetype=python:
#
# Copyright 2018 The Kythe Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from os import path, listdir
import json
import urllib.request
from buildbot.changes.github import GitHubPullrequestPoller
from buildbot.plugins import *
from twisted.internet import defer
c = BuildmasterConfig = {}
c['buildbotNetUsageData'] = None
# https://github.com/buildbot/buildbot/issues/3627
# db_url and a few other places don't support Interpolate, so do it manually.
secrets_dir = path.join(path.dirname(__file__), "../secrets")
secrets_dict = {p: open(path.join(secrets_dir, p)).read().rstrip("\n")
for p in listdir(secrets_dir)}
c['secretsProviders'] = [secrets.SecretInAFile(dirname=secrets_dir)]
####### WORKERS
# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password. The same
# worker name and password must be configured on the worker.
c['workers'] = [
worker.Worker("local-worker", secrets_dict["worker_password"]),
]
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}
####### CHANGESOURCES
def pullRequestFilter(pr):
# Blacklist anything that explicitly says 'nobuildbot'
for label in pr['labels']:
if label['name'] == 'nobuildbot':
return False
# Allow all Kythe maintainer Pull Requests
url = 'https://api.github.com/teams/2988296/memberships/'+pr['user']['login']
req = urllib.request.Request(
url, headers={
'User-Agent': 'Buildbot',
'Authorization': 'token {github_token}'.format(**secrets_dict)
})
try:
with urllib.request.urlopen(req) as f:
userMembership = json.loads(f.read().decode('utf-8'))
if userMembership['state'] == 'active' and userMembership['role'] == 'maintainer':
return True
except:
# User is not a member of the Kythe team
pass
# Allow explicitly labeled PRs by non-maintainers if they signed the CLA
signed_cla = False
requested = False
for label in pr['labels']:
if label['name'] == 'cla: yes':
signed_cla = True
elif label['name'] == 'buildbot':
requested = True
return signed_cla and requested
c['change_source'] = []
def add_github_change_sources(repo, owner='kythe'):
# Track master
c['change_source'].append(changes.GitPoller(
'git://github.com/%s/%s.git' % (owner, repo),
project='%s/%s' % (owner, repo),
workdir='%s-workdir' % repo, branches=['master'],
category='master', pollAtLaunch=True, pollInterval=300))
# Track pull requests
c['change_source'].append(GitHubPullrequestPoller(
owner=owner, repo=repo, repository_type='git',
pullrequest_filter=pullRequestFilter,
category='pull', magic_link=True,
pollAtLaunch=True, pollInterval=60,
token=secrets_dict['github_token']))
add_github_change_sources('kythe')
####### SCHEDULERS
goVersion = '1.12'
bazelBuilders = [
'bazel-minversion',
'bazel',
]
goBuilders = [
'go-'+goVersion+'-gopath',
'go-'+goVersion+'-module'
]
releaseBuilders = [
'bazel-release',
]
allBuilders = bazelBuilders + goBuilders + releaseBuilders
c['schedulers'] = []
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="bazel-master",
change_filter=util.ChangeFilter(
category='master',
branch='master',
project=['kythe/kythe'],
),
builderNames=bazelBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="bazel-pull-requests",
change_filter=util.ChangeFilter(
category='pull',
project=['kythe/kythe'],
),
builderNames=bazelBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="go-master",
change_filter=util.ChangeFilter(
category='master',
branch='master',
project=['kythe/kythe'],
),
builderNames=goBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="go-pull-requests",
change_filter=util.ChangeFilter(
category='pull',
project=['kythe/kythe'],
),
builderNames=goBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="release-master",
change_filter=util.ChangeFilter(
category='master',
branch='master',
project=['kythe/kythe'],
),
builderNames=releaseBuilders))
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="release-pull-requests",
change_filter=util.ChangeFilter(
category='pull',
project=['kythe/kythe'],
),
builderNames=releaseBuilders))
# Allow builders to be "forced"
c['schedulers'].append(schedulers.ForceScheduler(name="force", builderNames=allBuilders))
####### BUILDERS
bazelKytheSteps = util.BuildFactory()
bazelKytheSteps.addStep(steps.GitHub(repourl=util.Property('repository', 'git://github.com/kythe/kythe.git'),
mode='full', method='copy'))
bazelBinEnv = {'USE_BAZEL_VERSION': util.Property('bazel_version')}
bazelKytheSteps.addStep(steps.ShellCommand(
command=["bazel", "version"],
env=bazelBinEnv))
bazelKytheSteps.addStep(steps.ShellCommand(
command=["bazel", "test",
util.Property('bazel_flags', default=[]),
util.Property('bazel_targets', default=['//...'])],
env=bazelBinEnv))
goKytheSteps = util.BuildFactory()
goKytheSteps.addStep(steps.GitHub(repourl=util.Property('repository', 'git://github.com/kythe/kythe.git'),
workdir='src/kythe.io',
mode='full', method='copy'))
goKytheSteps.addStep(steps.ShellCommand(command=["go", "get", "-t", "kythe.io/kythe/go/..."],
env={"GOPATH": util.Property('builddir'),
"GO111MODULE": "off"},
haltOnFailure=True))
goKytheSteps.addStep(steps.ShellCommand(command=["go", "test", "kythe.io/kythe/go/..."],
env={"GOPATH": util.Property('builddir'),
"GO111MODULE": "off"}))
goModuleKytheSteps = util.BuildFactory()
goModuleKytheSteps.addStep(steps.GitHub(repourl=util.Property('repository', 'git://github.com/kythe/kythe.git'),
mode='full', method='copy'))
goModuleKytheSteps.addStep(steps.ShellCommand(command=["go", "test", "./kythe/..."],
env={"GO111MODULE": "on"},
haltOnFailure=True))
# Check that the Kythe module files are up-to-date
goModuleKytheSteps.addStep(steps.ShellCommand(command="! { git diff --color=always -- go.{mod,sum} | grep .; }",
warnOnFailure=True))
build_lock = util.WorkerLock("worker_builds", maxCount=1, maxCountForWorker={'local-worker': 3})
c['builders'] = []
c['builders'].append(
util.BuilderConfig(
name="bazel-minversion",
workernames=["local-worker"],
properties={'bazel_version': 'min',
'bazel_flags': ['--config=remote']},
locks=[build_lock.access('counting')],
factory=bazelKytheSteps))
c['builders'].append(
util.BuilderConfig(
name="bazel",
workernames=["local-worker"],
properties={'bazel_flags': ['--config=remote']},
locks=[build_lock.access('counting')],
factory=bazelKytheSteps))
c['builders'].append(
util.BuilderConfig(
name="bazel-release",
workernames=["local-worker"],
properties={'bazel_flags': ['--config=remote', '-c', 'opt', '--stamp'],
'bazel_targets': [
'//...',
'//kythe/docs/schema',
'//kythe/release:release_test'
]},
locks=[build_lock.access('counting')],
factory=bazelKytheSteps))
c['builders'].append(
util.BuilderConfig(
name="go-"+goVersion+"-gopath",
workernames=["local-worker"],
locks=[build_lock.access('counting')],
factory=goKytheSteps))
c['builders'].append(
util.BuilderConfig(
name="go-"+goVersion+"-module",
workernames=["local-worker"],
locks=[build_lock.access('counting')],
factory=goModuleKytheSteps))
####### BUILDBOT SERVICES
@util.renderer
@defer.inlineCallbacks
def renderBuildState(props):
build = yield props.master.data.get(('builders', props.getProperty('buildername'),
'builds', props.getProperty('buildnumber')))
defer.returnValue(build['state_string'].capitalize())
# Push GitHub status messages for every build start/end.
c['services'] = []
c['services'].append(reporters.GitHubStatusPush(token=secrets_dict['github_token'],
startDescription=renderBuildState,
endDescription=renderBuildState))
####### PROJECT IDENTITY
c['title'] = "Kythe CI"
c['titleURL'] = "https://buildbot-dot-kythe-repo.appspot.com/"
c['buildbotURL'] = "https://buildbot-dot-kythe-repo.appspot.com/"
c['www'] = {
'port': 8080, # AppEngine web port
'plugins': dict(waterfall_view={}, console_view={}, grid_view={}),
'auth': util.GitHubAuth(secrets_dict["github_auth_id"], secrets_dict["github_auth_secret"]),
'authz': util.Authz(
# restrict control access to users in the 'google' org
allowRules=[util.AnyControlEndpointMatcher(role="google")],
roleMatchers=[util.RolesFromGroups()]
),
}
####### DB URL
c['db'] = {
'db_url': "postgresql+psycopg2://{db_userpass}@/buildbot?host={db_host}".format(**secrets_dict),
}