| # Copyright 2020 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| ################################################################################ |
| """Cloud function to request builds.""" |
| import base64 |
| import concurrent.futures |
| import json |
| import sys |
| |
| import google.auth |
| from googleapiclient.discovery import build |
| from google.cloud import ndb |
| from google.cloud import storage |
| |
| import build_and_run_coverage |
| import build_project |
| from datastore_entities import BuildsHistory |
| from datastore_entities import LastSuccessfulBuild |
| from datastore_entities import Project |
| |
| BADGE_DIR = 'badge_images' |
| BADGE_IMAGE_TYPES = {'svg': 'image/svg+xml', 'png': 'image/png'} |
| DESTINATION_BADGE_DIR = 'badges' |
| MAX_BUILD_LOGS = 7 |
| |
| STATUS_BUCKET = 'oss-fuzz-build-logs' |
| |
| FUZZING_STATUS_FILENAME = 'status.json' |
| COVERAGE_STATUS_FILENAME = 'status-coverage.json' |
| |
| # pylint: disable=invalid-name |
| _client = None |
| |
| |
| class MissingBuildLogError(Exception): |
| """Missing build log file in cloud storage.""" |
| |
| |
| # pylint: disable=global-statement |
| def get_storage_client(): |
| """Return storage client.""" |
| global _client |
| if not _client: |
| _client = storage.Client() |
| |
| return _client |
| |
| |
| def is_build_successful(build_obj): |
| """Check build success.""" |
| return build_obj['status'] == 'SUCCESS' |
| |
| |
| def upload_status(data, status_filename): |
| """Upload json file to cloud storage.""" |
| bucket = get_storage_client().get_bucket(STATUS_BUCKET) |
| blob = bucket.blob(status_filename) |
| blob.cache_control = 'no-cache' |
| blob.upload_from_string(json.dumps(data), content_type='application/json') |
| |
| |
| def sort_projects(projects): |
| """Sort projects in order Failures, Successes, Not yet built.""" |
| |
| def key_func(project): |
| if not project['history']: |
| return 2 # Order projects without history last. |
| |
| if project['history'][0]['success']: |
| # Successful builds come second. |
| return 1 |
| |
| # Build failures come first. |
| return 0 |
| |
| projects.sort(key=key_func) |
| |
| |
| def get_build(cloudbuild, image_project, build_id): |
| """Get build object from cloudbuild.""" |
| return cloudbuild.projects().builds().get(projectId=image_project, |
| id=build_id).execute() |
| |
| |
| def update_last_successful_build(project, build_tag): |
| """Update last successful build.""" |
| last_successful_build = ndb.Key(LastSuccessfulBuild, |
| project['name'] + '-' + build_tag).get() |
| if not last_successful_build and 'last_successful_build' not in project: |
| return |
| |
| if 'last_successful_build' not in project: |
| project['last_successful_build'] = { |
| 'build_id': last_successful_build.build_id, |
| 'finish_time': last_successful_build.finish_time |
| } |
| else: |
| if last_successful_build: |
| last_successful_build.build_id = project['last_successful_build'][ |
| 'build_id'] |
| last_successful_build.finish_time = project['last_successful_build'][ |
| 'finish_time'] |
| else: |
| last_successful_build = LastSuccessfulBuild( |
| id=project['name'] + '-' + build_tag, |
| project=project['name'], |
| build_id=project['last_successful_build']['build_id'], |
| finish_time=project['last_successful_build']['finish_time']) |
| last_successful_build.put() |
| |
| |
| # pylint: disable=no-member |
| def get_build_history(build_ids): |
| """Returns build object for the last finished build of project.""" |
| credentials, image_project = google.auth.default() |
| cloudbuild = build('cloudbuild', |
| 'v1', |
| credentials=credentials, |
| cache_discovery=False) |
| |
| history = [] |
| last_successful_build = None |
| |
| for build_id in reversed(build_ids): |
| project_build = get_build(cloudbuild, image_project, build_id) |
| if project_build['status'] not in ('SUCCESS', 'FAILURE', 'TIMEOUT'): |
| continue |
| |
| if (not last_successful_build and is_build_successful(project_build)): |
| last_successful_build = { |
| 'build_id': build_id, |
| 'finish_time': project_build['finishTime'], |
| } |
| |
| if not upload_log(build_id): |
| log_name = 'log-{0}'.format(build_id) |
| raise MissingBuildLogError('Missing build log file {0}'.format(log_name)) |
| |
| history.append({ |
| 'build_id': build_id, |
| 'finish_time': project_build['finishTime'], |
| 'success': is_build_successful(project_build) |
| }) |
| |
| if len(history) == MAX_BUILD_LOGS: |
| break |
| |
| project = {'history': history} |
| if last_successful_build: |
| project['last_successful_build'] = last_successful_build |
| return project |
| |
| |
| # pylint: disable=too-many-locals |
| def update_build_status(build_tag, status_filename): |
| """Update build statuses.""" |
| projects = [] |
| |
| def process_project(project_build): |
| """Process a project.""" |
| project = get_build_history(project_build.build_ids) |
| project['name'] = project_build.project |
| print('Processing project', project['name']) |
| return project |
| |
| with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: |
| futures = [] |
| for project_build in BuildsHistory.query( |
| BuildsHistory.build_tag == build_tag).order('project'): |
| futures.append(executor.submit(process_project, project_build)) |
| |
| for future in concurrent.futures.as_completed(futures): |
| project = future.result() |
| update_last_successful_build(project, build_tag) |
| projects.append(project) |
| |
| sort_projects(projects) |
| data = {'projects': projects} |
| upload_status(data, status_filename) |
| |
| |
| def update_build_badges(project, last_build_successful, |
| last_coverage_build_successful): |
| """Upload badges of given project.""" |
| badge = 'building' |
| # last_coverage_build_successful is False if there was an unsuccessful build |
| # and None if the target does not support coverage (e.g. Python or Java |
| # targets). |
| if last_coverage_build_successful is False: |
| badge = 'coverage_failing' |
| if not last_build_successful: |
| badge = 'failing' |
| |
| print("[badge] {}: {}".format(project, badge)) |
| |
| for extension in BADGE_IMAGE_TYPES: |
| badge_name = '{badge}.{extension}'.format(badge=badge, extension=extension) |
| |
| # Copy blob from badge_images/badge_name to badges/project/ |
| blob_name = '{badge_dir}/{badge_name}'.format(badge_dir=BADGE_DIR, |
| badge_name=badge_name) |
| |
| destination_blob_name = '{badge_dir}/{project_name}.{extension}'.format( |
| badge_dir=DESTINATION_BADGE_DIR, |
| project_name=project, |
| extension=extension) |
| |
| status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) |
| badge_blob = status_bucket.blob(blob_name) |
| status_bucket.copy_blob(badge_blob, |
| status_bucket, |
| new_name=destination_blob_name) |
| |
| |
| def upload_log(build_id): |
| """Upload log file to GCS.""" |
| status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) |
| gcb_bucket = get_storage_client().get_bucket(build_project.GCB_LOGS_BUCKET) |
| log_name = 'log-{0}.txt'.format(build_id) |
| log = gcb_bucket.blob(log_name) |
| dest_log = status_bucket.blob(log_name) |
| |
| if not log.exists(): |
| print('Failed to find build log {0}'.format(log_name), file=sys.stderr) |
| return False |
| |
| if dest_log.exists(): |
| return True |
| |
| gcb_bucket.copy_blob(log, status_bucket) |
| return True |
| |
| |
| # pylint: disable=no-member |
| def update_status(event, context): |
| """Entry point for cloud function to update build statuses and badges.""" |
| del context |
| |
| if 'data' in event: |
| status_type = base64.b64decode(event['data']).decode() |
| else: |
| raise RuntimeError('No data') |
| |
| if status_type == 'badges': |
| update_badges() |
| return |
| |
| if status_type == 'fuzzing': |
| tag = build_project.FUZZING_BUILD_TAG |
| status_filename = FUZZING_STATUS_FILENAME |
| elif status_type == 'coverage': |
| tag = build_and_run_coverage.COVERAGE_BUILD_TAG |
| status_filename = COVERAGE_STATUS_FILENAME |
| else: |
| raise RuntimeError('Invalid build status type ' + status_type) |
| |
| with ndb.Client().context(): |
| update_build_status(tag, status_filename) |
| |
| |
| def load_status_from_gcs(filename): |
| """Load statuses from bucket.""" |
| status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) |
| status = json.loads(status_bucket.blob(filename).download_as_string()) |
| result = {} |
| |
| for project in status['projects']: |
| if project['history']: |
| result[project['name']] = project['history'][0]['success'] |
| |
| return result |
| |
| |
| def update_badges(): |
| """Update badges.""" |
| project_build_statuses = load_status_from_gcs(FUZZING_STATUS_FILENAME) |
| coverage_build_statuses = load_status_from_gcs(COVERAGE_STATUS_FILENAME) |
| |
| with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: |
| futures = [] |
| with ndb.Client().context(): |
| for project in Project.query(): |
| if project.name not in project_build_statuses: |
| continue |
| # Certain projects (e.g. JVM and Python) do not have any coverage |
| # builds, but should still receive a badge. |
| coverage_build_status = None |
| if project.name in coverage_build_statuses: |
| coverage_build_status = coverage_build_statuses[project.name] |
| |
| futures.append( |
| executor.submit(update_build_badges, project.name, |
| project_build_statuses[project.name], |
| coverage_build_status)) |
| concurrent.futures.wait(futures) |