| #!/usr/bin/env python |
| # |
| # Copyright 2018 - The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Gcloud setup runner.""" |
| |
| from __future__ import print_function |
| import logging |
| import os |
| import re |
| import subprocess |
| |
| from acloud import errors |
| from acloud.internal.lib import utils |
| from acloud.public import config |
| from acloud.setup import base_task_runner |
| from acloud.setup import google_sdk |
| |
| # APIs that need to be enabled for GCP project. |
| _ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com" |
| _COMPUTE_ENGINE_SERVICE = "compute.googleapis.com" |
| _GOOGLE_CLOUD_STORAGE_SERVICE = "storage-component.googleapis.com" |
| _GOOGLE_APIS = [ |
| _GOOGLE_CLOUD_STORAGE_SERVICE, _ANDROID_BUILD_SERVICE, |
| _COMPUTE_ENGINE_SERVICE |
| ] |
| _BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com" |
| _DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh") |
| _DEFAULT_SSH_KEY = "acloud_rsa" |
| _DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER, |
| _DEFAULT_SSH_KEY) |
| _DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER, |
| _DEFAULT_SSH_KEY + ".pub") |
| # Bucket naming parameters |
| _BUCKET_HEADER = "gs://" |
| _BUCKET_LENGTH_LIMIT = 63 |
| _DEFAULT_BUCKET_HEADER = "acloud" |
| _DEFAULT_BUCKET_REGION = "US" |
| _INVALID_BUCKET_NAME_END_CHARS = "_-" |
| _PROJECT_SEPARATOR = ":" |
| # Regular expression to get project/zone/bucket information. |
| _BUCKET_RE = re.compile(r"^gs://(?P<bucket>.+)/") |
| _BUCKET_REGION_RE = re.compile(r"^Location constraint:(?P<region>.+)") |
| _PROJECT_RE = re.compile(r"^project = (?P<project>.+)") |
| _ZONE_RE = re.compile(r"^zone = (?P<zone>.+)") |
| |
| logger = logging.getLogger(__name__) |
| |
| |
| def UpdateConfigFile(config_path, item, value): |
| """Update config data. |
| |
| Case A: config file contain this item. |
| In config, "project = A_project". New value is B_project |
| Set config "project = B_project". |
| Case B: config file didn't contain this item. |
| New value is B_project. |
| Setup config as "project = B_project". |
| |
| Args: |
| config_path: String, acloud config path. |
| item: String, item name in config file. EX: project, zone |
| value: String, value of item in config file. |
| |
| TODO(111574698): Refactor this to minimize writes to the config file. |
| TODO(111574698): Use proto method to update config. |
| """ |
| write_lines = [] |
| find_item = False |
| write_line = item + ": \"" + value + "\"\n" |
| if os.path.isfile(config_path): |
| with open(config_path, "r") as cfg_file: |
| for read_line in cfg_file.readlines(): |
| if read_line.startswith(item + ":"): |
| find_item = True |
| write_lines.append(write_line) |
| else: |
| write_lines.append(read_line) |
| if not find_item: |
| write_lines.append(write_line) |
| with open(config_path, "w") as cfg_file: |
| cfg_file.writelines(write_lines) |
| |
| |
| def SetupSSHKeys(config_path, private_key_path, public_key_path): |
| """Setup the pair of the ssh key for acloud.config. |
| |
| User can use the default path: "~/.ssh/acloud_rsa". |
| |
| Args: |
| config_path: String, acloud config path. |
| private_key_path: Path to the private key file. |
| e.g. ~/.ssh/acloud_rsa |
| public_key_path: Path to the public key file. |
| e.g. ~/.ssh/acloud_rsa.pub |
| """ |
| private_key_path = os.path.expanduser(private_key_path) |
| if (private_key_path == "" or public_key_path == "" |
| or private_key_path == _DEFAULT_SSH_PRIVATE_KEY): |
| utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY, |
| _DEFAULT_SSH_PUBLIC_KEY) |
| UpdateConfigFile(config_path, "ssh_private_key_path", |
| _DEFAULT_SSH_PRIVATE_KEY) |
| UpdateConfigFile(config_path, "ssh_public_key_path", |
| _DEFAULT_SSH_PUBLIC_KEY) |
| |
| |
| def _InputIsEmpty(input_string): |
| """Check input string is empty. |
| |
| Tool requests user to input client ID & client secret. |
| This basic check can detect user input is empty. |
| |
| Args: |
| input_string: String, user input string. |
| |
| Returns: |
| Boolean: True if input is empty, False otherwise. |
| """ |
| if input_string is None: |
| return True |
| if input_string == "": |
| print("Please enter a non-empty value.") |
| return True |
| return False |
| |
| |
| class GoogleSDKBins(object): |
| """Class to run tools in the Google SDK.""" |
| |
| def __init__(self, google_sdk_folder): |
| """GoogleSDKBins initialize. |
| |
| Args: |
| google_sdk_folder: String, google sdk path. |
| """ |
| self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud") |
| self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil") |
| |
| def RunGcloud(self, cmd, **kwargs): |
| """Run gcloud command. |
| |
| Args: |
| cmd: String list, command strings. |
| Ex: [config], then this function call "gcloud config". |
| **kwargs: dictionary of keyword based args to pass to func. |
| |
| Returns: |
| String, return message after execute gcloud command. |
| """ |
| return subprocess.check_output([self.gcloud_command_path] + cmd, **kwargs) |
| |
| def RunGsutil(self, cmd, **kwargs): |
| """Run gsutil command. |
| |
| Args: |
| cmd : String list, command strings. |
| Ex: [list], then this function call "gsutil list". |
| **kwargs: dictionary of keyword based args to pass to func. |
| |
| Returns: |
| String, return message after execute gsutil command. |
| """ |
| return subprocess.check_output([self.gsutil_command_path] + cmd, **kwargs) |
| |
| |
| class GcpTaskRunner(base_task_runner.BaseTaskRunner): |
| """Runner to setup google cloud user information.""" |
| |
| WELCOME_MESSAGE_TITLE = "Setup google cloud user information" |
| WELCOME_MESSAGE = ( |
| "This step will walk you through gcloud SDK installation." |
| "Then configure gcloud user information." |
| "Finally enable some gcloud API services.") |
| |
| def __init__(self, config_path): |
| """Initialize parameters. |
| |
| Load config file to get current values. |
| |
| Args: |
| config_path: String, acloud config path. |
| """ |
| # pylint: disable=invalid-name |
| config_mgr = config.AcloudConfigManager(config_path) |
| cfg = config_mgr.Load() |
| self.config_path = config_mgr.user_config_path |
| self.project = cfg.project |
| self.zone = cfg.zone |
| self.storage_bucket_name = cfg.storage_bucket_name |
| self.ssh_private_key_path = cfg.ssh_private_key_path |
| self.ssh_public_key_path = cfg.ssh_public_key_path |
| self.stable_host_image_name = cfg.stable_host_image_name |
| self.client_id = cfg.client_id |
| self.client_secret = cfg.client_secret |
| self.service_account_name = cfg.service_account_name |
| self.service_account_private_key_path = cfg.service_account_private_key_path |
| self.service_account_json_private_key_path = cfg.service_account_json_private_key_path |
| |
| def ShouldRun(self): |
| """Check if we actually need to run GCP setup. |
| |
| We'll only do the gcp setup if certain fields in the cfg are empty. |
| |
| Returns: |
| True if reqired config fields are empty, False otherwise. |
| """ |
| # We need to ensure the config has the proper auth-related fields set, |
| # so config requires just 1 of the following: |
| # 1. client id/secret |
| # 2. service account name/private key path |
| # 3. service account json private key path |
| if ((not self.client_id or not self.client_secret) |
| and (not self.service_account_name or not self.service_account_private_key_path) |
| and not self.service_account_json_private_key_path): |
| return True |
| |
| # If a project isn't set, then we need to run setup. |
| return not self.project |
| |
| def _Run(self): |
| """Run GCP setup task.""" |
| self._SetupGcloudInfo() |
| SetupSSHKeys(self.config_path, self.ssh_private_key_path, |
| self.ssh_public_key_path) |
| |
| def _SetupGcloudInfo(self): |
| """Setup Gcloud user information. |
| 1. Setup Gcloud SDK tools. |
| 2. Setup Gcloud project. |
| a. Setup Gcloud project and zone. |
| b. Setup Client ID and Client secret. |
| c. Setup Google Cloud Storage bucket. |
| 3. Enable Gcloud API services. |
| """ |
| google_sdk_init = google_sdk.GoogleSDK() |
| try: |
| google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath()) |
| self._SetupProject(google_sdk_runner) |
| self._EnableGcloudServices(google_sdk_runner) |
| self._CreateStableHostImage() |
| finally: |
| google_sdk_init.CleanUp() |
| |
| def _CreateStableHostImage(self): |
| """Create the stable host image.""" |
| # Write default stable_host_image_name with dummy value. |
| # TODO(113091773): An additional step to create the host image. |
| if not self.stable_host_image_name: |
| UpdateConfigFile(self.config_path, "stable_host_image_name", "") |
| |
| |
| def _NeedProjectSetup(self): |
| """Confirm project setup should run or not. |
| |
| If the project settings (project name and zone) are blank (either one), |
| we'll run the project setup flow. If they are set, we'll check with |
| the user if they want to update them. |
| |
| Returns: |
| Boolean: True if we need to setup the project, False otherwise. |
| """ |
| user_question = ( |
| "Your default Project/Zone settings are:\n" |
| "project:[%s]\n" |
| "zone:[%s]\n" |
| "Would you like to update them?[y/N]: \n") % (self.project, self.zone) |
| |
| if not self.project or not self.zone: |
| logger.info("Project or zone is empty. Start to run setup process.") |
| return True |
| return utils.GetUserAnswerYes(user_question) |
| |
| def _NeedClientIDSetup(self, project_changed): |
| """Confirm client setup should run or not. |
| |
| If project changed, client ID must also have to change. |
| So tool will force to run setup function. |
| If client ID or client secret is empty, tool force to run setup function. |
| If project didn't change and config hold user client ID/secret, tool |
| would skip client ID setup. |
| |
| Args: |
| project_changed: Boolean, True for project changed. |
| |
| Returns: |
| Boolean: True for run setup function. |
| """ |
| if project_changed: |
| logger.info("Your project changed. Start to run setup process.") |
| return True |
| elif not self.client_id or not self.client_secret: |
| logger.info("Client ID or client secret is empty. Start to run setup process.") |
| return True |
| logger.info("Project was unchanged and client ID didn't need to changed.") |
| return False |
| |
| def _SetupProject(self, gcloud_runner): |
| """Setup gcloud project information. |
| |
| Setup project and zone. |
| Setup client ID and client secret. |
| Setup Google Cloud Storage bucket. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| """ |
| project_changed = False |
| if self._NeedProjectSetup(): |
| project_changed = self._UpdateProject(gcloud_runner) |
| if self._NeedClientIDSetup(project_changed): |
| self._SetupClientIDSecret() |
| self._SetupStorageBucket(gcloud_runner) |
| |
| def _UpdateProject(self, gcloud_runner): |
| """Setup gcloud project name and zone name and check project changed. |
| |
| Run "gcloud init" to handle gcloud project setup. |
| Then "gcloud list" to get user settings information include "project" & "zone". |
| Record project_changed for next setup steps. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| |
| Returns: |
| project_changed: True for project settings changed. |
| """ |
| project_changed = False |
| gcloud_runner.RunGcloud(["init"]) |
| gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"]) |
| for line in gcp_config_list_out.splitlines(): |
| project_match = _PROJECT_RE.match(line) |
| if project_match: |
| project = project_match.group("project") |
| project_changed = (self.project != project) |
| self.project = project |
| continue |
| zone_match = _ZONE_RE.match(line) |
| if zone_match: |
| self.zone = zone_match.group("zone") |
| continue |
| UpdateConfigFile(self.config_path, "project", self.project) |
| UpdateConfigFile(self.config_path, "zone", self.zone) |
| return project_changed |
| |
| def _SetupClientIDSecret(self): |
| """Setup Client ID / Client Secret in config file. |
| |
| User can use input new values for Client ID and Client Secret. |
| """ |
| print("Please generate a new client ID/secret by following the instructions here:") |
| print("https://support.google.com/cloud/answer/6158849?hl=en") |
| # TODO: Create markdown readme instructions since the link isn't too helpful. |
| self.client_id = None |
| self.client_secret = None |
| while _InputIsEmpty(self.client_id): |
| self.client_id = str(raw_input("Enter Client ID: ").strip()) |
| while _InputIsEmpty(self.client_secret): |
| self.client_secret = str(raw_input("Enter Client Secret: ").strip()) |
| UpdateConfigFile(self.config_path, "client_id", self.client_id) |
| UpdateConfigFile(self.config_path, "client_secret", self.client_secret) |
| |
| def _SetupStorageBucket(self, gcloud_runner): |
| """Setup storage_bucket_name in config file. |
| |
| We handle the following cases: |
| 1. Bucket set in the config && bucket is valid. |
| - Configure the bucket. |
| 2. Bucket set in the config && bucket is invalid. |
| - Create a default acloud bucket and configure it |
| 3. Bucket is not set in the config. |
| - Create a default acloud bucket and configure it. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| """ |
| if (not self.storage_bucket_name |
| or not self._BucketIsValid(self.storage_bucket_name, gcloud_runner)): |
| self.storage_bucket_name = self._CreateDefaultBucket(gcloud_runner) |
| self._ConfigureBucket(gcloud_runner) |
| UpdateConfigFile(self.config_path, "storage_bucket_name", |
| self.storage_bucket_name) |
| logger.info("Storage bucket name set to [%s]", self.storage_bucket_name) |
| |
| def _ConfigureBucket(self, gcloud_runner): |
| """Setup write access right for Android Build service account. |
| |
| To avoid confuse user, we don't show messages for processing messages. |
| e.g. "No changes to gs://acloud-bucket/" |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| """ |
| gcloud_runner.RunGsutil([ |
| "acl", "ch", "-u", |
| "%s:W" % (_BUILD_SERVICE_ACCOUNT), |
| "%s" % (_BUCKET_HEADER + self.storage_bucket_name) |
| ], stderr=subprocess.STDOUT) |
| |
| def _BucketIsValid(self, bucket_name, gcloud_runner): |
| """Check bucket is valid or not. |
| |
| If bucket exists and region is in default region, |
| then this bucket is valid. |
| |
| Args: |
| bucket_name: String, name of storage bucket. |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| |
| Returns: |
| Boolean: True if bucket is valid, otherwise False. |
| """ |
| return (self._BucketExists(bucket_name, gcloud_runner) and |
| self._BucketInDefaultRegion(bucket_name, gcloud_runner)) |
| |
| def _CreateDefaultBucket(self, gcloud_runner): |
| """Setup bucket to default bucket name. |
| |
| Default bucket name is "acloud-{project}". |
| If default bucket exist and its region is not "US", |
| then default bucket name is changed as "acloud-{project}-us" |
| If default bucket didn't exist, tool will create it. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| |
| Returns: |
| String: string of bucket name. |
| """ |
| bucket_name = self._GenerateBucketName(self.project) |
| if (self._BucketExists(bucket_name, gcloud_runner) and |
| not self._BucketInDefaultRegion(bucket_name, gcloud_runner)): |
| bucket_name += ("-" + _DEFAULT_BUCKET_REGION.lower()) |
| if not self._BucketExists(bucket_name, gcloud_runner): |
| self._CreateBucket(bucket_name, gcloud_runner) |
| return bucket_name |
| |
| @staticmethod |
| def _GenerateBucketName(project_name): |
| """Generate GCS bucket name that meets the naming guidelines. |
| |
| Naming guidelines: https://cloud.google.com/storage/docs/naming |
| 1. Filter out organization name. |
| 2. Filter out illegal characters. |
| 3. Length limit. |
| 4. Name must end with a number or letter. |
| |
| Args: |
| project_name: String, name of project. |
| |
| Returns: |
| String: GCS bucket name compliant with naming guidelines. |
| """ |
| # Sanitize the project name by filtering out the org name (e.g. |
| # AOSP:fake_project -> fake_project) |
| if _PROJECT_SEPARATOR in project_name: |
| _, project_name = project_name.split(_PROJECT_SEPARATOR) |
| |
| bucket_name = "%s-%s" % (_DEFAULT_BUCKET_HEADER, project_name) |
| |
| # Rule 1: A bucket name can contain lowercase alphanumeric characters, |
| # hyphens, and underscores. |
| bucket_name = re.sub("[^a-zA-Z_/-]+", "", bucket_name).lower() |
| |
| # Rule 2: Bucket names must limit to 63 characters. |
| if len(bucket_name) > _BUCKET_LENGTH_LIMIT: |
| bucket_name = bucket_name[:_BUCKET_LENGTH_LIMIT] |
| |
| # Rule 3: Bucket names must end with a letter, strip out any ending |
| # "-" or "_" at the end of the name. |
| bucket_name = bucket_name.rstrip(_INVALID_BUCKET_NAME_END_CHARS) |
| |
| return bucket_name |
| |
| @staticmethod |
| def _BucketExists(bucket_name, gcloud_runner): |
| """Confirm bucket exist in project or not. |
| |
| Args: |
| bucket_name: String, name of storage bucket. |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| |
| Returns: |
| Boolean: True for bucket exist in project. |
| """ |
| output = gcloud_runner.RunGsutil(["list"]) |
| for output_line in output.splitlines(): |
| match = _BUCKET_RE.match(output_line) |
| if match.group("bucket") == bucket_name: |
| return True |
| return False |
| |
| @staticmethod |
| def _BucketInDefaultRegion(bucket_name, gcloud_runner): |
| """Confirm bucket region settings is "US" or not. |
| |
| Args: |
| bucket_name: String, name of storage bucket. |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| |
| Returns: |
| Boolean: True for bucket region is in default region. |
| |
| Raises: |
| errors.SetupError: For parsing bucket region information error. |
| """ |
| output = gcloud_runner.RunGsutil( |
| ["ls", "-L", "-b", "%s" % (_BUCKET_HEADER + bucket_name)]) |
| for region_line in output.splitlines(): |
| region_match = _BUCKET_REGION_RE.match(region_line.strip()) |
| if region_match: |
| region = region_match.group("region").strip() |
| logger.info("Bucket[%s] is in %s (checking for %s)", bucket_name, |
| region, _DEFAULT_BUCKET_REGION) |
| if region == _DEFAULT_BUCKET_REGION: |
| return True |
| return False |
| raise errors.ParseBucketRegionError("Could not determine bucket region.") |
| |
| @staticmethod |
| def _CreateBucket(bucket_name, gcloud_runner): |
| """Create new storage bucket in project. |
| |
| Args: |
| bucket_name: String, name of storage bucket. |
| gcloud_runner: A GcloudRunner class to run "gsutil" command. |
| """ |
| gcloud_runner.RunGsutil(["mb", "%s" % (_BUCKET_HEADER + bucket_name)]) |
| logger.info("Create bucket [%s].", bucket_name) |
| |
| @staticmethod |
| def _EnableGcloudServices(gcloud_runner): |
| """Enable 3 Gcloud API services. |
| |
| 1. Android build service |
| 2. Compute engine service |
| 3. Google cloud storage service |
| To avoid confuse user, we don't show messages for services processing |
| messages. e.g. "Waiting for async operation operations ...." |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| """ |
| for service in _GOOGLE_APIS: |
| gcloud_runner.RunGcloud(["services", "enable", service], |
| stderr=subprocess.STDOUT) |