| #!/usr/bin/env python |
| # |
| # Copyright 2018 - The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Gcloud setup runner.""" |
| |
| from __future__ import print_function |
| import logging |
| import os |
| import re |
| import subprocess |
| |
| import six |
| |
| from acloud import errors |
| from acloud.internal.lib import utils |
| from acloud.public import config |
| from acloud.setup import base_task_runner |
| from acloud.setup import google_sdk |
| |
| |
| logger = logging.getLogger(__name__) |
| |
| # APIs that need to be enabled for GCP project. |
| _ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com" |
| _ANDROID_BUILD_MSG = ( |
| "This service (%s) help to download images from Android Build. If it isn't " |
| "enabled, acloud only supports local images to create AVD." |
| % _ANDROID_BUILD_SERVICE) |
| _COMPUTE_ENGINE_SERVICE = "compute.googleapis.com" |
| _COMPUTE_ENGINE_MSG = ( |
| "This service (%s) help to create instance in google cloud platform. If it " |
| "isn't enabled, acloud can't work anymore." % _COMPUTE_ENGINE_SERVICE) |
| _OPEN_SERVICE_FAILED_MSG = ( |
| "\n[Open Service Failed]\n" |
| "Service name: %(service_name)s\n" |
| "%(service_msg)s\n") |
| |
| _BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com" |
| _BILLING_ENABLE_MSG = "billingEnabled: true" |
| _DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh") |
| _DEFAULT_SSH_KEY = "acloud_rsa" |
| _DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER, |
| _DEFAULT_SSH_KEY) |
| _DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER, |
| _DEFAULT_SSH_KEY + ".pub") |
| _ENV_CLOUDSDK_PYTHON = "CLOUDSDK_PYTHON" |
| _GCLOUD_COMPONENT_ALPHA = "alpha" |
| # Regular expression to get project/zone information. |
| _PROJECT_RE = re.compile(r"^project = (?P<project>.+)") |
| _ZONE_RE = re.compile(r"^zone = (?P<zone>.+)") |
| |
| |
| def UpdateConfigFile(config_path, item, value): |
| """Update config data. |
| |
| Case A: config file contain this item. |
| In config, "project = A_project". New value is B_project |
| Set config "project = B_project". |
| Case B: config file didn't contain this item. |
| New value is B_project. |
| Setup config as "project = B_project". |
| |
| Args: |
| config_path: String, acloud config path. |
| item: String, item name in config file. EX: project, zone |
| value: String, value of item in config file. |
| |
| TODO(111574698): Refactor this to minimize writes to the config file. |
| TODO(111574698): Use proto method to update config. |
| """ |
| write_lines = [] |
| find_item = False |
| write_line = item + ": \"" + value + "\"\n" |
| if os.path.isfile(config_path): |
| with open(config_path, "r") as cfg_file: |
| for read_line in cfg_file.readlines(): |
| if read_line.startswith(item + ":"): |
| find_item = True |
| write_lines.append(write_line) |
| else: |
| write_lines.append(read_line) |
| if not find_item: |
| write_lines.append(write_line) |
| with open(config_path, "w") as cfg_file: |
| cfg_file.writelines(write_lines) |
| |
| |
| def SetupSSHKeys(config_path, private_key_path, public_key_path): |
| """Setup the pair of the ssh key for acloud.config. |
| |
| User can use the default path: "~/.ssh/acloud_rsa". |
| |
| Args: |
| config_path: String, acloud config path. |
| private_key_path: Path to the private key file. |
| e.g. ~/.ssh/acloud_rsa |
| public_key_path: Path to the public key file. |
| e.g. ~/.ssh/acloud_rsa.pub |
| """ |
| private_key_path = os.path.expanduser(private_key_path) |
| if (private_key_path == "" or public_key_path == "" |
| or private_key_path == _DEFAULT_SSH_PRIVATE_KEY): |
| utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY, |
| _DEFAULT_SSH_PUBLIC_KEY) |
| UpdateConfigFile(config_path, "ssh_private_key_path", |
| _DEFAULT_SSH_PRIVATE_KEY) |
| UpdateConfigFile(config_path, "ssh_public_key_path", |
| _DEFAULT_SSH_PUBLIC_KEY) |
| |
| |
| def _InputIsEmpty(input_string): |
| """Check input string is empty. |
| |
| Tool requests user to input client ID & client secret. |
| This basic check can detect user input is empty. |
| |
| Args: |
| input_string: String, user input string. |
| |
| Returns: |
| Boolean: True if input is empty, False otherwise. |
| """ |
| if input_string is None: |
| return True |
| if input_string == "": |
| print("Please enter a non-empty value.") |
| return True |
| return False |
| |
| |
| class GoogleSDKBins(object): |
| """Class to run tools in the Google SDK.""" |
| |
| def __init__(self, google_sdk_folder): |
| """GoogleSDKBins initialize. |
| |
| Args: |
| google_sdk_folder: String, google sdk path. |
| """ |
| self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud") |
| self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil") |
| # TODO(137195528): Remove python2 environment after acloud support python3. |
| self._env = os.environ.copy() |
| self._env[_ENV_CLOUDSDK_PYTHON] = "python2" |
| |
| def RunGcloud(self, cmd, **kwargs): |
| """Run gcloud command. |
| |
| Args: |
| cmd: String list, command strings. |
| Ex: [config], then this function call "gcloud config". |
| **kwargs: dictionary of keyword based args to pass to func. |
| |
| Returns: |
| String, return message after execute gcloud command. |
| """ |
| return utils.CheckOutput([self.gcloud_command_path] + cmd, |
| env=self._env, **kwargs) |
| |
| def RunGsutil(self, cmd, **kwargs): |
| """Run gsutil command. |
| |
| Args: |
| cmd : String list, command strings. |
| Ex: [list], then this function call "gsutil list". |
| **kwargs: dictionary of keyword based args to pass to func. |
| |
| Returns: |
| String, return message after execute gsutil command. |
| """ |
| return utils.CheckOutput([self.gsutil_command_path] + cmd, |
| env=self._env, **kwargs) |
| |
| |
| class GoogleAPIService(object): |
| """Class to enable api service in the gcp project.""" |
| |
| def __init__(self, service_name, error_msg, required=False): |
| """GoogleAPIService initialize. |
| |
| Args: |
| service_name: String, name of api service. |
| error_msg: String, show messages if api service enable failed. |
| required: Boolean, True for service must be enabled for acloud. |
| """ |
| self._name = service_name |
| self._error_msg = error_msg |
| self._required = required |
| |
| def EnableService(self, gcloud_runner): |
| """Enable api service. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| """ |
| try: |
| gcloud_runner.RunGcloud(["services", "enable", self._name], |
| stderr=subprocess.STDOUT) |
| except subprocess.CalledProcessError as error: |
| self.ShowFailMessages(error.output) |
| |
| def ShowFailMessages(self, error): |
| """Show fail messages. |
| |
| Show the fail messages to hint users the impact if the api service |
| isn't enabled. |
| |
| Args: |
| error: String of error message when opening api service failed. |
| """ |
| msg_color = (utils.TextColors.FAIL if self._required else |
| utils.TextColors.WARNING) |
| utils.PrintColorString( |
| error + _OPEN_SERVICE_FAILED_MSG % { |
| "service_name": self._name, |
| "service_msg": self._error_msg} |
| , msg_color) |
| |
| @property |
| def name(self): |
| """Return name.""" |
| return self._name |
| |
| |
| class GcpTaskRunner(base_task_runner.BaseTaskRunner): |
| """Runner to setup google cloud user information.""" |
| |
| WELCOME_MESSAGE_TITLE = "Setup google cloud user information" |
| WELCOME_MESSAGE = ( |
| "This step will walk you through gcloud SDK installation." |
| "Then configure gcloud user information." |
| "Finally enable some gcloud API services.") |
| |
| def __init__(self, config_path): |
| """Initialize parameters. |
| |
| Load config file to get current values. |
| |
| Args: |
| config_path: String, acloud config path. |
| """ |
| # pylint: disable=invalid-name |
| config_mgr = config.AcloudConfigManager(config_path) |
| cfg = config_mgr.Load() |
| self.config_path = config_mgr.user_config_path |
| self.project = cfg.project |
| self.zone = cfg.zone |
| self.ssh_private_key_path = cfg.ssh_private_key_path |
| self.ssh_public_key_path = cfg.ssh_public_key_path |
| self.stable_host_image_name = cfg.stable_host_image_name |
| self.client_id = cfg.client_id |
| self.client_secret = cfg.client_secret |
| self.service_account_name = cfg.service_account_name |
| self.service_account_private_key_path = cfg.service_account_private_key_path |
| self.service_account_json_private_key_path = cfg.service_account_json_private_key_path |
| |
| def ShouldRun(self): |
| """Check if we actually need to run GCP setup. |
| |
| We'll only do the gcp setup if certain fields in the cfg are empty. |
| |
| Returns: |
| True if reqired config fields are empty, False otherwise. |
| """ |
| # We need to ensure the config has the proper auth-related fields set, |
| # so config requires just 1 of the following: |
| # 1. client id/secret |
| # 2. service account name/private key path |
| # 3. service account json private key path |
| if ((not self.client_id or not self.client_secret) |
| and (not self.service_account_name or not self.service_account_private_key_path) |
| and not self.service_account_json_private_key_path): |
| return True |
| |
| # If a project isn't set, then we need to run setup. |
| return not self.project |
| |
| def _Run(self): |
| """Run GCP setup task.""" |
| self._SetupGcloudInfo() |
| SetupSSHKeys(self.config_path, self.ssh_private_key_path, |
| self.ssh_public_key_path) |
| |
| def _SetupGcloudInfo(self): |
| """Setup Gcloud user information. |
| 1. Setup Gcloud SDK tools. |
| 2. Setup Gcloud project. |
| a. Setup Gcloud project and zone. |
| b. Setup Client ID and Client secret. |
| c. Setup Google Cloud Storage bucket. |
| 3. Enable Gcloud API services. |
| """ |
| google_sdk_init = google_sdk.GoogleSDK() |
| try: |
| google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath()) |
| google_sdk_init.InstallGcloudComponent(google_sdk_runner, |
| _GCLOUD_COMPONENT_ALPHA) |
| self._SetupProject(google_sdk_runner) |
| self._EnableGcloudServices(google_sdk_runner) |
| self._CreateStableHostImage() |
| finally: |
| google_sdk_init.CleanUp() |
| |
| def _CreateStableHostImage(self): |
| """Create the stable host image.""" |
| # Write default stable_host_image_name with unused value. |
| # TODO(113091773): An additional step to create the host image. |
| if not self.stable_host_image_name: |
| UpdateConfigFile(self.config_path, "stable_host_image_name", "") |
| |
| |
| def _NeedProjectSetup(self): |
| """Confirm project setup should run or not. |
| |
| If the project settings (project name and zone) are blank (either one), |
| we'll run the project setup flow. If they are set, we'll check with |
| the user if they want to update them. |
| |
| Returns: |
| Boolean: True if we need to setup the project, False otherwise. |
| """ |
| user_question = ( |
| "Your default Project/Zone settings are:\n" |
| "project:[%s]\n" |
| "zone:[%s]\n" |
| "Would you like to update them?[y/N]: \n") % (self.project, self.zone) |
| |
| if not self.project or not self.zone: |
| logger.info("Project or zone is empty. Start to run setup process.") |
| return True |
| return utils.GetUserAnswerYes(user_question) |
| |
| def _NeedClientIDSetup(self, project_changed): |
| """Confirm client setup should run or not. |
| |
| If project changed, client ID must also have to change. |
| So tool will force to run setup function. |
| If client ID or client secret is empty, tool force to run setup function. |
| If project didn't change and config hold user client ID/secret, tool |
| would skip client ID setup. |
| |
| Args: |
| project_changed: Boolean, True for project changed. |
| |
| Returns: |
| Boolean: True for run setup function. |
| """ |
| if project_changed: |
| logger.info("Your project changed. Start to run setup process.") |
| return True |
| elif not self.client_id or not self.client_secret: |
| logger.info("Client ID or client secret is empty. Start to run setup process.") |
| return True |
| logger.info("Project was unchanged and client ID didn't need to changed.") |
| return False |
| |
| def _SetupProject(self, gcloud_runner): |
| """Setup gcloud project information. |
| |
| Setup project and zone. |
| Setup client ID and client secret. |
| Make sure billing account enabled in project. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| """ |
| project_changed = False |
| if self._NeedProjectSetup(): |
| project_changed = self._UpdateProject(gcloud_runner) |
| if self._NeedClientIDSetup(project_changed): |
| self._SetupClientIDSecret() |
| self._CheckBillingEnable(gcloud_runner) |
| |
| def _UpdateProject(self, gcloud_runner): |
| """Setup gcloud project name and zone name and check project changed. |
| |
| Run "gcloud init" to handle gcloud project setup. |
| Then "gcloud list" to get user settings information include "project" & "zone". |
| Record project_changed for next setup steps. |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| |
| Returns: |
| project_changed: True for project settings changed. |
| """ |
| project_changed = False |
| gcloud_runner.RunGcloud(["init"]) |
| gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"]) |
| for line in gcp_config_list_out.splitlines(): |
| project_match = _PROJECT_RE.match(line) |
| if project_match: |
| project = project_match.group("project") |
| project_changed = (self.project != project) |
| self.project = project |
| continue |
| zone_match = _ZONE_RE.match(line) |
| if zone_match: |
| self.zone = zone_match.group("zone") |
| continue |
| UpdateConfigFile(self.config_path, "project", self.project) |
| UpdateConfigFile(self.config_path, "zone", self.zone) |
| return project_changed |
| |
| def _SetupClientIDSecret(self): |
| """Setup Client ID / Client Secret in config file. |
| |
| User can use input new values for Client ID and Client Secret. |
| """ |
| print("Please generate a new client ID/secret by following the instructions here:") |
| print("https://support.google.com/cloud/answer/6158849?hl=en") |
| # TODO: Create markdown readme instructions since the link isn't too helpful. |
| self.client_id = None |
| self.client_secret = None |
| while _InputIsEmpty(self.client_id): |
| self.client_id = str(six.moves.input("Enter Client ID: ").strip()) |
| while _InputIsEmpty(self.client_secret): |
| self.client_secret = str(six.moves.input("Enter Client Secret: ").strip()) |
| UpdateConfigFile(self.config_path, "client_id", self.client_id) |
| UpdateConfigFile(self.config_path, "client_secret", self.client_secret) |
| |
| def _CheckBillingEnable(self, gcloud_runner): |
| """Check billing enabled in gcp project. |
| |
| The billing info get by gcloud alpha command. Here is one example: |
| $ gcloud alpha billing projects describe project_name |
| billingAccountName: billingAccounts/011BXX-A30XXX-9XXXX |
| billingEnabled: true |
| name: projects/project_name/billingInfo |
| projectId: project_name |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| |
| Raises: |
| NoBillingError: gcp project doesn't enable billing account. |
| """ |
| billing_info = gcloud_runner.RunGcloud( |
| ["alpha", "billing", "projects", "describe", self.project]) |
| if _BILLING_ENABLE_MSG not in billing_info: |
| raise errors.NoBillingError( |
| "Please set billing account to project(%s) by following the " |
| "instructions here: " |
| "https://cloud.google.com/billing/docs/how-to/modify-project" |
| % self.project) |
| |
| @staticmethod |
| def _EnableGcloudServices(gcloud_runner): |
| """Enable 3 Gcloud API services. |
| |
| 1. Android build service |
| 2. Compute engine service |
| To avoid confuse user, we don't show messages for services processing |
| messages. e.g. "Waiting for async operation operations ...." |
| |
| Args: |
| gcloud_runner: A GcloudRunner class to run "gcloud" command. |
| """ |
| google_apis = [ |
| GoogleAPIService(_ANDROID_BUILD_SERVICE, _ANDROID_BUILD_MSG), |
| GoogleAPIService(_COMPUTE_ENGINE_SERVICE, _COMPUTE_ENGINE_MSG, required=True) |
| ] |
| enabled_services = gcloud_runner.RunGcloud( |
| ["services", "list", "--enabled", "--format", "value(NAME)"], |
| stderr=subprocess.STDOUT).splitlines() |
| |
| for service in google_apis: |
| if service.name not in enabled_services: |
| service.EnableService(gcloud_runner) |
