| #!/usr/bin/env python |
| # |
| # Copyright 2016 - The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Base Cloud API Client. |
| |
| BasicCloudApiCliend does basic setup for a cloud API. |
| """ |
| import httplib |
| import logging |
| import socket |
| import ssl |
| |
| # pylint: disable=import-error |
| from apiclient import errors as gerrors |
| from apiclient.discovery import build |
| import apiclient.http |
| import httplib2 |
| from oauth2client import client |
| |
| from acloud import errors |
| from acloud.internal.lib import utils |
| |
| logger = logging.getLogger(__name__) |
| |
| |
| class BaseCloudApiClient(object): |
| """A class that does basic setup for a cloud API.""" |
| |
| # To be overriden by subclasses. |
| API_NAME = "" |
| API_VERSION = "v1" |
| SCOPE = "" |
| |
| # Defaults for retry. |
| RETRY_COUNT = 5 |
| RETRY_BACKOFF_FACTOR = 1.5 |
| RETRY_SLEEP_MULTIPLIER = 2 |
| RETRY_HTTP_CODES = [ |
| # 403 is to retry the "Rate Limit Exceeded" error. |
| # We could retry on a finer-grained error message later if necessary. |
| 403, |
| 500, # Internal Server Error |
| 502, # Bad Gateway |
| 503, # Service Unavailable |
| ] |
| RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error, |
| socket.error, ssl.SSLError) |
| RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, ) |
| |
| def __init__(self, oauth2_credentials): |
| """Initialize. |
| |
| Args: |
| oauth2_credentials: An oauth2client.OAuth2Credentials instance. |
| """ |
| self._service = self.InitResourceHandle(oauth2_credentials) |
| |
| @classmethod |
| def InitResourceHandle(cls, oauth2_credentials): |
| """Authenticate and initialize a Resource object. |
| |
| Authenticate http and create a Resource object with methods |
| for interacting with the service. |
| |
| Args: |
| oauth2_credentials: An oauth2client.OAuth2Credentials instance. |
| |
| Returns: |
| An apiclient.discovery.Resource object |
| """ |
| http_auth = oauth2_credentials.authorize(httplib2.Http()) |
| return utils.RetryExceptionType( |
| exception_types=cls.RETRIABLE_AUTH_ERRORS, |
| max_retries=cls.RETRY_COUNT, |
| functor=build, |
| sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER, |
| retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR, |
| serviceName=cls.API_NAME, |
| version=cls.API_VERSION, |
| # This is workaround for a known issue of some veriosn |
| # of api client. |
| # https://github.com/google/google-api-python-client/issues/435 |
| cache_discovery=False, |
| http=http_auth) |
| |
| @staticmethod |
| def _ShouldRetry(exception, retry_http_codes, |
| other_retriable_errors): |
| """Check if exception is retriable. |
| |
| Args: |
| exception: An instance of Exception. |
| retry_http_codes: a list of integers, retriable HTTP codes of |
| HttpError |
| other_retriable_errors: a tuple of error types to retry other than |
| HttpError. |
| |
| Returns: |
| Boolean, True if retriable, False otherwise. |
| """ |
| if isinstance(exception, other_retriable_errors): |
| return True |
| |
| if isinstance(exception, errors.HttpError): |
| if exception.code in retry_http_codes: |
| return True |
| else: |
| logger.debug("_ShouldRetry: Exception code %s not in %s: %s", |
| exception.code, retry_http_codes, str(exception)) |
| |
| logger.debug("_ShouldRetry: Exception %s is not one of %s: %s", |
| type(exception), |
| list(other_retriable_errors) + [errors.HttpError], |
| str(exception)) |
| return False |
| |
| @staticmethod |
| def _TranslateError(exception): |
| """Translate the exception to a desired type. |
| |
| Args: |
| exception: An instance of Exception. |
| |
| Returns: |
| gerrors.HttpError will be translated to errors.HttpError. |
| If the error code is errors.HTTP_NOT_FOUND_CODE, it will |
| be translated to errors.ResourceNotFoundError. |
| Unrecognized error type will not be translated and will |
| be returned as is. |
| """ |
| if isinstance(exception, gerrors.HttpError): |
| exception = errors.HttpError.CreateFromHttpError(exception) |
| if exception.code == errors.HTTP_NOT_FOUND_CODE: |
| exception = errors.ResourceNotFoundError( |
| exception.code, str(exception)) |
| return exception |
| |
| def ExecuteOnce(self, api): |
| """Execute an api and parse the errors. |
| |
| Args: |
| api: An apiclient.http.HttpRequest, representing the api to execute. |
| |
| Returns: |
| Execution result of the api. |
| |
| Raises: |
| errors.ResourceNotFoundError: For 404 error. |
| errors.HttpError: For other types of http error. |
| """ |
| try: |
| return api.execute() |
| except gerrors.HttpError as e: |
| raise self._TranslateError(e) |
| |
| def Execute(self, |
| api, |
| retry_http_codes=None, |
| max_retry=None, |
| sleep=None, |
| backoff_factor=None, |
| other_retriable_errors=None): |
| """Execute an api with retry. |
| |
| Call ExecuteOnce and retry on http error with given codes. |
| |
| Args: |
| api: An apiclient.http.HttpRequest, representing the api to execute: |
| retry_http_codes: A list of http codes to retry. |
| max_retry: See utils.Retry. |
| sleep: See utils.Retry. |
| backoff_factor: See utils.Retry. |
| other_retriable_errors: A tuple of error types that should be retried |
| other than errors.HttpError. |
| |
| Returns: |
| Execution result of the api. |
| |
| Raises: |
| See ExecuteOnce. |
| """ |
| retry_http_codes = (self.RETRY_HTTP_CODES |
| if retry_http_codes is None else retry_http_codes) |
| max_retry = (self.RETRY_COUNT if max_retry is None else max_retry) |
| sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep) |
| backoff_factor = (self.RETRY_BACKOFF_FACTOR |
| if backoff_factor is None else backoff_factor) |
| other_retriable_errors = (self.RETRIABLE_ERRORS |
| if other_retriable_errors is None else |
| other_retriable_errors) |
| |
| def _Handler(exc): |
| """Check if |exc| is a retriable exception. |
| |
| Args: |
| exc: An exception. |
| |
| Returns: |
| True if exc is an errors.HttpError and code exists in |retry_http_codes| |
| False otherwise. |
| """ |
| if self._ShouldRetry(exc, retry_http_codes, |
| other_retriable_errors): |
| logger.debug("Will retry error: %s", str(exc)) |
| return True |
| return False |
| |
| return utils.Retry( |
| _Handler, |
| max_retries=max_retry, |
| functor=self.ExecuteOnce, |
| sleep_multiplier=sleep, |
| retry_backoff_factor=backoff_factor, |
| api=api) |
| |
| def BatchExecuteOnce(self, requests): |
| """Execute requests in a batch. |
| |
| Args: |
| requests: A dictionary where key is request id and value |
| is an http request. |
| |
| Returns: |
| results, a dictionary in the following format |
| {request_id: (response, exception)} |
| request_ids are those from requests; response |
| is the http response for the request or None on error; |
| exception is an instance of DriverError or None if no error. |
| """ |
| results = {} |
| |
| def _CallBack(request_id, response, exception): |
| results[request_id] = (response, self._TranslateError(exception)) |
| |
| batch = apiclient.http.BatchHttpRequest() |
| for request_id, request in requests.iteritems(): |
| batch.add( |
| request=request, callback=_CallBack, request_id=request_id) |
| batch.execute() |
| return results |
| |
| def BatchExecute(self, |
| requests, |
| retry_http_codes=None, |
| max_retry=None, |
| sleep=None, |
| backoff_factor=None, |
| other_retriable_errors=None): |
| """Batch execute multiple requests with retry. |
| |
| Call BatchExecuteOnce and retry on http error with given codes. |
| |
| Args: |
| requests: A dictionary where key is request id picked by caller, |
| and value is a apiclient.http.HttpRequest. |
| retry_http_codes: A list of http codes to retry. |
| max_retry: See utils.Retry. |
| sleep: See utils.Retry. |
| backoff_factor: See utils.Retry. |
| other_retriable_errors: A tuple of error types that should be retried |
| other than errors.HttpError. |
| |
| Returns: |
| results, a dictionary in the following format |
| {request_id: (response, exception)} |
| request_ids are those from requests; response |
| is the http response for the request or None on error; |
| exception is an instance of DriverError or None if no error. |
| """ |
| executor = utils.BatchHttpRequestExecutor( |
| self.BatchExecuteOnce, |
| requests=requests, |
| retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES, |
| max_retry=max_retry or self.RETRY_COUNT, |
| sleep=sleep or self.RETRY_SLEEP_MULTIPLIER, |
| backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR, |
| other_retriable_errors=other_retriable_errors |
| or self.RETRIABLE_ERRORS) |
| executor.Execute() |
| return executor.GetResults() |
| |
| def ListWithMultiPages(self, api_resource, *args, **kwargs): |
| """Call an api that list a type of resource. |
| |
| Multiple google services support listing a type of |
| resource (e.g list gce instances, list storage objects). |
| The querying pattern is similar -- |
| Step 1: execute the api and get a response object like, |
| { |
| "items": [..list of resource..], |
| # The continuation token that can be used |
| # to get the next page. |
| "nextPageToken": "A String", |
| } |
| Step 2: execute the api again with the nextPageToken to |
| retrieve more pages and get a response object. |
| |
| Step 3: Repeat Step 2 until no more page. |
| |
| This method encapsulates the generic logic of |
| calling such listing api. |
| |
| Args: |
| api_resource: An apiclient.discovery.Resource object |
| used to create an http request for the listing api. |
| *args: Arguments used to create the http request. |
| **kwargs: Keyword based arguments to create the http |
| request. |
| |
| Returns: |
| A list of items. |
| """ |
| items = [] |
| next_page_token = None |
| while True: |
| api = api_resource(pageToken=next_page_token, *args, **kwargs) |
| response = self.Execute(api) |
| items.extend(response.get("items", [])) |
| next_page_token = response.get("nextPageToken") |
| if not next_page_token: |
| break |
| return items |
| |
| @property |
| def service(self): |
| """Return self._service as a property.""" |
| return self._service |
