| # -*- coding: utf-8 -*- |
| # Copyright 2013 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Gsutil API for interacting with cloud storage providers.""" |
| |
| from __future__ import absolute_import |
| |
| |
| class CloudApi(object): |
| """Abstract base class for interacting with cloud storage providers. |
| |
| Implementations of the gsutil Cloud API are not guaranteed to be thread-safe. |
| Behavior when calling a gsutil Cloud API instance simultaneously across |
| threads is undefined and doing so will likely cause errors. Therefore, |
| a separate instance of the gsutil Cloud API should be instantiated per-thread. |
| """ |
| |
| def __init__(self, bucket_storage_uri_class, logger, provider=None, |
| debug=0, trace_token=None, perf_trace_token=None): |
| """Performs necessary setup for interacting with the cloud storage provider. |
| |
| Args: |
| bucket_storage_uri_class: boto storage_uri class, used by APIs that |
| provide boto translation or mocking. |
| logger: logging.logger for outputting log messages. |
| provider: Default provider prefix describing cloud storage provider to |
| connect to. |
| debug: Debug level for the API implementation (0..3). |
| trace_token: Google internal trace token to pass to the API |
| implementation (string). |
| perf_trace_token: Performance trace token to use when making API calls. |
| """ |
| self.bucket_storage_uri_class = bucket_storage_uri_class |
| self.logger = logger |
| self.provider = provider |
| self.debug = debug |
| self.trace_token = trace_token |
| self.perf_trace_token = perf_trace_token |
| |
| def GetBucket(self, bucket_name, provider=None, fields=None): |
| """Gets Bucket metadata. |
| |
| Args: |
| bucket_name: Name of the bucket. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Bucket metadata fields, for |
| example, ['logging', 'defaultObjectAcl'] |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Bucket object. |
| """ |
| raise NotImplementedError('GetBucket must be overloaded') |
| |
| def ListBuckets(self, project_id=None, provider=None, fields=None): |
| """Lists bucket metadata for the given project. |
| |
| Args: |
| project_id: Project owning the buckets, default from config if None. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these metadata fields for the listing, |
| for example: |
| ['items/logging', 'items/defaultObjectAcl']. |
| Note that the WildcardIterator class should be used to list |
| buckets instead of calling this function directly. It amends |
| the fields definition from get-like syntax such as |
| ['logging', 'defaultObjectAcl'] so that the caller does not |
| need to prepend 'items/' or specify fields necessary for listing |
| (like nextPageToken). |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Iterator over Bucket objects. |
| """ |
| raise NotImplementedError('ListBuckets must be overloaded') |
| |
| def PatchBucket(self, bucket_name, metadata, canned_acl=None, |
| canned_def_acl=None, preconditions=None, provider=None, |
| fields=None): |
| """Updates bucket metadata for the bucket with patch semantics. |
| |
| Args: |
| bucket_name: Name of bucket to update. |
| metadata: Bucket object defining metadata to be updated. |
| canned_acl: Canned ACL to apply to the bucket. |
| canned_def_acl: Canned default object ACL to apply to the bucket. |
| preconditions: Preconditions for the request. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Bucket metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Bucket object describing new bucket metadata. |
| """ |
| raise NotImplementedError('PatchBucket must be overloaded') |
| |
| def CreateBucket(self, bucket_name, project_id=None, metadata=None, |
| provider=None, fields=None): |
| """Creates a new bucket with the specified metadata. |
| |
| Args: |
| bucket_name: Name of the new bucket. |
| project_id: Project owner of the new bucket, default from config if None. |
| metadata: Bucket object defining new bucket metadata. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Bucket metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Bucket object describing new bucket metadata. |
| """ |
| raise NotImplementedError('CreateBucket must be overloaded') |
| |
| def DeleteBucket(self, bucket_name, preconditions=None, provider=None): |
| """Deletes a bucket. |
| |
| Args: |
| bucket_name: Name of the bucket to delete. |
| preconditions: Preconditions for the request. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| None. |
| """ |
| raise NotImplementedError('DeleteBucket must be overloaded') |
| |
| class CsObjectOrPrefixType(object): |
| """Enum class for describing CsObjectOrPrefix types.""" |
| OBJECT = 'object' # Cloud object |
| PREFIX = 'prefix' # Cloud bucket subdirectory |
| |
| class CsObjectOrPrefix(object): |
| """Container class for ListObjects results.""" |
| |
| def __init__(self, data, datatype): |
| """Stores a ListObjects result. |
| |
| Args: |
| data: Root object, either an apitools Object or a string Prefix. |
| datatype: CsObjectOrPrefixType of data. |
| """ |
| self.data = data |
| self.datatype = datatype |
| |
| def ListObjects(self, bucket_name, prefix=None, delimiter=None, |
| all_versions=None, provider=None, fields=None): |
| """Lists objects (with metadata) and prefixes in a bucket. |
| |
| Args: |
| bucket_name: Bucket containing the objects. |
| prefix: Prefix for directory-like behavior. |
| delimiter: Delimiter for directory-like behavior. |
| all_versions: If true, list all object versions. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these metadata fields for the listing, |
| for example: |
| ['items/acl', 'items/updated', 'prefixes']. |
| Note that the WildcardIterator class should be used to list |
| objects instead of calling this function directly. It amends |
| the fields definition from get-like syntax such as |
| ['acl', 'updated'] so that the caller does not need to |
| prepend 'items/' or specify any fields necessary for listing |
| (such as prefixes or nextPageToken). |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Iterator over CsObjectOrPrefix wrapper class. |
| """ |
| raise NotImplementedError('ListObjects must be overloaded') |
| |
| def GetObjectMetadata(self, bucket_name, object_name, generation=None, |
| provider=None, fields=None): |
| """Gets object metadata. |
| |
| If decryption is supported by the implementing class, this function will |
| read decryption keys from configuration and appropriately retry requests to |
| encrypted objects with the correct key. |
| |
| Args: |
| bucket_name: Bucket containing the object. |
| object_name: Object name. |
| generation: Generation of the object to retrieve. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields, for |
| example, ['acl', 'updated']. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Object object. |
| """ |
| raise NotImplementedError('GetObjectMetadata must be overloaded') |
| |
| def PatchObjectMetadata(self, bucket_name, object_name, metadata, |
| canned_acl=None, generation=None, preconditions=None, |
| provider=None, fields=None): |
| """Updates object metadata with patch semantics. |
| |
| Args: |
| bucket_name: Bucket containing the object. |
| object_name: Object name for object. |
| metadata: Object object defining metadata to be updated. |
| canned_acl: Canned ACL to be set on the object. |
| generation: Generation (or version) of the object to update. |
| preconditions: Preconditions for the request. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Updated object metadata. |
| """ |
| raise NotImplementedError('PatchObjectMetadata must be overloaded') |
| |
| class DownloadStrategy(object): |
| """Enum class for specifying download strategy.""" |
| ONE_SHOT = 'oneshot' |
| RESUMABLE = 'resumable' |
| |
| def GetObjectMedia(self, bucket_name, object_name, download_stream, |
| provider=None, generation=None, object_size=None, |
| compressed_encoding=False, |
| download_strategy=DownloadStrategy.ONE_SHOT, start_byte=0, |
| end_byte=None, progress_callback=None, |
| serialization_data=None, digesters=None, |
| decryption_tuple=None): |
| """Gets object data. |
| |
| Args: |
| bucket_name: Bucket containing the object. |
| object_name: Object name. |
| download_stream: Stream to send the object data to. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| generation: Generation of the object to retrieve. |
| object_size: Total size of the object being downloaded. |
| compressed_encoding: If true, object is stored with a compressed encoding. |
| download_strategy: Cloud API download strategy to use for download. |
| start_byte: Starting point for download (for resumable downloads and |
| range requests). Can be set to negative to request a range |
| of bytes (python equivalent of [:-3]) |
| end_byte: Ending byte number, inclusive, for download (for range |
| requests). If None, download the rest of the object. |
| progress_callback: Optional callback function for progress notifications. |
| Receives calls with arguments |
| (bytes_transferred, total_size). |
| serialization_data: Implementation-specific JSON string of a dict |
| containing serialization information for the download. |
| digesters: Dict of {string : digester}, where string is a name of a hash |
| algorithm, and digester is a validation digester that supports |
| update(bytes) and digest() using that algorithm. |
| Implementation can set the digester value to None to indicate |
| bytes were not successfully digested on-the-fly. |
| decryption_tuple: Optional CryptoTuple for decrypting an encrypted |
| object. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Content-encoding string if it was detected that the server sent an encoded |
| object during transfer, None otherwise. |
| """ |
| raise NotImplementedError('GetObjectMedia must be overloaded') |
| |
| def UploadObject(self, upload_stream, object_metadata, canned_acl=None, |
| size=None, preconditions=None, progress_callback=None, |
| encryption_tuple=None, provider=None, fields=None): |
| """Uploads object data and metadata. |
| |
| Args: |
| upload_stream: Seekable stream of object data. |
| object_metadata: Object metadata for new object. Must include bucket |
| and object name. |
| canned_acl: Optional canned ACL to apply to object. Overrides ACL set |
| in object_metadata. |
| size: Optional object size. |
| preconditions: Preconditions for the request. |
| progress_callback: Optional callback function for progress notifications. |
| Receives calls with arguments |
| (bytes_transferred, total_size). |
| encryption_tuple: Optional CryptoTuple for encrypting the uploaded |
| object. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Object object for newly created destination object. |
| """ |
| raise NotImplementedError('UploadObject must be overloaded') |
| |
| def UploadObjectStreaming(self, upload_stream, object_metadata, |
| canned_acl=None, preconditions=None, |
| progress_callback=None, encryption_tuple=None, |
| provider=None, fields=None): |
| """Uploads object data and metadata. |
| |
| Args: |
| upload_stream: Stream of object data. May not be seekable. |
| object_metadata: Object metadata for new object. Must include bucket |
| and object name. |
| canned_acl: Optional canned ACL to apply to object. Overrides ACL set |
| in object_metadata. |
| preconditions: Preconditions for the request. |
| progress_callback: Optional callback function for progress notifications. |
| Receives calls with arguments |
| (bytes_transferred, total_size), but fills in only |
| bytes_transferred. |
| encryption_tuple: Optional CryptoTuple for encrypting the uploaded |
| object. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Object object for newly created destination object. |
| """ |
| raise NotImplementedError('UploadObjectStreaming must be overloaded') |
| |
| def UploadObjectResumable( |
| self, upload_stream, object_metadata, canned_acl=None, |
| size=None, preconditions=None, serialization_data=None, |
| tracker_callback=None, progress_callback=None, encryption_tuple=None, |
| provider=None, fields=None): |
| """Uploads object data and metadata using a resumable upload strategy. |
| |
| Args: |
| upload_stream: Seekable stream of object data. |
| object_metadata: Object metadata for new object. Must include bucket |
| and object name. |
| canned_acl: Optional canned ACL to apply to object. Overrides ACL set |
| in object_metadata. |
| size: Total size of the object. |
| preconditions: Preconditions for the request. |
| serialization_data: Dict of {'url' : UploadURL} allowing for uploads to |
| be resumed. |
| tracker_callback: Callback function taking a upload URL string. |
| Guaranteed to be called when the implementation gets an |
| upload URL, allowing the caller to resume the upload |
| across process breaks by saving the upload URL in |
| a tracker file. |
| progress_callback: Optional callback function for progress notifications. |
| Receives calls with arguments |
| (bytes_transferred, total_size). |
| encryption_tuple: Optional CryptoTuple for encrypting the uploaded |
| object. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields when the |
| upload is complete. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Object object for newly created destination object. |
| """ |
| raise NotImplementedError('UploadObjectResumable must be overloaded') |
| |
| def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, |
| canned_acl=None, preconditions=None, progress_callback=None, |
| max_bytes_per_call=None, encryption_tuple=None, |
| decryption_tuple=None, provider=None, fields=None): |
| """Copies an object in the cloud. |
| |
| Args: |
| src_obj_metadata: Object metadata for source object. Must include |
| bucket name, object name, and etag. |
| dst_obj_metadata: Object metadata for new object. Must include bucket |
| and object name. |
| src_generation: Generation of the source object to copy. |
| canned_acl: Optional canned ACL to apply to destination object. Overrides |
| ACL set in dst_obj_metadata. |
| preconditions: Destination object preconditions for the request. |
| progress_callback: Optional callback function for progress notifications. |
| Receives calls with arguments |
| (bytes_transferred, total_size). |
| max_bytes_per_call: Integer describing maximum number of bytes |
| to rewrite per service call. |
| encryption_tuple: Optional CryptoTuple for encrypting the destination |
| object. |
| decryption_tuple: Optional CryptoTuple for decrypting the source |
| object. If supplied without encryption_tuple, destination object will |
| be written without customer-supplied encryption. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Object object for newly created destination object. |
| """ |
| raise NotImplementedError('CopyObject must be overloaded') |
| |
| def ComposeObject(self, src_objs_metadata, dst_obj_metadata, |
| preconditions=None, encryption_tuple=None, |
| provider=None, fields=None): |
| """Composes an object in the cloud. |
| |
| Args: |
| src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries |
| specifying the objects to compose. |
| dst_obj_metadata: Metadata for the destination object including bucket |
| and object name. |
| preconditions: Destination object preconditions for the request. |
| encryption_tuple: Optional CryptoTuple for decrypting source objects |
| and encrypting the destination object. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Object metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Composed object metadata. |
| """ |
| raise NotImplementedError('ComposeObject must be overloaded') |
| |
| def DeleteObject(self, bucket_name, object_name, preconditions=None, |
| generation=None, provider=None): |
| """Deletes an object. |
| |
| Args: |
| bucket_name: Name of the containing bucket. |
| object_name: Name of the object to delete. |
| preconditions: Preconditions for the request. |
| generation: Generation (or version) of the object to delete; if None, |
| deletes the live object. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| None. |
| """ |
| raise NotImplementedError('DeleteObject must be overloaded') |
| |
| def WatchBucket(self, bucket_name, address, channel_id, token=None, |
| provider=None, fields=None): |
| """Creates a notification subscription for changes to objects in a bucket. |
| |
| Args: |
| bucket_name: Bucket containing the objects. |
| address: Address to which to send notifications. |
| channel_id: Unique ID string for the channel. |
| token: If present, token string is delivered with each notification. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| fields: If present, return only these Channel metadata fields. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| Channel object describing the notification subscription. |
| """ |
| raise NotImplementedError('WatchBucket must be overloaded') |
| |
| def StopChannel(self, channel_id, resource_id, provider=None): |
| """Stops a notification channel. |
| |
| Args: |
| channel_id: Unique ID string for the channel. |
| resource_id: Version-agnostic ID string for the channel. |
| provider: Cloud storage provider to connect to. If not present, |
| class-wide default is used. |
| |
| Raises: |
| ArgumentException for errors during input validation. |
| ServiceException for errors interacting with cloud storage providers. |
| |
| Returns: |
| None. |
| """ |
| raise NotImplementedError('StopChannel must be overloaded') |
| |
| |
| class CryptoTuple(object): |
| """Class describing an encryption/decryption key for cloud API requests.""" |
| |
| def __init__(self, crypto_key): |
| """Initialize the CryptoTuple. |
| |
| Args: |
| crypto_key: Base64-encoded string of encryption key. |
| """ |
| self.crypto_key = crypto_key |
| self.crypto_alg = 'AES256' # Only currently supported encryption algorithm. |
| |
| |
| class Preconditions(object): |
| """Preconditions class for specifying preconditions to cloud API requests.""" |
| |
| def __init__(self, gen_match=None, meta_gen_match=None): |
| """Instantiates a Preconditions object. |
| |
| Args: |
| gen_match: Perform request only if generation of target object |
| matches the given integer. Ignored for bucket requests. |
| meta_gen_match: Perform request only if metageneration of target |
| object/bucket matches the given integer. |
| """ |
| self.gen_match = gen_match |
| self.meta_gen_match = meta_gen_match |
| |
| |
| class EncryptionException(Exception): |
| """Exception raised when an encrypted resource cannot be decrypted.""" |
| |
| |
| class ArgumentException(Exception): |
| """Exception raised when arguments to a Cloud API method are invalid. |
| |
| This exception is never raised as a result of a failed call to a cloud |
| storage provider. |
| """ |
| |
| def __init__(self, reason): |
| Exception.__init__(self) |
| self.reason = reason |
| |
| def __repr__(self): |
| return str(self) |
| |
| def __str__(self): |
| return '%s: %s' % (self.__class__.__name__, self.reason) |
| |
| |
| class ProjectIdException(ArgumentException): |
| """Exception raised when a Project ID argument is required but not present.""" |
| |
| |
| class ServiceException(Exception): |
| """Exception raised when a cloud storage provider request fails. |
| |
| This exception is raised only as a result of a failed remote call. |
| """ |
| |
| def __init__(self, reason, status=None, body=None): |
| Exception.__init__(self) |
| self.reason = reason |
| self.status = status |
| self.body = body |
| |
| def __repr__(self): |
| return str(self) |
| |
| def __str__(self): |
| message = '%s:' % self.__class__.__name__ |
| if self.status: |
| message += ' %s' % self.status |
| message += ' %s' % self.reason |
| if self.body: |
| message += '\n%s' % self.body |
| return message |
| |
| |
| class RetryableServiceException(ServiceException): |
| """Exception class for retryable exceptions.""" |
| |
| |
| class ResumableDownloadException(RetryableServiceException): |
| """Exception raised for res. downloads that can be retried later.""" |
| |
| |
| class ResumableUploadException(RetryableServiceException): |
| """Exception raised for res. uploads that can be retried w/ same upload ID.""" |
| |
| |
| class ResumableUploadStartOverException(RetryableServiceException): |
| """Exception raised for res. uploads that can be retried w/ new upload ID.""" |
| |
| |
| class ResumableUploadAbortException(ServiceException): |
| """Exception raised for resumable uploads that cannot be retried later.""" |
| |
| |
| class AuthenticationException(ServiceException): |
| """Exception raised for errors during the authentication process.""" |
| |
| |
| class PreconditionException(ServiceException): |
| """Exception raised for precondition failures.""" |
| |
| |
| class NotFoundException(ServiceException): |
| """Exception raised when a resource is not found (404).""" |
| |
| |
| class BucketNotFoundException(NotFoundException): |
| """Exception raised when a bucket resource is not found (404).""" |
| |
| def __init__(self, reason, bucket_name, status=None, body=None): |
| super(BucketNotFoundException, self).__init__(reason, status=status, |
| body=body) |
| self.bucket_name = bucket_name |
| |
| |
| class NotEmptyException(ServiceException): |
| """Exception raised when trying to delete a bucket is not empty.""" |
| |
| |
| class BadRequestException(ServiceException): |
| """Exception raised for malformed requests. |
| |
| Where it is possible to detect invalid arguments prior to sending them |
| to the server, an ArgumentException should be raised instead. |
| """ |
| |
| |
| class AccessDeniedException(ServiceException): |
| """Exception raised when authenticated user has insufficient access rights. |
| |
| This is raised when the authentication process succeeded but the |
| authenticated user does not have access rights to the requested resource. |
| """ |
| |
| |