| # -*- coding: utf-8 -*- |
| # Copyright 2014 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Utility module for translating XML API objects to/from JSON objects.""" |
| |
| from __future__ import absolute_import |
| |
| import datetime |
| import json |
| import re |
| import textwrap |
| import xml.etree.ElementTree |
| |
| from apitools.base.py import encoding |
| import boto |
| from boto.gs.acl import ACL |
| from boto.gs.acl import ALL_AUTHENTICATED_USERS |
| from boto.gs.acl import ALL_USERS |
| from boto.gs.acl import Entries |
| from boto.gs.acl import Entry |
| from boto.gs.acl import GROUP_BY_DOMAIN |
| from boto.gs.acl import GROUP_BY_EMAIL |
| from boto.gs.acl import GROUP_BY_ID |
| from boto.gs.acl import USER_BY_EMAIL |
| from boto.gs.acl import USER_BY_ID |
| |
| from gslib.cloud_api import ArgumentException |
| from gslib.cloud_api import BucketNotFoundException |
| from gslib.cloud_api import NotFoundException |
| from gslib.cloud_api import Preconditions |
| from gslib.exception import CommandException |
| from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages |
| |
| # In Python 2.6, ElementTree raises ExpatError instead of ParseError. |
| # pylint: disable=g-import-not-at-top |
| try: |
| from xml.etree.ElementTree import ParseError as XmlParseError |
| except ImportError: |
| from xml.parsers.expat import ExpatError as XmlParseError |
| |
| CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I) |
| CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I) |
| CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I) |
| CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I) |
| CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I) |
| CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I) |
| GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I) |
| GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I) |
| GOOG_METAGENERATION_MATCH_REGEX = re.compile( |
| r'^x-goog-if-metageneration-match', re.I) |
| CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.*)', |
| re.I) |
| CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.*)', re.I) |
| CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I) |
| |
| # gsutil-specific GUIDs for marking special metadata for S3 compatibility. |
| S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker' |
| S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker' |
| S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID] |
| # This distinguishes S3 custom headers from S3 metadata on objects. |
| S3_HEADER_PREFIX = 'custom-amz-header' |
| |
| DEFAULT_CONTENT_TYPE = 'application/octet-stream' |
| |
| # Because CORS is just a list in apitools, we need special handling or blank |
| # CORS lists will get sent with other configuration commands such as lifecycle, |
| # which would cause CORS configuration to be unintentionally removed. |
| # Protorpc defaults list values to an empty list, and won't allow us to set the |
| # value to None like other configuration fields, so there is no way to |
| # distinguish the default value from when we actually want to remove the CORS |
| # configuration. To work around this, we create a dummy CORS entry that |
| # signifies that we should nullify the CORS configuration. |
| # A value of [] means don't modify the CORS configuration. |
| # A value of REMOVE_CORS_CONFIG means remove the CORS configuration. |
| REMOVE_CORS_CONFIG = [apitools_messages.Bucket.CorsValueListEntry( |
| maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG'])] |
| |
| # Similar to CORS above, we need a sentinel value allowing us to specify |
| # when a default object ACL should be private (containing no entries). |
| # A defaultObjectAcl value of [] means don't modify the default object ACL. |
| # A value of [PRIVATE_DEFAULT_OBJ_ACL] means create an empty/private default |
| # object ACL. |
| PRIVATE_DEFAULT_OBJ_ACL = apitools_messages.ObjectAccessControl( |
| id='PRIVATE_DEFAULT_OBJ_ACL') |
| |
| |
| def ObjectMetadataFromHeaders(headers): |
| """Creates object metadata according to the provided headers. |
| |
| gsutil -h allows specifiying various headers (originally intended |
| to be passed to boto in gsutil v3). For the JSON API to be compatible with |
| this option, we need to parse these headers into gsutil_api Object fields. |
| |
| Args: |
| headers: Dict of headers passed via gsutil -h |
| |
| Raises: |
| ArgumentException if an invalid header is encountered. |
| |
| Returns: |
| apitools Object with relevant fields populated from headers. |
| """ |
| obj_metadata = apitools_messages.Object() |
| for header, value in headers.items(): |
| if CACHE_CONTROL_REGEX.match(header): |
| obj_metadata.cacheControl = value.strip() |
| elif CONTENT_DISPOSITION_REGEX.match(header): |
| obj_metadata.contentDisposition = value.strip() |
| elif CONTENT_ENCODING_REGEX.match(header): |
| obj_metadata.contentEncoding = value.strip() |
| elif CONTENT_MD5_REGEX.match(header): |
| obj_metadata.md5Hash = value.strip() |
| elif CONTENT_LANGUAGE_REGEX.match(header): |
| obj_metadata.contentLanguage = value.strip() |
| elif CONTENT_TYPE_REGEX.match(header): |
| if not value: |
| obj_metadata.contentType = DEFAULT_CONTENT_TYPE |
| else: |
| obj_metadata.contentType = value.strip() |
| elif GOOG_API_VERSION_REGEX.match(header): |
| # API version is only relevant for XML, ignore and rely on the XML API |
| # to add the appropriate version. |
| continue |
| elif GOOG_GENERATION_MATCH_REGEX.match(header): |
| # Preconditions are handled elsewhere, but allow these headers through. |
| continue |
| elif GOOG_METAGENERATION_MATCH_REGEX.match(header): |
| # Preconditions are handled elsewhere, but allow these headers through. |
| continue |
| else: |
| custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header) |
| custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header) |
| custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header) |
| header_key = None |
| if custom_goog_metadata_match: |
| header_key = custom_goog_metadata_match.group('header_key') |
| elif custom_amz_metadata_match: |
| header_key = custom_amz_metadata_match.group('header_key') |
| elif custom_amz_header_match: |
| # If we got here we are guaranteed by the prior statement that this is |
| # not an x-amz-meta- header. |
| header_key = (S3_HEADER_PREFIX + |
| custom_amz_header_match.group('header_key')) |
| if header_key: |
| if header_key.lower() == 'x-goog-content-language': |
| # Work around content-language being inserted into custom metadata. |
| continue |
| if not obj_metadata.metadata: |
| obj_metadata.metadata = apitools_messages.Object.MetadataValue() |
| if not obj_metadata.metadata.additionalProperties: |
| obj_metadata.metadata.additionalProperties = [] |
| obj_metadata.metadata.additionalProperties.append( |
| apitools_messages.Object.MetadataValue.AdditionalProperty( |
| key=header_key, value=value)) |
| else: |
| raise ArgumentException( |
| 'Invalid header specifed: %s:%s' % (header, value)) |
| return obj_metadata |
| |
| |
| def HeadersFromObjectMetadata(dst_obj_metadata, provider): |
| """Creates a header dictionary based on existing object metadata. |
| |
| Args: |
| dst_obj_metadata: Object metadata to create the headers from. |
| provider: Provider string ('gs' or 's3') |
| |
| Returns: |
| Headers dictionary. |
| """ |
| headers = {} |
| if not dst_obj_metadata: |
| return |
| # Metadata values of '' mean suppress/remove this header. |
| if dst_obj_metadata.cacheControl is not None: |
| if not dst_obj_metadata.cacheControl: |
| headers['cache-control'] = None |
| else: |
| headers['cache-control'] = dst_obj_metadata.cacheControl.strip() |
| if dst_obj_metadata.contentDisposition: |
| if not dst_obj_metadata.contentDisposition: |
| headers['content-disposition'] = None |
| else: |
| headers['content-disposition'] = ( |
| dst_obj_metadata.contentDisposition.strip()) |
| if dst_obj_metadata.contentEncoding: |
| if not dst_obj_metadata.contentEncoding: |
| headers['content-encoding'] = None |
| else: |
| headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip() |
| if dst_obj_metadata.contentLanguage: |
| if not dst_obj_metadata.contentLanguage: |
| headers['content-language'] = None |
| else: |
| headers['content-language'] = dst_obj_metadata.contentLanguage.strip() |
| if dst_obj_metadata.md5Hash: |
| if not dst_obj_metadata.md5Hash: |
| headers['Content-MD5'] = None |
| else: |
| headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip() |
| if dst_obj_metadata.contentType is not None: |
| if not dst_obj_metadata.contentType: |
| headers['content-type'] = None |
| else: |
| headers['content-type'] = dst_obj_metadata.contentType.strip() |
| if (dst_obj_metadata.metadata and |
| dst_obj_metadata.metadata.additionalProperties): |
| for additional_property in dst_obj_metadata.metadata.additionalProperties: |
| # Work around content-language being inserted into custom metadata by |
| # the XML API. |
| if additional_property.key == 'content-language': |
| continue |
| # Don't translate special metadata markers. |
| if additional_property.key in S3_MARKER_GUIDS: |
| continue |
| if provider == 'gs': |
| header_name = 'x-goog-meta-' + additional_property.key |
| elif provider == 's3': |
| if additional_property.key.startswith(S3_HEADER_PREFIX): |
| header_name = ('x-amz-' + |
| additional_property.key[len(S3_HEADER_PREFIX):]) |
| else: |
| header_name = 'x-amz-meta-' + additional_property.key |
| else: |
| raise ArgumentException('Invalid provider specified: %s' % provider) |
| if (additional_property.value is not None and |
| not additional_property.value): |
| headers[header_name] = None |
| else: |
| headers[header_name] = additional_property.value |
| return headers |
| |
| |
| def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False): |
| """Copies metadata from src_obj_metadata to dst_obj_metadata. |
| |
| Args: |
| src_obj_metadata: Metadata from source object |
| dst_obj_metadata: Initialized metadata for destination object |
| override: If true, will overwrite metadata in destination object. |
| If false, only writes metadata for values that don't already |
| exist. |
| """ |
| if override or not dst_obj_metadata.cacheControl: |
| dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl |
| if override or not dst_obj_metadata.contentDisposition: |
| dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition |
| if override or not dst_obj_metadata.contentEncoding: |
| dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding |
| if override or not dst_obj_metadata.contentLanguage: |
| dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage |
| if override or not dst_obj_metadata.contentType: |
| dst_obj_metadata.contentType = src_obj_metadata.contentType |
| if override or not dst_obj_metadata.md5Hash: |
| dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash |
| |
| # TODO: Apitools should ideally treat metadata like a real dictionary instead |
| # of a list of key/value pairs (with an O(N^2) lookup). In practice the |
| # number of values is typically small enough not to matter. |
| # Work around this by creating our own dictionary. |
| if (src_obj_metadata.metadata and |
| src_obj_metadata.metadata.additionalProperties): |
| if not dst_obj_metadata.metadata: |
| dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue() |
| if not dst_obj_metadata.metadata.additionalProperties: |
| dst_obj_metadata.metadata.additionalProperties = [] |
| dst_metadata_dict = {} |
| for dst_prop in dst_obj_metadata.metadata.additionalProperties: |
| dst_metadata_dict[dst_prop.key] = dst_prop.value |
| for src_prop in src_obj_metadata.metadata.additionalProperties: |
| if src_prop.key in dst_metadata_dict: |
| if override: |
| # Metadata values of '' mean suppress/remove this header. |
| if src_prop.value is not None and not src_prop.value: |
| dst_metadata_dict[src_prop.key] = None |
| else: |
| dst_metadata_dict[src_prop.key] = src_prop.value |
| else: |
| dst_metadata_dict[src_prop.key] = src_prop.value |
| # Rewrite the list with our updated dict. |
| dst_obj_metadata.metadata.additionalProperties = [] |
| for k, v in dst_metadata_dict.iteritems(): |
| dst_obj_metadata.metadata.additionalProperties.append( |
| apitools_messages.Object.MetadataValue.AdditionalProperty(key=k, |
| value=v)) |
| |
| |
| def PreconditionsFromHeaders(headers): |
| """Creates bucket or object preconditions acccording to the provided headers. |
| |
| Args: |
| headers: Dict of headers passed via gsutil -h |
| |
| Returns: |
| gsutil Cloud API Preconditions object fields populated from headers, or None |
| if no precondition headers are present. |
| """ |
| return_preconditions = Preconditions() |
| try: |
| for header, value in headers.items(): |
| if GOOG_GENERATION_MATCH_REGEX.match(header): |
| return_preconditions.gen_match = long(value) |
| if GOOG_METAGENERATION_MATCH_REGEX.match(header): |
| return_preconditions.meta_gen_match = long(value) |
| except ValueError, _: |
| raise ArgumentException('Invalid precondition header specified. ' |
| 'x-goog-if-generation-match and ' |
| 'x-goog-if-metageneration match must be specified ' |
| 'with a positive integer value.') |
| return return_preconditions |
| |
| |
| def CreateNotFoundExceptionForObjectWrite( |
| dst_provider, dst_bucket_name, src_provider=None, |
| src_bucket_name=None, src_object_name=None, src_generation=None): |
| """Creates a NotFoundException for an object upload or copy. |
| |
| This is necessary because 404s don't necessarily specify which resource |
| does not exist. |
| |
| Args: |
| dst_provider: String abbreviation of destination provider, e.g., 'gs'. |
| dst_bucket_name: Destination bucket name for the write operation. |
| src_provider: String abbreviation of source provider, i.e. 'gs', if any. |
| src_bucket_name: Source bucket name, if any (for the copy case). |
| src_object_name: Source object name, if any (for the copy case). |
| src_generation: Source object generation, if any (for the copy case). |
| |
| Returns: |
| NotFoundException with appropriate message. |
| """ |
| dst_url_string = '%s://%s' % (dst_provider, dst_bucket_name) |
| if src_bucket_name and src_object_name: |
| src_url_string = '%s://%s/%s' % (src_provider, src_bucket_name, |
| src_object_name) |
| if src_generation: |
| src_url_string += '#%s' % str(src_generation) |
| return NotFoundException( |
| 'The source object %s or the destination bucket %s does not exist.' % |
| (src_url_string, dst_url_string)) |
| |
| return NotFoundException( |
| 'The destination bucket %s does not exist or the write to the ' |
| 'destination must be restarted' % dst_url_string) |
| |
| |
| def CreateBucketNotFoundException(code, provider, bucket_name): |
| return BucketNotFoundException('%s://%s bucket does not exist.' % |
| (provider, bucket_name), bucket_name, |
| status=code) |
| |
| |
| def CreateObjectNotFoundException(code, provider, bucket_name, object_name, |
| generation=None): |
| uri_string = '%s://%s/%s' % (provider, bucket_name, object_name) |
| if generation: |
| uri_string += '#%s' % str(generation) |
| return NotFoundException('%s does not exist.' % uri_string, status=code) |
| |
| |
| def EncodeStringAsLong(string_to_convert): |
| """Encodes an ASCII string as a python long. |
| |
| This is used for modeling S3 version_id's as apitools generation. Because |
| python longs can be arbitrarily large, this works. |
| |
| Args: |
| string_to_convert: ASCII string to convert to a long. |
| |
| Returns: |
| Long that represents the input string. |
| """ |
| return long(string_to_convert.encode('hex'), 16) |
| |
| |
| def _DecodeLongAsString(long_to_convert): |
| """Decodes an encoded python long into an ASCII string. |
| |
| This is used for modeling S3 version_id's as apitools generation. |
| |
| Args: |
| long_to_convert: long to convert to ASCII string. If this is already a |
| string, it is simply returned. |
| |
| Returns: |
| String decoded from the input long. |
| """ |
| if isinstance(long_to_convert, basestring): |
| # Already converted. |
| return long_to_convert |
| return hex(long_to_convert)[2:-1].decode('hex') |
| |
| |
| def GenerationFromUrlAndString(url, generation): |
| """Decodes a generation from a StorageURL and a generation string. |
| |
| This is used to represent gs and s3 versioning. |
| |
| Args: |
| url: StorageUrl representing the object. |
| generation: Long or string representing the object's generation or |
| version. |
| |
| Returns: |
| Valid generation string for use in URLs. |
| """ |
| if url.scheme == 's3' and generation: |
| return _DecodeLongAsString(generation) |
| return generation |
| |
| |
| def CheckForXmlConfigurationAndRaise(config_type_string, json_txt): |
| """Checks a JSON parse exception for provided XML configuration.""" |
| try: |
| xml.etree.ElementTree.fromstring(str(json_txt)) |
| raise ArgumentException('\n'.join(textwrap.wrap( |
| 'XML {0} data provided; Google Cloud Storage {0} configuration ' |
| 'now uses JSON format. To convert your {0}, set the desired XML ' |
| 'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then ' |
| 'use \'gsutil {1} get ...\' with gsutil version 4 or greater to ' |
| 'get the corresponding JSON {0}.'.format(config_type_string, |
| config_type_string.lower())))) |
| except XmlParseError: |
| pass |
| raise ArgumentException('JSON %s data could not be loaded ' |
| 'from: %s' % (config_type_string, json_txt)) |
| |
| |
| class LifecycleTranslation(object): |
| """Functions for converting between various lifecycle formats. |
| |
| This class handles conversation to and from Boto Cors objects, JSON text, |
| and apitools Message objects. |
| """ |
| |
| @classmethod |
| def BotoLifecycleFromMessage(cls, lifecycle_message): |
| """Translates an apitools message to a boto lifecycle object.""" |
| boto_lifecycle = boto.gs.lifecycle.LifecycleConfig() |
| if lifecycle_message: |
| for rule_message in lifecycle_message.rule: |
| boto_rule = boto.gs.lifecycle.Rule() |
| if (rule_message.action and rule_message.action.type and |
| rule_message.action.type.lower() == 'delete'): |
| boto_rule.action = boto.gs.lifecycle.DELETE |
| if rule_message.condition: |
| if rule_message.condition.age: |
| boto_rule.conditions[boto.gs.lifecycle.AGE] = ( |
| str(rule_message.condition.age)) |
| if rule_message.condition.createdBefore: |
| boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = ( |
| str(rule_message.condition.createdBefore)) |
| if rule_message.condition.isLive: |
| boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = ( |
| str(rule_message.condition.isLive)) |
| if rule_message.condition.numNewerVersions: |
| boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = ( |
| str(rule_message.condition.numNewerVersions)) |
| boto_lifecycle.append(boto_rule) |
| return boto_lifecycle |
| |
| @classmethod |
| def BotoLifecycleToMessage(cls, boto_lifecycle): |
| """Translates a boto lifecycle object to an apitools message.""" |
| lifecycle_message = None |
| if boto_lifecycle: |
| lifecycle_message = apitools_messages.Bucket.LifecycleValue() |
| for boto_rule in boto_lifecycle: |
| lifecycle_rule = ( |
| apitools_messages.Bucket.LifecycleValue.RuleValueListEntry()) |
| lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue. |
| RuleValueListEntry.ConditionValue()) |
| if boto_rule.action and boto_rule.action == boto.gs.lifecycle.DELETE: |
| lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. |
| RuleValueListEntry.ActionValue( |
| type='Delete')) |
| if boto.gs.lifecycle.AGE in boto_rule.conditions: |
| lifecycle_rule.condition.age = int( |
| boto_rule.conditions[boto.gs.lifecycle.AGE]) |
| if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions: |
| lifecycle_rule.condition.createdBefore = ( |
| LifecycleTranslation.TranslateBotoLifecycleTimestamp( |
| boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE])) |
| if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions: |
| lifecycle_rule.condition.isLive = bool( |
| boto_rule.conditions[boto.gs.lifecycle.IS_LIVE]) |
| if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions: |
| lifecycle_rule.condition.numNewerVersions = int( |
| boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS]) |
| lifecycle_message.rule.append(lifecycle_rule) |
| return lifecycle_message |
| |
| @classmethod |
| def JsonLifecycleFromMessage(cls, lifecycle_message): |
| """Translates an apitools message to lifecycle JSON.""" |
| return str(encoding.MessageToJson(lifecycle_message)) + '\n' |
| |
| @classmethod |
| def JsonLifecycleToMessage(cls, json_txt): |
| """Translates lifecycle JSON to an apitools message.""" |
| try: |
| deserialized_lifecycle = json.loads(json_txt) |
| # If lifecycle JSON is the in the following format |
| # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key |
| # and reduce it to the following format |
| # {'rule': ... |
| if 'lifecycle' in deserialized_lifecycle: |
| deserialized_lifecycle = deserialized_lifecycle['lifecycle'] |
| lifecycle = encoding.DictToMessage( |
| deserialized_lifecycle, apitools_messages.Bucket.LifecycleValue) |
| return lifecycle |
| except ValueError: |
| CheckForXmlConfigurationAndRaise('lifecycle', json_txt) |
| |
| @classmethod |
| def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime): |
| """Parses the timestamp from the boto lifecycle into a datetime object.""" |
| return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date() |
| |
| |
| class CorsTranslation(object): |
| """Functions for converting between various CORS formats. |
| |
| This class handles conversation to and from Boto Cors objects, JSON text, |
| and apitools Message objects. |
| """ |
| |
| @classmethod |
| def BotoCorsFromMessage(cls, cors_message): |
| """Translates an apitools message to a boto Cors object.""" |
| cors = boto.gs.cors.Cors() |
| cors.cors = [] |
| for collection_message in cors_message: |
| collection_elements = [] |
| if collection_message.maxAgeSeconds: |
| collection_elements.append((boto.gs.cors.MAXAGESEC, |
| str(collection_message.maxAgeSeconds))) |
| if collection_message.method: |
| method_elements = [] |
| for method in collection_message.method: |
| method_elements.append((boto.gs.cors.METHOD, method)) |
| collection_elements.append((boto.gs.cors.METHODS, method_elements)) |
| if collection_message.origin: |
| origin_elements = [] |
| for origin in collection_message.origin: |
| origin_elements.append((boto.gs.cors.ORIGIN, origin)) |
| collection_elements.append((boto.gs.cors.ORIGINS, origin_elements)) |
| if collection_message.responseHeader: |
| header_elements = [] |
| for header in collection_message.responseHeader: |
| header_elements.append((boto.gs.cors.HEADER, header)) |
| collection_elements.append((boto.gs.cors.HEADERS, header_elements)) |
| cors.cors.append(collection_elements) |
| return cors |
| |
| @classmethod |
| def BotoCorsToMessage(cls, boto_cors): |
| """Translates a boto Cors object to an apitools message.""" |
| message_cors = [] |
| if boto_cors.cors: |
| for cors_collection in boto_cors.cors: |
| if cors_collection: |
| collection_message = apitools_messages.Bucket.CorsValueListEntry() |
| for element_tuple in cors_collection: |
| if element_tuple[0] == boto.gs.cors.MAXAGESEC: |
| collection_message.maxAgeSeconds = int(element_tuple[1]) |
| if element_tuple[0] == boto.gs.cors.METHODS: |
| for method_tuple in element_tuple[1]: |
| collection_message.method.append(method_tuple[1]) |
| if element_tuple[0] == boto.gs.cors.ORIGINS: |
| for origin_tuple in element_tuple[1]: |
| collection_message.origin.append(origin_tuple[1]) |
| if element_tuple[0] == boto.gs.cors.HEADERS: |
| for header_tuple in element_tuple[1]: |
| collection_message.responseHeader.append(header_tuple[1]) |
| message_cors.append(collection_message) |
| return message_cors |
| |
| @classmethod |
| def JsonCorsToMessageEntries(cls, json_cors): |
| """Translates CORS JSON to an apitools message. |
| |
| Args: |
| json_cors: JSON string representing CORS configuration. |
| |
| Returns: |
| List of apitools Bucket.CorsValueListEntry. An empty list represents |
| no CORS configuration. |
| """ |
| try: |
| deserialized_cors = json.loads(json_cors) |
| cors = [] |
| for cors_entry in deserialized_cors: |
| cors.append(encoding.DictToMessage( |
| cors_entry, apitools_messages.Bucket.CorsValueListEntry)) |
| return cors |
| except ValueError: |
| CheckForXmlConfigurationAndRaise('CORS', json_cors) |
| |
| @classmethod |
| def MessageEntriesToJson(cls, cors_message): |
| """Translates an apitools message to CORS JSON.""" |
| json_text = '' |
| # Because CORS is a MessageField, serialize/deserialize as JSON list. |
| json_text += '[' |
| printed_one = False |
| for cors_entry in cors_message: |
| if printed_one: |
| json_text += ',' |
| else: |
| printed_one = True |
| json_text += encoding.MessageToJson(cors_entry) |
| json_text += ']\n' |
| return json_text |
| |
| |
| def S3MarkerAclFromObjectMetadata(object_metadata): |
| """Retrieves GUID-marked S3 ACL from object metadata, if present. |
| |
| Args: |
| object_metadata: Object metadata to check. |
| |
| Returns: |
| S3 ACL text, if present, None otherwise. |
| """ |
| if (object_metadata and object_metadata.metadata and |
| object_metadata.metadata.additionalProperties): |
| for prop in object_metadata.metadata.additionalProperties: |
| if prop.key == S3_ACL_MARKER_GUID: |
| return prop.value |
| |
| |
| def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text): |
| """Adds a GUID-marked S3 ACL to the object metadata. |
| |
| Args: |
| object_metadata: Object metadata to add the acl to. |
| acl_text: S3 ACL text to add. |
| """ |
| if not object_metadata.metadata: |
| object_metadata.metadata = apitools_messages.Object.MetadataValue() |
| if not object_metadata.metadata.additionalProperties: |
| object_metadata.metadata.additionalProperties = [] |
| |
| object_metadata.metadata.additionalProperties.append( |
| apitools_messages.Object.MetadataValue.AdditionalProperty( |
| key=S3_ACL_MARKER_GUID, value=acl_text)) |
| |
| |
| class AclTranslation(object): |
| """Functions for converting between various ACL formats. |
| |
| This class handles conversion to and from Boto ACL objects, JSON text, |
| and apitools Message objects. |
| """ |
| |
| JSON_TO_XML_ROLES = {'READER': 'READ', 'WRITER': 'WRITE', |
| 'OWNER': 'FULL_CONTROL'} |
| XML_TO_JSON_ROLES = {'READ': 'READER', 'WRITE': 'WRITER', |
| 'FULL_CONTROL': 'OWNER'} |
| |
| @classmethod |
| def BotoAclFromJson(cls, acl_json): |
| acl = ACL() |
| acl.parent = None |
| acl.entries = cls.BotoEntriesFromJson(acl_json, acl) |
| return acl |
| |
| @classmethod |
| # acl_message is a list of messages, either object or bucketaccesscontrol |
| def BotoAclFromMessage(cls, acl_message): |
| acl_dicts = [] |
| for message in acl_message: |
| if message == PRIVATE_DEFAULT_OBJ_ACL: |
| # Sentinel value indicating acl_dicts should be an empty list to create |
| # a private (no entries) default object ACL. |
| break |
| acl_dicts.append(encoding.MessageToDict(message)) |
| return cls.BotoAclFromJson(acl_dicts) |
| |
| @classmethod |
| def BotoAclToJson(cls, acl): |
| if hasattr(acl, 'entries'): |
| return cls.BotoEntriesToJson(acl.entries) |
| return [] |
| |
| @classmethod |
| def BotoObjectAclToMessage(cls, acl): |
| for entry in cls.BotoAclToJson(acl): |
| message = encoding.DictToMessage(entry, |
| apitools_messages.ObjectAccessControl) |
| message.kind = u'storage#objectAccessControl' |
| yield message |
| |
| @classmethod |
| def BotoBucketAclToMessage(cls, acl): |
| for entry in cls.BotoAclToJson(acl): |
| message = encoding.DictToMessage(entry, |
| apitools_messages.BucketAccessControl) |
| message.kind = u'storage#bucketAccessControl' |
| yield message |
| |
| @classmethod |
| def BotoEntriesFromJson(cls, acl_json, parent): |
| entries = Entries(parent) |
| entries.parent = parent |
| entries.entry_list = [cls.BotoEntryFromJson(entry_json) |
| for entry_json in acl_json] |
| return entries |
| |
| @classmethod |
| def BotoEntriesToJson(cls, entries): |
| return [cls.BotoEntryToJson(entry) for entry in entries.entry_list] |
| |
| @classmethod |
| def BotoEntryFromJson(cls, entry_json): |
| """Converts a JSON entry into a Boto ACL entry.""" |
| entity = entry_json['entity'] |
| permission = cls.JSON_TO_XML_ROLES[entry_json['role']] |
| if entity.lower() == ALL_USERS.lower(): |
| return Entry(type=ALL_USERS, permission=permission) |
| elif entity.lower() == ALL_AUTHENTICATED_USERS.lower(): |
| return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission) |
| elif entity.startswith('project'): |
| raise CommandException('XML API does not support project scopes, ' |
| 'cannot translate ACL.') |
| elif 'email' in entry_json: |
| if entity.startswith('user'): |
| scope_type = USER_BY_EMAIL |
| elif entity.startswith('group'): |
| scope_type = GROUP_BY_EMAIL |
| return Entry(type=scope_type, email_address=entry_json['email'], |
| permission=permission) |
| elif 'entityId' in entry_json: |
| if entity.startswith('user'): |
| scope_type = USER_BY_ID |
| elif entity.startswith('group'): |
| scope_type = GROUP_BY_ID |
| return Entry(type=scope_type, id=entry_json['entityId'], |
| permission=permission) |
| elif 'domain' in entry_json: |
| if entity.startswith('domain'): |
| scope_type = GROUP_BY_DOMAIN |
| return Entry(type=scope_type, domain=entry_json['domain'], |
| permission=permission) |
| raise CommandException('Failed to translate JSON ACL to XML.') |
| |
| @classmethod |
| def BotoEntryToJson(cls, entry): |
| """Converts a Boto ACL entry to a valid JSON dictionary.""" |
| acl_entry_json = {} |
| # JSON API documentation uses camel case. |
| scope_type_lower = entry.scope.type.lower() |
| if scope_type_lower == ALL_USERS.lower(): |
| acl_entry_json['entity'] = 'allUsers' |
| elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower(): |
| acl_entry_json['entity'] = 'allAuthenticatedUsers' |
| elif scope_type_lower == USER_BY_EMAIL.lower(): |
| acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address |
| acl_entry_json['email'] = entry.scope.email_address |
| elif scope_type_lower == USER_BY_ID.lower(): |
| acl_entry_json['entity'] = 'user-%s' % entry.scope.id |
| acl_entry_json['entityId'] = entry.scope.id |
| elif scope_type_lower == GROUP_BY_EMAIL.lower(): |
| acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address |
| acl_entry_json['email'] = entry.scope.email_address |
| elif scope_type_lower == GROUP_BY_ID.lower(): |
| acl_entry_json['entity'] = 'group-%s' % entry.scope.id |
| acl_entry_json['entityId'] = entry.scope.id |
| elif scope_type_lower == GROUP_BY_DOMAIN.lower(): |
| acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain |
| acl_entry_json['domain'] = entry.scope.domain |
| else: |
| raise ArgumentException('ACL contains invalid scope type: %s' % |
| scope_type_lower) |
| |
| acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission] |
| return acl_entry_json |
| |
| @classmethod |
| def JsonToMessage(cls, json_data, message_type): |
| """Converts the input JSON data into list of Object/BucketAccessControls. |
| |
| Args: |
| json_data: String of JSON to convert. |
| message_type: Which type of access control entries to return, |
| either ObjectAccessControl or BucketAccessControl. |
| |
| Raises: |
| ArgumentException on invalid JSON data. |
| |
| Returns: |
| List of ObjectAccessControl or BucketAccessControl elements. |
| """ |
| try: |
| deserialized_acl = json.loads(json_data) |
| |
| acl = [] |
| for acl_entry in deserialized_acl: |
| acl.append(encoding.DictToMessage(acl_entry, message_type)) |
| return acl |
| except ValueError: |
| CheckForXmlConfigurationAndRaise('ACL', json_data) |
| |
| @classmethod |
| def JsonFromMessage(cls, acl): |
| """Strips unnecessary fields from an ACL message and returns valid JSON. |
| |
| Args: |
| acl: iterable ObjectAccessControl or BucketAccessControl |
| |
| Returns: |
| ACL JSON string. |
| """ |
| serializable_acl = [] |
| if acl is not None: |
| for acl_entry in acl: |
| if acl_entry.kind == u'storage#objectAccessControl': |
| acl_entry.object = None |
| acl_entry.generation = None |
| acl_entry.kind = None |
| acl_entry.bucket = None |
| acl_entry.id = None |
| acl_entry.selfLink = None |
| acl_entry.etag = None |
| serializable_acl.append(encoding.MessageToDict(acl_entry)) |
| return json.dumps(serializable_acl, sort_keys=True, |
| indent=2, separators=(',', ': ')) |
