| # -*- coding: utf-8 -*- |
| # Copyright 2013 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """File and Cloud URL representation classes.""" |
| |
| from __future__ import absolute_import |
| |
| import os |
| import re |
| |
| from gslib.exception import InvalidUrlError |
| |
| # Matches provider strings of the form 'gs://' |
| PROVIDER_REGEX = re.compile(r'(?P<provider>[^:]*)://$') |
| # Matches bucket strings of the form 'gs://bucket' |
| BUCKET_REGEX = re.compile(r'(?P<provider>[^:]*)://(?P<bucket>[^/]*)/{0,1}$') |
| # Matches object strings of the form 'gs://bucket/obj' |
| OBJECT_REGEX = re.compile( |
| r'(?P<provider>[^:]*)://(?P<bucket>[^/]*)/(?P<object>.*)') |
| # Matches versioned object strings of the form 'gs://bucket/obj#1234' |
| GS_GENERATION_REGEX = re.compile(r'(?P<object>.+)#(?P<generation>[0-9]+)$') |
| # Matches versioned object strings of the form 's3://bucket/obj#NULL' |
| S3_VERSION_REGEX = re.compile(r'(?P<object>.+)#(?P<version_id>.+)$') |
| # Matches file strings of the form 'file://dir/filename' |
| FILE_OBJECT_REGEX = re.compile(r'([^:]*://)(?P<filepath>.*)') |
| # Regex to disallow buckets violating charset or not [3..255] chars total. |
| BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$') |
| # Regex to disallow buckets with individual DNS labels longer than 63. |
| TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}') |
| # Regex to determine if a string contains any wildcards. |
| WILDCARD_REGEX = re.compile(r'[*?\[\]]') |
| |
| |
| class StorageUrl(object): |
| """Abstract base class for file and Cloud Storage URLs.""" |
| |
| def Clone(self): |
| raise NotImplementedError('Clone not overridden') |
| |
| def IsFileUrl(self): |
| raise NotImplementedError('IsFileUrl not overridden') |
| |
| def IsCloudUrl(self): |
| raise NotImplementedError('IsCloudUrl not overridden') |
| |
| def IsStream(self): |
| raise NotImplementedError('IsStream not overridden') |
| |
| def CreatePrefixUrl(self, wildcard_suffix=None): |
| """Returns a prefix of this URL that can be used for iterating. |
| |
| Args: |
| wildcard_suffix: If supplied, this wildcard suffix will be appended to the |
| prefix with a trailing slash before being returned. |
| |
| Returns: |
| A prefix of this URL that can be used for iterating. |
| |
| If this URL contains a trailing slash, it will be stripped to create the |
| prefix. This helps avoid infinite looping when prefixes are iterated, but |
| preserves other slashes so that objects with '/' in the name are handled |
| properly. |
| |
| For example, when recursively listing a bucket with the following contents: |
| gs://bucket// <-- object named slash |
| gs://bucket//one-dir-deep |
| a top-level expansion with '/' as a delimiter will result in the following |
| URL strings: |
| 'gs://bucket//' : OBJECT |
| 'gs://bucket//' : PREFIX |
| If we right-strip all slashes from the prefix entry and add a wildcard |
| suffix, we will get 'gs://bucket/*' which will produce identical results |
| (and infinitely recurse). |
| |
| Example return values: |
| ('gs://bucket/subdir/', '*') becomes 'gs://bucket/subdir/*' |
| ('gs://bucket/', '*') becomes 'gs://bucket/*' |
| ('gs://bucket/', None) becomes 'gs://bucket' |
| ('gs://bucket/subdir//', '*') becomes 'gs://bucket/subdir//*' |
| ('gs://bucket/subdir///', '**') becomes 'gs://bucket/subdir///**' |
| ('gs://bucket/subdir/', '*') where 'subdir/' is an object becomes |
| 'gs://bucket/subdir/*', but iterating on this will return 'subdir/' |
| as a BucketListingObject, so we will not recurse on it as a subdir |
| during listing. |
| """ |
| raise NotImplementedError('CreatePrefixUrl not overridden') |
| |
| @property |
| def url_string(self): |
| raise NotImplementedError('url_string not overridden') |
| |
| @property |
| def versionless_url_string(self): |
| raise NotImplementedError('versionless_url_string not overridden') |
| |
| def __eq__(self, other): |
| return isinstance(other, StorageUrl) and self.url_string == other.url_string |
| |
| def __hash__(self): |
| return hash(self.url_string) |
| |
| |
| class _FileUrl(StorageUrl): |
| """File URL class providing parsing and convenience methods. |
| |
| This class assists with usage and manipulation of an |
| (optionally wildcarded) file URL string. Depending on the string |
| contents, this class represents one or more directories or files. |
| |
| For File URLs, scheme is always file, bucket_name is always blank, |
| and object_name contains the file/directory path. |
| """ |
| |
| def __init__(self, url_string, is_stream=False): |
| self.scheme = 'file' |
| self.bucket_name = '' |
| match = FILE_OBJECT_REGEX.match(url_string) |
| if match and match.lastindex == 2: |
| self.object_name = match.group(2) |
| else: |
| self.object_name = url_string |
| self.generation = None |
| self.is_stream = is_stream |
| self.delim = os.sep |
| |
| def Clone(self): |
| return _FileUrl(self.url_string) |
| |
| def IsFileUrl(self): |
| return True |
| |
| def IsCloudUrl(self): |
| return False |
| |
| def IsStream(self): |
| return self.is_stream |
| |
| def IsDirectory(self): |
| return not self.IsStream() and os.path.isdir(self.object_name) |
| |
| def CreatePrefixUrl(self, wildcard_suffix=None): |
| return self.url_string |
| |
| @property |
| def url_string(self): |
| return '%s://%s' % (self.scheme, self.object_name) |
| |
| @property |
| def versionless_url_string(self): |
| return self.url_string |
| |
| def __str__(self): |
| return self.url_string |
| |
| |
| class _CloudUrl(StorageUrl): |
| """Cloud URL class providing parsing and convenience methods. |
| |
| This class assists with usage and manipulation of an |
| (optionally wildcarded) cloud URL string. Depending on the string |
| contents, this class represents a provider, bucket(s), or object(s). |
| |
| This class operates only on strings. No cloud storage API calls are |
| made from this class. |
| """ |
| |
| def __init__(self, url_string): |
| self.scheme = None |
| self.bucket_name = None |
| self.object_name = None |
| self.generation = None |
| self.delim = '/' |
| provider_match = PROVIDER_REGEX.match(url_string) |
| bucket_match = BUCKET_REGEX.match(url_string) |
| if provider_match: |
| self.scheme = provider_match.group('provider') |
| elif bucket_match: |
| self.scheme = bucket_match.group('provider') |
| self.bucket_name = bucket_match.group('bucket') |
| if (not ContainsWildcard(self.bucket_name) and |
| (not BUCKET_NAME_RE.match(self.bucket_name) or |
| TOO_LONG_DNS_NAME_COMP.search(self.bucket_name))): |
| raise InvalidUrlError('Invalid bucket name in URL "%s"' % url_string) |
| else: |
| object_match = OBJECT_REGEX.match(url_string) |
| if object_match: |
| self.scheme = object_match.group('provider') |
| self.bucket_name = object_match.group('bucket') |
| self.object_name = object_match.group('object') |
| if self.scheme == 'gs': |
| generation_match = GS_GENERATION_REGEX.match(self.object_name) |
| if generation_match: |
| self.object_name = generation_match.group('object') |
| self.generation = generation_match.group('generation') |
| elif self.scheme == 's3': |
| version_match = S3_VERSION_REGEX.match(self.object_name) |
| if version_match: |
| self.object_name = version_match.group('object') |
| self.generation = version_match.group('version_id') |
| else: |
| raise InvalidUrlError( |
| 'CloudUrl: URL string %s did not match URL regex' % url_string) |
| |
| def Clone(self): |
| return _CloudUrl(self.url_string) |
| |
| def IsFileUrl(self): |
| return False |
| |
| def IsCloudUrl(self): |
| return True |
| |
| def IsStream(self): |
| raise NotImplementedError('IsStream not supported on CloudUrl') |
| |
| def IsBucket(self): |
| return bool(self.bucket_name and not self.object_name) |
| |
| def IsObject(self): |
| return bool(self.bucket_name and self.object_name) |
| |
| def HasGeneration(self): |
| return bool(self.generation) |
| |
| def IsProvider(self): |
| return bool(self.scheme and not self.bucket_name) |
| |
| def CreatePrefixUrl(self, wildcard_suffix=None): |
| prefix = StripOneSlash(self.versionless_url_string) |
| if wildcard_suffix: |
| prefix = '%s/%s' % (prefix, wildcard_suffix) |
| return prefix |
| |
| @property |
| def bucket_url_string(self): |
| return '%s://%s/' % (self.scheme, self.bucket_name) |
| |
| @property |
| def url_string(self): |
| url_str = self.versionless_url_string |
| if self.HasGeneration(): |
| url_str += '#%s' % self.generation |
| return url_str |
| |
| @property |
| def versionless_url_string(self): |
| if self.IsProvider(): |
| return '%s://' % self.scheme |
| elif self.IsBucket(): |
| return self.bucket_url_string |
| return '%s://%s/%s' % (self.scheme, self.bucket_name, self.object_name) |
| |
| def __str__(self): |
| return self.url_string |
| |
| |
| def _GetSchemeFromUrlString(url_str): |
| """Returns scheme component of a URL string.""" |
| |
| end_scheme_idx = url_str.find('://') |
| if end_scheme_idx == -1: |
| # File is the default scheme. |
| return 'file' |
| else: |
| return url_str[0:end_scheme_idx].lower() |
| |
| |
| def _GetPathFromUrlString(url_str): |
| """Returns path component of a URL string.""" |
| |
| end_scheme_idx = url_str.find('://') |
| if end_scheme_idx == -1: |
| return url_str |
| else: |
| return url_str[end_scheme_idx + 3:] |
| |
| |
| def IsFileUrlString(url_str): |
| """Returns whether a string is a file URL.""" |
| |
| return _GetSchemeFromUrlString(url_str) == 'file' |
| |
| |
| def StorageUrlFromString(url_str): |
| """Static factory function for creating a StorageUrl from a string.""" |
| |
| scheme = _GetSchemeFromUrlString(url_str) |
| |
| if scheme not in ('file', 's3', 'gs'): |
| raise InvalidUrlError('Unrecognized scheme "%s"' % scheme) |
| if scheme == 'file': |
| path = _GetPathFromUrlString(url_str) |
| is_stream = (path == '-') |
| return _FileUrl(url_str, is_stream=is_stream) |
| return _CloudUrl(url_str) |
| |
| |
| def StripOneSlash(url_str): |
| if url_str and url_str.endswith('/'): |
| return url_str[:-1] |
| return url_str |
| |
| |
| def ContainsWildcard(url_string): |
| """Checks whether url_string contains a wildcard. |
| |
| Args: |
| url_string: URL string to check. |
| |
| Returns: |
| bool indicator. |
| """ |
| return bool(WILDCARD_REGEX.search(url_string)) |