| # -*- coding: utf-8 -*- |
| # Copyright 2010 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Wildcard iterator class and supporting functions.""" |
| |
| from __future__ import absolute_import |
| |
| import fnmatch |
| import glob |
| import os |
| import re |
| import sys |
| import textwrap |
| |
| from gslib.bucket_listing_ref import BucketListingBucket |
| from gslib.bucket_listing_ref import BucketListingObject |
| from gslib.bucket_listing_ref import BucketListingPrefix |
| from gslib.cloud_api import AccessDeniedException |
| from gslib.cloud_api import CloudApi |
| from gslib.cloud_api import NotFoundException |
| from gslib.exception import CommandException |
| from gslib.storage_url import ContainsWildcard |
| from gslib.storage_url import StorageUrlFromString |
| from gslib.storage_url import StripOneSlash |
| from gslib.storage_url import WILDCARD_REGEX |
| from gslib.translation_helper import GenerationFromUrlAndString |
| from gslib.util import UTF8 |
| |
| |
| FLAT_LIST_REGEX = re.compile(r'(?P<before>.*?)\*\*(?P<after>.*)') |
| |
| |
| class WildcardIterator(object): |
| """Class for iterating over Google Cloud Storage strings containing wildcards. |
| |
| The base class is abstract; you should instantiate using the |
| wildcard_iterator() static factory method, which chooses the right |
| implementation depending on the base string. |
| """ |
| |
| # TODO: Standardize on __str__ and __repr__ here and elsewhere. Define both |
| # and make one return the other. |
| def __repr__(self): |
| """Returns string representation of WildcardIterator.""" |
| return 'WildcardIterator(%s)' % self.wildcard_url.url_string |
| |
| |
| class CloudWildcardIterator(WildcardIterator): |
| """WildcardIterator subclass for buckets, bucket subdirs and objects. |
| |
| Iterates over BucketListingRef matching the Url string wildcard. It's |
| much more efficient to first get metadata that's available in the Bucket |
| (for example to get the name and size of each object), because that |
| information is available in the object list results. |
| """ |
| |
| def __init__(self, wildcard_url, gsutil_api, all_versions=False, |
| debug=0, project_id=None): |
| """Instantiates an iterator that matches the wildcard URL. |
| |
| Args: |
| wildcard_url: CloudUrl that contains the wildcard to iterate. |
| gsutil_api: Cloud storage interface. Passed in for thread safety, also |
| settable for testing/mocking. |
| all_versions: If true, the iterator yields all versions of objects |
| matching the wildcard. If false, yields just the live |
| object version. |
| debug: Debug level to control debug output for iterator. |
| project_id: Project ID to use for bucket listings. |
| """ |
| self.wildcard_url = wildcard_url |
| self.all_versions = all_versions |
| self.debug = debug |
| self.gsutil_api = gsutil_api |
| self.project_id = project_id |
| |
| def __iter__(self, bucket_listing_fields=None, |
| expand_top_level_buckets=False): |
| """Iterator that gets called when iterating over the cloud wildcard. |
| |
| In the case where no wildcard is present, returns a single matching object, |
| single matching prefix, or one of each if both exist. |
| |
| Args: |
| bucket_listing_fields: Iterable fields to include in bucket listings. |
| Ex. ['name', 'acl']. Iterator is |
| responsible for converting these to list-style |
| format ['items/name', 'items/acl'] as well as |
| adding any fields necessary for listing such as |
| prefixes. API implemenation is responsible for |
| adding pagination fields. If this is None, |
| all fields are returned. |
| expand_top_level_buckets: If true, yield no BUCKET references. Instead, |
| expand buckets into top-level objects and |
| prefixes. |
| |
| Yields: |
| BucketListingRef of type BUCKET, OBJECT or PREFIX. |
| """ |
| single_version_request = self.wildcard_url.HasGeneration() |
| |
| # For wildcard expansion purposes, we need at a minimum the name of |
| # each object and prefix. If we're not using the default of requesting |
| # all fields, make sure at least these are requested. The Cloud API |
| # tolerates specifying the same field twice. |
| get_fields = None |
| if bucket_listing_fields: |
| get_fields = set() |
| for field in bucket_listing_fields: |
| get_fields.add(field) |
| bucket_listing_fields = self._GetToListFields( |
| get_fields=bucket_listing_fields) |
| bucket_listing_fields.update(['items/name', 'prefixes']) |
| get_fields.update(['name']) |
| # If we're making versioned requests, ensure generation and |
| # metageneration are also included. |
| if single_version_request or self.all_versions: |
| bucket_listing_fields.update(['items/generation', |
| 'items/metageneration']) |
| get_fields.update(['generation', 'metageneration']) |
| |
| # Handle bucket wildcarding, if any, in _ExpandBucketWildcards. Then |
| # iterate over the expanded bucket strings and handle any object |
| # wildcarding. |
| for bucket_listing_ref in self._ExpandBucketWildcards(bucket_fields=['id']): |
| bucket_url_string = bucket_listing_ref.url_string |
| if self.wildcard_url.IsBucket(): |
| # IsBucket() guarantees there are no prefix or object wildcards, and |
| # thus this is a top-level listing of buckets. |
| if expand_top_level_buckets: |
| url = StorageUrlFromString(bucket_url_string) |
| for obj_or_prefix in self.gsutil_api.ListObjects( |
| url.bucket_name, delimiter='/', all_versions=self.all_versions, |
| provider=self.wildcard_url.scheme, |
| fields=bucket_listing_fields): |
| if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: |
| yield self._GetObjectRef(bucket_url_string, obj_or_prefix.data, |
| with_version=self.all_versions) |
| else: # CloudApi.CsObjectOrPrefixType.PREFIX: |
| yield self._GetPrefixRef(bucket_url_string, obj_or_prefix.data) |
| else: |
| yield bucket_listing_ref |
| else: |
| # By default, assume a non-wildcarded URL is an object, not a prefix. |
| # This prevents unnecessary listings (which are slower, more expensive, |
| # and also subject to eventual consistency). |
| if (not ContainsWildcard(self.wildcard_url.url_string) and |
| self.wildcard_url.IsObject() and not self.all_versions): |
| try: |
| get_object = self.gsutil_api.GetObjectMetadata( |
| self.wildcard_url.bucket_name, |
| self.wildcard_url.object_name, |
| generation=self.wildcard_url.generation, |
| provider=self.wildcard_url.scheme, |
| fields=get_fields) |
| yield self._GetObjectRef( |
| self.wildcard_url.bucket_url_string, get_object, |
| with_version=(self.all_versions or single_version_request)) |
| return |
| except (NotFoundException, AccessDeniedException): |
| # It's possible this is a prefix - try to list instead. |
| pass |
| |
| # Expand iteratively by building prefix/delimiter bucket listing |
| # request, filtering the results per the current level's wildcard |
| # (if present), and continuing with the next component of the |
| # wildcard. See _BuildBucketFilterStrings() documentation for details. |
| if single_version_request: |
| url_string = '%s%s#%s' % (bucket_url_string, |
| self.wildcard_url.object_name, |
| self.wildcard_url.generation) |
| else: |
| # Rstrip any prefixes to correspond with rstripped prefix wildcard |
| # from _BuildBucketFilterStrings(). |
| url_string = '%s%s' % (bucket_url_string, |
| StripOneSlash(self.wildcard_url.object_name) |
| or '/') # Cover root object named '/' case. |
| urls_needing_expansion = [url_string] |
| while urls_needing_expansion: |
| url = StorageUrlFromString(urls_needing_expansion.pop(0)) |
| (prefix, delimiter, prefix_wildcard, suffix_wildcard) = ( |
| self._BuildBucketFilterStrings(url.object_name)) |
| prog = re.compile(fnmatch.translate(prefix_wildcard)) |
| |
| # List bucket for objects matching prefix up to delimiter. |
| for obj_or_prefix in self.gsutil_api.ListObjects( |
| url.bucket_name, prefix=prefix, delimiter=delimiter, |
| all_versions=self.all_versions or single_version_request, |
| provider=self.wildcard_url.scheme, |
| fields=bucket_listing_fields): |
| if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: |
| gcs_object = obj_or_prefix.data |
| if prog.match(gcs_object.name): |
| if not suffix_wildcard or ( |
| StripOneSlash(gcs_object.name) == suffix_wildcard): |
| if not single_version_request or ( |
| self._SingleVersionMatches(gcs_object.generation)): |
| yield self._GetObjectRef( |
| bucket_url_string, gcs_object, with_version=( |
| self.all_versions or single_version_request)) |
| else: # CloudApi.CsObjectOrPrefixType.PREFIX |
| prefix = obj_or_prefix.data |
| # If the prefix ends with a slash, remove it. Note that we only |
| # remove one slash so that we can successfully enumerate dirs |
| # containing multiple slashes. |
| rstripped_prefix = StripOneSlash(prefix) |
| if prog.match(rstripped_prefix): |
| if suffix_wildcard and rstripped_prefix != suffix_wildcard: |
| # There's more wildcard left to expand. |
| url_append_string = '%s%s' % ( |
| bucket_url_string, rstripped_prefix + '/' + |
| suffix_wildcard) |
| urls_needing_expansion.append(url_append_string) |
| else: |
| # No wildcard to expand, just yield the prefix |
| yield self._GetPrefixRef(bucket_url_string, prefix) |
| |
| def _BuildBucketFilterStrings(self, wildcard): |
| """Builds strings needed for querying a bucket and filtering results. |
| |
| This implements wildcard object name matching. |
| |
| Args: |
| wildcard: The wildcard string to match to objects. |
| |
| Returns: |
| (prefix, delimiter, prefix_wildcard, suffix_wildcard) |
| where: |
| prefix is the prefix to be sent in bucket GET request. |
| delimiter is the delimiter to be sent in bucket GET request. |
| prefix_wildcard is the wildcard to be used to filter bucket GET results. |
| suffix_wildcard is wildcard to be appended to filtered bucket GET |
| results for next wildcard expansion iteration. |
| For example, given the wildcard gs://bucket/abc/d*e/f*.txt we |
| would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and |
| suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket |
| listing request will then produce a listing result set that can be |
| filtered using this prefix_wildcard; and we'd use this suffix_wildcard |
| to feed into the next call(s) to _BuildBucketFilterStrings(), for the |
| next iteration of listing/filtering. |
| |
| Raises: |
| AssertionError if wildcard doesn't contain any wildcard chars. |
| """ |
| # Generate a request prefix if the object name part of the wildcard starts |
| # with a non-wildcard string (e.g., that's true for 'gs://bucket/abc*xyz'). |
| match = WILDCARD_REGEX.search(wildcard) |
| if not match: |
| # Input "wildcard" has no wildcard chars, so just return tuple that will |
| # cause a bucket listing to match the given input wildcard. Example: if |
| # previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc, |
| # the next iteration will call _BuildBucketFilterStrings() with |
| # gs://bucket/dir/abc, and we will return prefix ='dir/abc', |
| # delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''. |
| prefix = wildcard |
| delimiter = '/' |
| prefix_wildcard = wildcard |
| suffix_wildcard = '' |
| else: |
| if match.start() > 0: |
| # Wildcard does not occur at beginning of object name, so construct a |
| # prefix string to send to server. |
| prefix = wildcard[:match.start()] |
| wildcard_part = wildcard[match.start():] |
| else: |
| prefix = None |
| wildcard_part = wildcard |
| end = wildcard_part.find('/') |
| if end != -1: |
| wildcard_part = wildcard_part[:end+1] |
| # Remove trailing '/' so we will match gs://bucket/abc* as well as |
| # gs://bucket/abc*/ with the same wildcard regex. |
| prefix_wildcard = StripOneSlash((prefix or '') + wildcard_part) |
| suffix_wildcard = wildcard[match.end():] |
| end = suffix_wildcard.find('/') |
| if end == -1: |
| suffix_wildcard = '' |
| else: |
| suffix_wildcard = suffix_wildcard[end+1:] |
| # To implement recursive (**) wildcarding, if prefix_wildcard |
| # suffix_wildcard starts with '**' don't send a delimiter, and combine |
| # suffix_wildcard at end of prefix_wildcard. |
| if prefix_wildcard.find('**') != -1: |
| delimiter = None |
| prefix_wildcard += suffix_wildcard |
| suffix_wildcard = '' |
| else: |
| delimiter = '/' |
| # The following debug output is useful for tracing how the algorithm |
| # walks through a multi-part wildcard like gs://bucket/abc/d*e/f*.txt |
| if self.debug > 1: |
| sys.stderr.write( |
| 'DEBUG: wildcard=%s, prefix=%s, delimiter=%s, ' |
| 'prefix_wildcard=%s, suffix_wildcard=%s\n' % |
| (wildcard, prefix, delimiter, prefix_wildcard, suffix_wildcard)) |
| return (prefix, delimiter, prefix_wildcard, suffix_wildcard) |
| |
| def _SingleVersionMatches(self, listed_generation): |
| decoded_generation = GenerationFromUrlAndString(self.wildcard_url, |
| listed_generation) |
| return str(self.wildcard_url.generation) == str(decoded_generation) |
| |
| def _ExpandBucketWildcards(self, bucket_fields=None): |
| """Expands bucket and provider wildcards. |
| |
| Builds a list of bucket url strings that can be iterated on. |
| |
| Args: |
| bucket_fields: If present, populate only these metadata fields for |
| buckets. Example value: ['acl', 'defaultObjectAcl'] |
| |
| Yields: |
| BucketListingRefereneces of type BUCKET. |
| """ |
| bucket_url = StorageUrlFromString(self.wildcard_url.bucket_url_string) |
| if (bucket_fields and set(bucket_fields) == set(['id']) and |
| not ContainsWildcard(self.wildcard_url.bucket_name)): |
| # If we just want the name of a non-wildcarded bucket URL, |
| # don't make an RPC. |
| yield BucketListingBucket(bucket_url) |
| elif(self.wildcard_url.IsBucket() and |
| not ContainsWildcard(self.wildcard_url.bucket_name)): |
| # If we have a non-wildcarded bucket URL, get just that bucket. |
| yield BucketListingBucket( |
| bucket_url, root_object=self.gsutil_api.GetBucket( |
| self.wildcard_url.bucket_name, provider=self.wildcard_url.scheme, |
| fields=bucket_fields)) |
| else: |
| regex = fnmatch.translate(self.wildcard_url.bucket_name) |
| prog = re.compile(regex) |
| |
| fields = self._GetToListFields(bucket_fields) |
| if fields: |
| fields.add('items/id') |
| for bucket in self.gsutil_api.ListBuckets( |
| fields=fields, project_id=self.project_id, |
| provider=self.wildcard_url.scheme): |
| if prog.match(bucket.id): |
| url = StorageUrlFromString( |
| '%s://%s/' % (self.wildcard_url.scheme, bucket.id)) |
| yield BucketListingBucket(url, root_object=bucket) |
| |
| def _GetToListFields(self, get_fields=None): |
| """Prepends 'items/' to the input fields and converts it to a set. |
| |
| This way field sets requested for GetBucket can be used in ListBucket calls. |
| Note that the input set must contain only bucket or object fields; listing |
| fields such as prefixes or nextPageToken should be added after calling |
| this function. |
| |
| Args: |
| get_fields: Iterable fields usable in GetBucket/GetObject calls. |
| |
| Returns: |
| Set of fields usable in ListBuckets/ListObjects calls. |
| """ |
| if get_fields: |
| list_fields = set() |
| for field in get_fields: |
| list_fields.add('items/' + field) |
| return list_fields |
| |
| def _GetObjectRef(self, bucket_url_string, gcs_object, with_version=False): |
| """Creates a BucketListingRef of type OBJECT from the arguments. |
| |
| Args: |
| bucket_url_string: Wildcardless string describing the containing bucket. |
| gcs_object: gsutil_api root Object for populating the BucketListingRef. |
| with_version: If true, return a reference with a versioned string. |
| |
| Returns: |
| BucketListingRef of type OBJECT. |
| """ |
| # Generation can be None in test mocks, so just return the |
| # live object for simplicity. |
| if with_version and gcs_object.generation is not None: |
| generation_str = GenerationFromUrlAndString(self.wildcard_url, |
| gcs_object.generation) |
| object_string = '%s%s#%s' % (bucket_url_string, gcs_object.name, |
| generation_str) |
| else: |
| object_string = '%s%s' % (bucket_url_string, gcs_object.name) |
| object_url = StorageUrlFromString(object_string) |
| return BucketListingObject(object_url, root_object=gcs_object) |
| |
| def _GetPrefixRef(self, bucket_url_string, prefix): |
| """Creates a BucketListingRef of type PREFIX from the arguments. |
| |
| Args: |
| bucket_url_string: Wildcardless string describing the containing bucket. |
| prefix: gsutil_api Prefix for populating the BucketListingRef |
| |
| Returns: |
| BucketListingRef of type PREFIX. |
| """ |
| prefix_url = StorageUrlFromString('%s%s' % (bucket_url_string, prefix)) |
| return BucketListingPrefix(prefix_url, root_object=prefix) |
| |
| def IterBuckets(self, bucket_fields=None): |
| """Iterates over the wildcard, returning refs for each expanded bucket. |
| |
| This ignores the object part of the URL entirely and expands only the |
| the bucket portion. It will yield BucketListingRefs of type BUCKET only. |
| |
| Args: |
| bucket_fields: Iterable fields to include in bucket listings. |
| Ex. ['defaultObjectAcl', 'logging']. This function is |
| responsible for converting these to listing-style |
| format ['items/defaultObjectAcl', 'items/logging'], as |
| well as adding any fields necessary for listing such as |
| 'items/id'. API implemenation is responsible for |
| adding pagination fields. If this is None, all fields are |
| returned. |
| |
| Yields: |
| BucketListingRef of type BUCKET, or empty iterator if no matches. |
| """ |
| for blr in self._ExpandBucketWildcards(bucket_fields=bucket_fields): |
| yield blr |
| |
| def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False): |
| """Iterates over the wildcard, yielding bucket, prefix or object refs. |
| |
| Args: |
| bucket_listing_fields: If present, populate only these metadata |
| fields for listed objects. |
| expand_top_level_buckets: If true and the wildcard expands only to |
| Bucket(s), yields the expansion of each bucket |
| into a top-level listing of prefixes and objects |
| in that bucket instead of a BucketListingRef |
| to that bucket. |
| |
| Yields: |
| BucketListingRef, or empty iterator if no matches. |
| """ |
| for blr in self.__iter__( |
| bucket_listing_fields=bucket_listing_fields, |
| expand_top_level_buckets=expand_top_level_buckets): |
| yield blr |
| |
| def IterObjects(self, bucket_listing_fields=None): |
| """Iterates over the wildcard, yielding only object BucketListingRefs. |
| |
| Args: |
| bucket_listing_fields: If present, populate only these metadata |
| fields for listed objects. |
| |
| Yields: |
| BucketListingRefs of type OBJECT or empty iterator if no matches. |
| """ |
| for blr in self.__iter__(bucket_listing_fields=bucket_listing_fields, |
| expand_top_level_buckets=True): |
| if blr.IsObject(): |
| yield blr |
| |
| |
| class FileWildcardIterator(WildcardIterator): |
| """WildcardIterator subclass for files and directories. |
| |
| If you use recursive wildcards ('**') only a single such wildcard is |
| supported. For example you could use the wildcard '**/*.txt' to list all .txt |
| files in any subdirectory of the current directory, but you couldn't use a |
| wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt |
| files in any subdirectory named 'abc'). |
| """ |
| |
| def __init__(self, wildcard_url, debug=0): |
| """Instantiates an iterator over BucketListingRefs matching wildcard URL. |
| |
| Args: |
| wildcard_url: FileUrl that contains the wildcard to iterate. |
| debug: Debug level (range 0..3). |
| """ |
| self.wildcard_url = wildcard_url |
| self.debug = debug |
| |
| def __iter__(self): |
| """Iterator that gets called when iterating over the file wildcard. |
| |
| In the case where no wildcard is present, returns a single matching file |
| or directory. |
| |
| Raises: |
| WildcardException: if invalid wildcard found. |
| |
| Yields: |
| BucketListingRef of type OBJECT (for files) or PREFIX (for directories) |
| """ |
| wildcard = self.wildcard_url.object_name |
| match = FLAT_LIST_REGEX.match(wildcard) |
| if match: |
| # Recursive wildcarding request ('.../**/...'). |
| # Example input: wildcard = '/tmp/tmp2pQJAX/**/*' |
| base_dir = match.group('before')[:-1] |
| remaining_wildcard = match.group('after') |
| # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and |
| # remaining_wildcard = '/*' |
| if remaining_wildcard.startswith('*'): |
| raise WildcardException('Invalid wildcard with more than 2 consecutive ' |
| '*s (%s)' % wildcard) |
| # If there was no remaining wildcard past the recursive wildcard, |
| # treat it as if it were a '*'. For example, file://tmp/** is equivalent |
| # to file://tmp/**/* |
| if not remaining_wildcard: |
| remaining_wildcard = '*' |
| # Skip slash(es). |
| remaining_wildcard = remaining_wildcard.lstrip(os.sep) |
| filepaths = self._IterDir(base_dir, remaining_wildcard) |
| else: |
| # Not a recursive wildcarding request. |
| filepaths = glob.iglob(wildcard) |
| for filepath in filepaths: |
| expanded_url = StorageUrlFromString(filepath) |
| if os.path.isdir(filepath): |
| yield BucketListingPrefix(expanded_url) |
| else: |
| yield BucketListingObject(expanded_url) |
| |
| def _IterDir(self, directory, wildcard): |
| """An iterator over the specified dir and wildcard.""" |
| # UTF8-encode directory before passing it to os.walk() so if there are |
| # non-valid UTF8 chars in the file name (e.g., that can happen if the file |
| # originated on Windows) os.walk() will not attempt to decode and then die |
| # with a "codec can't decode byte" error, and instead we can catch the error |
| # at yield time and print a more informative error message. |
| for dirpath, unused_dirnames, filenames in os.walk(directory.encode(UTF8)): |
| for f in fnmatch.filter(filenames, wildcard): |
| try: |
| yield os.path.join(dirpath, f).decode(UTF8) |
| except UnicodeDecodeError: |
| # Note: We considered several ways to deal with this, but each had |
| # problems: |
| # 1. Raise an exception and try to catch in a higher layer (the |
| # gsutil cp command), so we can properly support the gsutil cp -c |
| # option. That doesn't work because raising an exception during |
| # iteration terminates the generator. |
| # 2. Accumulate a list of bad filenames and skip processing each |
| # during iteration, then raise at the end, with exception text |
| # printing the bad paths. That doesn't work because iteration is |
| # wrapped in PluralityCheckableIterator, so it's possible there |
| # are not-yet-performed copy operations at the time we reach the |
| # end of the iteration and raise the exception - which would cause |
| # us to skip copying validly named files. Moreover, the gsutil |
| # cp command loops over argv, so if you run the command gsutil cp |
| # -rc dir1 dir2 gs://bucket, an invalid unicode name inside dir1 |
| # would cause dir2 never to be visited. |
| # 3. Print the invalid pathname and skip it during iteration. That |
| # would work but would mean gsutil cp could exit with status 0 |
| # even though some files weren't copied. |
| # 4. Change the WildcardIterator to include an error status along with |
| # the result. That would solve the problem but would be a |
| # substantial change (WildcardIterator is used in many parts of |
| # gsutil), and we didn't feel that magnitude of change was |
| # warranted by this relatively uncommon corner case. |
| # Instead we chose to abort when one such file is encountered, and |
| # require the user to remove or rename the files and try again. |
| raise CommandException('\n'.join(textwrap.wrap( |
| 'Invalid Unicode path encountered (%s). gsutil cannot proceed ' |
| 'with such files present. Please remove or rename this file and ' |
| 'try again. NOTE: the path printed above replaces the ' |
| 'problematic characters with a hex-encoded printable ' |
| 'representation. For more details (including how to convert to a ' |
| 'gsutil-compatible encoding) see `gsutil help encoding`.' % |
| repr(os.path.join(dirpath, f))))) |
| |
| # pylint: disable=unused-argument |
| def IterObjects(self, bucket_listing_fields=None): |
| """Iterates over the wildcard, yielding only object (file) refs. |
| |
| Args: |
| bucket_listing_fields: Ignored as filesystems don't have buckets. |
| |
| Yields: |
| BucketListingRefs of type OBJECT or empty iterator if no matches. |
| """ |
| for bucket_listing_ref in self.IterAll(): |
| if bucket_listing_ref.IsObject(): |
| yield bucket_listing_ref |
| |
| # pylint: disable=unused-argument |
| def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False): |
| """Iterates over the wildcard, yielding BucketListingRefs. |
| |
| Args: |
| bucket_listing_fields: Ignored; filesystems don't have buckets. |
| expand_top_level_buckets: Ignored; filesystems don't have buckets. |
| |
| Yields: |
| BucketListingRefs of type OBJECT (file) or PREFIX (directory), |
| or empty iterator if no matches. |
| """ |
| for bucket_listing_ref in self.__iter__(): |
| yield bucket_listing_ref |
| |
| def IterBuckets(self, unused_bucket_fields=None): |
| """Placeholder to allow polymorphic use of WildcardIterator. |
| |
| Args: |
| unused_bucket_fields: Ignored; filesystems don't have buckets. |
| |
| Raises: |
| WildcardException: in all cases. |
| """ |
| raise WildcardException( |
| 'Iterating over Buckets not possible for file wildcards') |
| |
| |
| class WildcardException(StandardError): |
| """Exception raised for invalid wildcard URLs.""" |
| |
| def __init__(self, reason): |
| StandardError.__init__(self) |
| self.reason = reason |
| |
| def __repr__(self): |
| return 'WildcardException: %s' % self.reason |
| |
| def __str__(self): |
| return 'WildcardException: %s' % self.reason |
| |
| |
| def CreateWildcardIterator(url_str, gsutil_api, all_versions=False, debug=0, |
| project_id=None): |
| """Instantiate a WildcardIterator for the given URL string. |
| |
| Args: |
| url_str: URL string naming wildcard object(s) to iterate. |
| gsutil_api: Cloud storage interface. Passed in for thread safety, also |
| settable for testing/mocking. |
| all_versions: If true, the iterator yields all versions of objects |
| matching the wildcard. If false, yields just the live |
| object version. |
| debug: Debug level to control debug output for iterator. |
| project_id: Project id to use for bucket listings. |
| |
| Returns: |
| A WildcardIterator that handles the requested iteration. |
| """ |
| |
| url = StorageUrlFromString(url_str) |
| if url.IsFileUrl(): |
| return FileWildcardIterator(url, debug=debug) |
| else: # Cloud URL |
| return CloudWildcardIterator( |
| url, gsutil_api, all_versions=all_versions, debug=debug, |
| project_id=project_id) |
