| # -*- coding: utf-8 -*- |
| # Copyright 2013 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Integration tests for rm command.""" |
| |
| from __future__ import absolute_import |
| |
| import re |
| |
| import gslib.tests.testcase as testcase |
| from gslib.tests.testcase.base import MAX_BUCKET_LENGTH |
| from gslib.tests.testcase.integration_testcase import SkipForS3 |
| from gslib.tests.util import GenerationFromURI as urigen |
| from gslib.tests.util import ObjectToURI as suri |
| from gslib.util import Retry |
| |
| |
| class TestRm(testcase.GsUtilIntegrationTestCase): |
| """Integration tests for rm command.""" |
| |
| def _RunRemoveCommandAndCheck(self, command_and_args, objects_to_remove=None, |
| buckets_to_remove=None, stdin=None): |
| """Tests a remove command in the presence of eventual listing consistency. |
| |
| Eventual listing consistency means that a remove command may not see all |
| of the objects to be removed at once. When removing multiple objects |
| (or buckets via -r), some calls may return no matches and multiple calls |
| to the rm command may be necessary to reach the desired state. This function |
| retries the rm command, incrementally tracking what has been removed and |
| ensuring that the exact set of objects/buckets are removed across all |
| retried calls. |
| |
| The caller is responsible for confirming the existence of buckets/objects |
| prior to calling this function. |
| |
| Args: |
| command_and_args: List of strings representing the rm command+args to run. |
| objects_to_remove: List of object URL strings (optionally including |
| generation) that should be removed by the command, if any. |
| buckets_to_remove: List of bucket URL strings that should be removed by |
| the command, if any. |
| stdin: String of data to pipe to the process as standard input (for |
| testing -I option). |
| """ |
| cumulative_stderr_lines = set() |
| bucket_strings = [] |
| for bucket_to_remove in buckets_to_remove or []: |
| bucket_strings.append('Removing %s/...' % bucket_to_remove) |
| object_strings = [] |
| for object_to_remove in objects_to_remove or []: |
| object_strings.append('Removing %s...' % object_to_remove) |
| expected_stderr_lines = set(object_strings + bucket_strings) |
| |
| @Retry(AssertionError, tries=5, timeout_secs=1) |
| def _RunRmCommandAndCheck(): |
| """Runs the command with retries, updating+checking cumulative output.""" |
| stderr = self.RunGsUtil(command_and_args, return_stderr=True, |
| expected_status=None, stdin=stdin) |
| update_lines = True |
| # Retry 404's and 409's due to eventual listing consistency, but don't add |
| # the output to the set. |
| if ('No URLs matched' in stderr or |
| '409 BucketNotEmpty' in stderr or |
| '409 VersionedBucketNotEmpty' in stderr): |
| update_lines = False |
| |
| # For recursive deletes of buckets, it is possible that the bucket is |
| # deleted before the objects are all present in the listing, in which case |
| # we will never see all of the expected "Removing object..." messages. |
| # Since this is still a successful outcome, just return successfully. |
| if '-r' in command_and_args and 'bucket does not exist' in stderr: |
| for bucket_to_remove in buckets_to_remove: |
| matching_bucket = re.match(r'.*404\s+%s\s+bucket does not exist' % |
| re.escape(bucket_to_remove), stderr) |
| if matching_bucket: |
| for line in cumulative_stderr_lines: |
| if 'Removing %s/...' % bucket_to_remove in line: |
| return |
| if 'Removing %s/...' % bucket_to_remove in stderr: |
| return |
| |
| if update_lines: |
| cumulative_stderr_lines.update(set(stderr.splitlines())) |
| |
| # Ensure all of the expected strings are present. |
| self.assertEqual(cumulative_stderr_lines, expected_stderr_lines) |
| |
| _RunRmCommandAndCheck() |
| |
| def test_all_versions_current(self): |
| """Test that 'rm -a' for an object with a current version works.""" |
| bucket_uri = self.CreateVersionedBucket() |
| key_uri = bucket_uri.clone_replace_name('foo') |
| key_uri.set_contents_from_string('bar') |
| g1 = urigen(key_uri) |
| key_uri.set_contents_from_string('baz') |
| g2 = urigen(key_uri) |
| self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) |
| # Use @Retry as hedge against bucket listing eventual consistency. |
| @Retry(AssertionError, tries=3, timeout_secs=1) |
| def _Check1(stderr_lines): |
| stderr = self.RunGsUtil(['-m', 'rm', '-a', suri(key_uri)], |
| return_stderr=True) |
| stderr_lines.update(set(stderr.splitlines())) |
| stderr = '\n'.join(stderr_lines) |
| self.assertEqual(stderr.count('Removing %s://' % self.default_provider), |
| 2) |
| self.assertIn('Removing %s#%s...' % (suri(key_uri), g1), stderr) |
| self.assertIn('Removing %s#%s...' % (suri(key_uri), g2), stderr) |
| all_stderr_lines = set() |
| _Check1(all_stderr_lines) |
| self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) |
| |
| def test_all_versions_no_current(self): |
| """Test that 'rm -a' for an object without a current version works.""" |
| bucket_uri = self.CreateVersionedBucket() |
| key_uri = bucket_uri.clone_replace_name('foo') |
| key_uri.set_contents_from_string('bar') |
| g1 = urigen(key_uri) |
| key_uri.set_contents_from_string('baz') |
| g2 = urigen(key_uri) |
| self._RunRemoveCommandAndCheck( |
| ['-m', 'rm', '-a', suri(key_uri)], |
| objects_to_remove=['%s#%s' % (suri(key_uri), g1), |
| '%s#%s' % (suri(key_uri), g2)]) |
| self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) |
| |
| def test_fails_for_missing_obj(self): |
| bucket_uri = self.CreateVersionedBucket() |
| stderr = self.RunGsUtil(['rm', '-a', '%s' % suri(bucket_uri, 'foo')], |
| return_stderr=True, expected_status=1) |
| self.assertIn('No URLs matched', stderr) |
| |
| def test_remove_all_versions_recursive_on_bucket(self): |
| """Test that 'rm -r' works on bucket.""" |
| bucket_uri = self.CreateVersionedBucket() |
| k1_uri = bucket_uri.clone_replace_name('foo') |
| k2_uri = bucket_uri.clone_replace_name('foo2') |
| k1_uri.set_contents_from_string('bar') |
| k2_uri.set_contents_from_string('bar2') |
| k1g1 = urigen(k1_uri) |
| k2g1 = urigen(k2_uri) |
| k1_uri.set_contents_from_string('baz') |
| k2_uri.set_contents_from_string('baz2') |
| k1g2 = urigen(k1_uri) |
| k2g2 = urigen(k2_uri) |
| |
| self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True) |
| |
| self._RunRemoveCommandAndCheck( |
| ['rm', '-r', suri(bucket_uri)], |
| objects_to_remove=['%s#%s' % (suri(k1_uri), k1g1), |
| '%s#%s' % (suri(k1_uri), k1g2), |
| '%s#%s' % (suri(k2_uri), k2g1), |
| '%s#%s' % (suri(k2_uri), k2g2)], |
| buckets_to_remove=[suri(bucket_uri)]) |
| |
| # Use @Retry as hedge against bucket listing eventual consistency. |
| @Retry(AssertionError, tries=3, timeout_secs=1) |
| def _Check(): |
| # Bucket should no longer exist. |
| stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)], |
| return_stderr=True, expected_status=1) |
| self.assertIn('bucket does not exist', stderr) |
| _Check() |
| |
| def test_remove_all_versions_recursive_on_subdir(self): |
| """Test that 'rm -r' works on subdir.""" |
| bucket_uri = self.CreateVersionedBucket() |
| k1_uri = bucket_uri.clone_replace_name('dir/foo') |
| k2_uri = bucket_uri.clone_replace_name('dir/foo2') |
| k1_uri.set_contents_from_string('bar') |
| k2_uri.set_contents_from_string('bar2') |
| k1g1 = urigen(k1_uri) |
| k2g1 = urigen(k2_uri) |
| k1_uri.set_contents_from_string('baz') |
| k2_uri.set_contents_from_string('baz2') |
| k1g2 = urigen(k1_uri) |
| k2g2 = urigen(k2_uri) |
| |
| self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True) |
| |
| self._RunRemoveCommandAndCheck( |
| ['rm', '-r', '%s' % suri(bucket_uri, 'dir')], |
| objects_to_remove=['%s#%s' % (suri(k1_uri), k1g1), |
| '%s#%s' % (suri(k1_uri), k1g2), |
| '%s#%s' % (suri(k2_uri), k2g1), |
| '%s#%s' % (suri(k2_uri), k2g2)]) |
| self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) |
| |
| def test_missing_first_force(self): |
| bucket_uri = self.CreateBucket() |
| object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='present', |
| contents='foo') |
| self.AssertNObjectsInBucket(bucket_uri, 1) |
| self.RunGsUtil(['rm', '%s' % suri(bucket_uri, 'missing'), |
| suri(object_uri)], expected_status=1) |
| stderr = self.RunGsUtil( |
| ['rm', '-f', '%s' % suri(bucket_uri, 'missing'), suri(object_uri)], |
| return_stderr=True, expected_status=1) |
| self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1) |
| self.RunGsUtil(['stat', suri(object_uri)], expected_status=1) |
| |
| def test_some_missing(self): |
| """Test that 'rm -a' fails when some but not all uris don't exist.""" |
| bucket_uri = self.CreateVersionedBucket() |
| key_uri = bucket_uri.clone_replace_name('foo') |
| key_uri.set_contents_from_string('bar') |
| self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True) |
| stderr = self.RunGsUtil(['rm', '-a', suri(key_uri), |
| '%s' % suri(bucket_uri, 'missing')], |
| return_stderr=True, expected_status=1) |
| self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1) |
| self.assertIn('No URLs matched', stderr) |
| |
| def test_some_missing_force(self): |
| """Test that 'rm -af' succeeds despite hidden first uri.""" |
| bucket_uri = self.CreateVersionedBucket() |
| key_uri = bucket_uri.clone_replace_name('foo') |
| key_uri.set_contents_from_string('bar') |
| self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True) |
| stderr = self.RunGsUtil( |
| ['rm', '-af', suri(key_uri), '%s' % suri(bucket_uri, 'missing')], |
| return_stderr=True, expected_status=1) |
| self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1) |
| self.AssertNObjectsInBucket(bucket_uri, 0) |
| |
| def test_folder_objects_deleted(self): |
| """Test for 'rm -r' of a folder with a dir_$folder$ marker.""" |
| bucket_uri = self.CreateVersionedBucket() |
| key_uri = bucket_uri.clone_replace_name('abc/o1') |
| key_uri.set_contents_from_string('foobar') |
| folder_uri = bucket_uri.clone_replace_name('abc_$folder$') |
| folder_uri.set_contents_from_string('') |
| self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) |
| # This could fail due to eventual listing consistency, so use retry and |
| # expected_status=None to guard against No URLs matched exceptions. |
| @Retry(AssertionError, tries=3, timeout_secs=1) |
| def _RemoveAndCheck(): |
| self.RunGsUtil(['rm', '-r', '%s' % suri(bucket_uri, 'abc')], |
| expected_status=None) |
| self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) |
| _RemoveAndCheck() |
| # Bucket should not be deleted (Should not get ServiceException). |
| bucket_uri.get_location(validate=False) |
| |
| def test_folder_objects_deleted_with_wildcard(self): |
| """Test for 'rm -r' of a folder with a dir_$folder$ marker.""" |
| bucket_uri = self.CreateVersionedBucket() |
| key_uri = bucket_uri.clone_replace_name('abc/o1') |
| key_uri.set_contents_from_string('foobar') |
| folder_uri = bucket_uri.clone_replace_name('abc_$folder$') |
| folder_uri.set_contents_from_string('') |
| |
| self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) |
| self._RunRemoveCommandAndCheck( |
| ['rm', '-r', '%s' % suri(bucket_uri, '**')], |
| objects_to_remove=['%s#%s' % (suri(key_uri), urigen(key_uri)), |
| '%s#%s' % (suri(folder_uri), urigen(folder_uri))]) |
| self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) |
| # Bucket should not be deleted (Should not get ServiceException). |
| bucket_uri.get_location(validate=False) |
| |
| @SkipForS3('Listing/removing S3 DeleteMarkers is not supported') |
| def test_recursive_bucket_rm(self): |
| """Test for 'rm -r' of a bucket.""" |
| bucket_uri = self.CreateBucket() |
| object_uri = self.CreateObject(bucket_uri, contents='foo') |
| self.AssertNObjectsInBucket(bucket_uri, 1) |
| self._RunRemoveCommandAndCheck( |
| ['rm', '-r', suri(bucket_uri)], |
| objects_to_remove=['%s#%s' % (suri(object_uri), urigen(object_uri))], |
| buckets_to_remove=[suri(bucket_uri)]) |
| |
| # Use @Retry as hedge against bucket listing eventual consistency. |
| @Retry(AssertionError, tries=3, timeout_secs=1) |
| def _Check1(): |
| # Bucket should be deleted. |
| stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], |
| return_stderr=True, expected_status=1) |
| self.assertIn('bucket does not exist', stderr) |
| _Check1() |
| |
| # Now try same thing, but for a versioned bucket with multiple versions of |
| # an object present. |
| bucket_uri = self.CreateVersionedBucket() |
| self.CreateObject(bucket_uri, 'obj', 'z') |
| self.CreateObject(bucket_uri, 'obj', 'z') |
| final_uri = self.CreateObject(bucket_uri, 'obj', 'z') |
| self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True) |
| self._RunRemoveCommandAndCheck(['rm', suri(bucket_uri, '**')], |
| objects_to_remove=['%s' % final_uri]) |
| |
| stderr = self.RunGsUtil(['rb', suri(bucket_uri)], |
| return_stderr=True, expected_status=1) |
| self.assertIn('Bucket is not empty', stderr) |
| |
| # Now try with rm -r. |
| @Retry(AssertionError, tries=3, timeout_secs=1) |
| def _Check2(): |
| self.RunGsUtil(['rm', '-r', suri(bucket_uri)]) |
| # Bucket should be deleted. |
| stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], |
| return_stderr=True, expected_status=1) |
| self.assertIn('bucket does not exist', stderr) |
| _Check2() |
| |
| def test_recursive_bucket_rm_with_wildcarding(self): |
| """Tests removing all objects and buckets matching a bucket wildcard.""" |
| buri_base = 'gsutil-test-%s' % self.GetTestMethodName() |
| buri_base = buri_base[:MAX_BUCKET_LENGTH-20] |
| buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString()) |
| buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base) |
| buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base) |
| buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base) |
| ouri1 = self.CreateObject(bucket_uri=buri1, object_name='o1', contents='z') |
| ouri2 = self.CreateObject(bucket_uri=buri2, object_name='o2', contents='z') |
| self.CreateObject(bucket_uri=buri3, object_name='o3', contents='z') |
| |
| self.AssertNObjectsInBucket(buri1, 1) |
| self.AssertNObjectsInBucket(buri2, 1) |
| self.AssertNObjectsInBucket(buri3, 1) |
| |
| self._RunRemoveCommandAndCheck( |
| ['rm', '-r', '%s://%s-tbu*' % (self.default_provider, buri_base)], |
| objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1)), |
| '%s#%s' % (suri(ouri2), urigen(ouri2))], |
| buckets_to_remove=[suri(buri1), suri(buri2)]) |
| |
| self.AssertNObjectsInBucket(buri3, 1) |
| |
| def test_rm_quiet(self): |
| """Test that 'rm -q' outputs no progress indications.""" |
| bucket_uri = self.CreateBucket() |
| key_uri = self.CreateObject(bucket_uri=bucket_uri, contents='foo') |
| self.AssertNObjectsInBucket(bucket_uri, 1) |
| self._RunRemoveCommandAndCheck(['-q', 'rm', suri(key_uri)], []) |
| self.AssertNObjectsInBucket(bucket_uri, 0) |
| |
| def test_rm_object_with_slash(self): |
| """Tests removing a bucket that has an object with a slash in it.""" |
| bucket_uri = self.CreateVersionedBucket() |
| ouri1 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='/dirwithslash/foo', contents='z') |
| ouri2 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='dirnoslash/foo', contents='z') |
| ouri3 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='dirnoslash/foo2', contents='z') |
| |
| self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True) |
| |
| self._RunRemoveCommandAndCheck( |
| ['rm', '-r', suri(bucket_uri)], |
| objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1)), |
| '%s#%s' % (suri(ouri2), urigen(ouri2)), |
| '%s#%s' % (suri(ouri3), urigen(ouri3))], |
| buckets_to_remove=[suri(bucket_uri)]) |
| |
| def test_slasher_horror_film(self): |
| """Tests removing a bucket with objects that are filled with slashes.""" |
| bucket_uri = self.CreateVersionedBucket() |
| ouri1 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='h/e/l//lo', |
| contents='Halloween') |
| ouri2 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='/h/e/l/l/o', |
| contents='A Nightmare on Elm Street') |
| ouri3 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='//h//e/l//l/o', |
| contents='Friday the 13th') |
| ouri4 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='//h//e//l//l//o', |
| contents='I Know What You Did Last Summer') |
| ouri5 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='/', |
| contents='Scream') |
| ouri6 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='//', |
| contents='Child\'s Play') |
| ouri7 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='///', |
| contents='The Prowler') |
| ouri8 = self.CreateObject(bucket_uri=bucket_uri, |
| object_name='////', |
| contents='Black Christmas') |
| ouri9 = self.CreateObject( |
| bucket_uri=bucket_uri, |
| object_name='everything/is/better/with/slashes///////', |
| contents='Maniac') |
| |
| self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True) |
| |
| # We add a slash to URIs with a trailing slash, |
| # because ObjectToURI (suri) removes one trailing slash. |
| objects_to_remove = [ |
| '%s#%s' % (suri(ouri1), urigen(ouri1)), |
| '%s#%s' % (suri(ouri2), urigen(ouri2)), |
| '%s#%s' % (suri(ouri3), urigen(ouri3)), |
| '%s#%s' % (suri(ouri4), urigen(ouri4)), |
| '%s#%s' % (suri(ouri5) + '/', urigen(ouri5)), |
| '%s#%s' % (suri(ouri6) + '/', urigen(ouri6)), |
| '%s#%s' % (suri(ouri7) + '/', urigen(ouri7)), |
| '%s#%s' % (suri(ouri8) + '/', urigen(ouri8)), |
| '%s#%s' % (suri(ouri9) + '/', urigen(ouri9))] |
| |
| self._RunRemoveCommandAndCheck(['-m', 'rm', '-r', suri(bucket_uri)], |
| objects_to_remove=objects_to_remove, |
| buckets_to_remove=[suri(bucket_uri)]) |
| |
| @SkipForS3('GCS versioning headers not supported by S3') |
| def test_rm_failing_precondition(self): |
| """Test for '-h x-goog-if-generation-match:value rm' of an object.""" |
| bucket_uri = self.CreateBucket() |
| object_uri = self.CreateObject(bucket_uri, contents='foo') |
| stderr = self.RunGsUtil(['-h', 'x-goog-if-generation-match:12345', 'rm', |
| suri(object_uri)], return_stderr=True, |
| expected_status=1) |
| self.assertRegexpMatches( |
| stderr, r'PreconditionException: 412 Precondition\s*Failed') |
| |
| def test_stdin_args(self): |
| """Tests rm with the -I option.""" |
| buri1 = self.CreateVersionedBucket() |
| ouri1 = self.CreateObject(bucket_uri=buri1, |
| object_name='foo', |
| contents='foocontents') |
| self.CreateObject(bucket_uri=buri1, object_name='bar', |
| contents='barcontents') |
| ouri3 = self.CreateObject(bucket_uri=buri1, |
| object_name='baz', |
| contents='bazcontents') |
| buri2 = self.CreateVersionedBucket() |
| ouri4 = self.CreateObject(bucket_uri=buri2, |
| object_name='moo', |
| contents='moocontents') |
| self.AssertNObjectsInBucket(buri1, 3, versioned=True) |
| self.AssertNObjectsInBucket(buri2, 1, versioned=True) |
| |
| objects_to_remove = ['%s#%s' % (suri(ouri1), urigen(ouri1)), |
| '%s#%s' % (suri(ouri3), urigen(ouri3)), |
| '%s#%s' % (suri(ouri4), urigen(ouri4))] |
| stdin = '\n'.join(objects_to_remove) |
| self._RunRemoveCommandAndCheck(['rm', '-I'], |
| objects_to_remove=objects_to_remove, |
| stdin=stdin) |
| self.AssertNObjectsInBucket(buri1, 1, versioned=True) |
| self.AssertNObjectsInBucket(buri2, 0, versioned=True) |
| |
| def test_rm_nonexistent_bucket_recursive(self): |
| stderr = self.RunGsUtil( |
| ['rm', '-rf', '%s://%s' % (self.default_provider, |
| self.nonexistent_bucket_name)], |
| return_stderr=True, expected_status=1) |
| self.assertIn('Encountered non-existent bucket', stderr) |
| |
| def test_rm_multiple_nonexistent_objects(self): |
| bucket_uri = self.CreateBucket() |
| nonexistent_object1 = suri(bucket_uri, 'nonexistent1') |
| nonexistent_object2 = suri(bucket_uri, 'nonexistent1') |
| stderr = self.RunGsUtil( |
| ['rm', '-rf', nonexistent_object1, nonexistent_object2], |
| return_stderr=True, expected_status=1) |
| self.assertIn('2 files/objects could not be removed.', stderr) |
