catapult/third_party/gsutil/gslib/commands/rm.py

# -*- coding: utf-8 -*-
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of Unix-like rm command for cloud storage providers."""

from __future__ import absolute_import

from gslib.cloud_api import NotEmptyException
from gslib.cloud_api import ServiceException
from gslib.command import Command
from gslib.command import GetFailureCount
from gslib.command import ResetFailureCount
from gslib.command_argument import CommandArgument
from gslib.cs_api_map import ApiSelector
from gslib.exception import CommandException
from gslib.name_expansion import NameExpansionIterator
from gslib.storage_url import StorageUrlFromString
from gslib.translation_helper import PreconditionsFromHeaders
from gslib.util import GetCloudApiInstance
from gslib.util import NO_MAX
from gslib.util import Retry
from gslib.util import StdinIterator


_SYNOPSIS = """
  gsutil rm [-f] [-r] url...
  gsutil rm [-f] [-r] -I
"""

_DETAILED_HELP_TEXT = ("""
<B>SYNOPSIS</B>
""" + _SYNOPSIS + """


<B>DESCRIPTION</B>
  The gsutil rm command removes objects.
  For example, the command:

    gsutil rm gs://bucket/subdir/*

  will remove all objects in gs://bucket/subdir, but not in any of its
  sub-directories. In contrast:

    gsutil rm gs://bucket/subdir/**

  will remove all objects under gs://bucket/subdir or any of its
  subdirectories.

  You can also use the -r option to specify recursive object deletion. Thus, for
  example, either of the following two commands will remove gs://bucket/subdir
  and all objects and subdirectories under it:

    gsutil rm gs://bucket/subdir**
    gsutil rm -r gs://bucket/subdir

  The -r option will also delete all object versions in the subdirectory for
  versioning-enabled buckets, whereas the ** command will only delete the live
  version of each object in the subdirectory.

  Running gsutil rm -r on a bucket will delete all versions of all objects in
  the bucket, and then delete the bucket:

    gsutil rm -r gs://bucket

  If you want to delete all objects in the bucket, but not the bucket itself,
  this command will work:

    gsutil rm gs://bucket/**

  If you have a large number of objects to remove you might want to use the
  gsutil -m option, to perform a parallel (multi-threaded/multi-processing)
  removes:

    gsutil -m rm -r gs://my_bucket/subdir

  You can pass a list of URLs (one per line) to remove on stdin instead of as
  command line arguments by using the -I option. This allows you to use gsutil
  in a pipeline to remove objects identified by a program, such as:

    some_program | gsutil -m rm -I

  The contents of stdin can name cloud URLs and wildcards of cloud URLs.

  Note that gsutil rm will refuse to remove files from the local
  file system. For example this will fail:

    gsutil rm *.txt

  WARNING: Object removal cannot be undone. Google Cloud Storage is designed
  to give developers a high amount of flexibility and control over their data,
  and Google maintains strict controls over the processing and purging of
  deleted data. To protect yourself from mistakes, you can configure object
  versioning on your bucket(s). See 'gsutil help versions' for details.


<B>DATA RESTORATION FROM ACCIDENTAL DELETION OR OVERWRITES</B>
Google Cloud Storage does not provide support for restoring data lost
or overwritten due to customer errors. If you have concerns that your
application software (or your users) may at some point erroneously delete or
overwrite data, you can protect yourself from that risk by enabling Object
Versioning (see "gsutil help versioning"). Doing so increases storage costs,
which can be partially mitigated by configuring Lifecycle Management to delete
older object versions (see "gsutil help lifecycle").


<B>OPTIONS</B>
  -f          Continues silently (without printing error messages) despite
              errors when removing multiple objects. If some of the objects
              could not be removed, gsutil's exit status will be non-zero even
              if this flag is set. This option is implicitly set when running
              "gsutil -m rm ...".

  -I          Causes gsutil to read the list of objects to remove from stdin.
              This allows you to run a program that generates the list of
              objects to remove.

  -R, -r      Causes bucket or bucket subdirectory contents (all objects and
              subdirectories that it contains) to be removed recursively. If
              used with a bucket-only URL (like gs://bucket), after deleting
              objects and subdirectories gsutil will delete the bucket.  The -r
              flag implies the -a flag and will delete all object versions.

  -a          Delete all versions of an object.
""")


def _RemoveExceptionHandler(cls, e):
  """Simple exception handler to allow post-completion status."""
  if not cls.continue_on_error:
    cls.logger.error(str(e))
  cls.everything_removed_okay = False


# pylint: disable=unused-argument
def _RemoveFoldersExceptionHandler(cls, e):
  """When removing folders, we don't mind if none exist."""
  if (isinstance(e, CommandException.__class__) and
      'No URLs matched' in e.message):
    pass
  else:
    raise e


def _RemoveFuncWrapper(cls, name_expansion_result, thread_state=None):
  cls.RemoveFunc(name_expansion_result, thread_state=thread_state)


class RmCommand(Command):
  """Implementation of gsutil rm command."""

  # Command specification. See base class for documentation.
  command_spec = Command.CreateCommandSpec(
      'rm',
      command_name_aliases=['del', 'delete', 'remove'],
      usage_synopsis=_SYNOPSIS,
      min_args=0,
      max_args=NO_MAX,
      supported_sub_args='afIrR',
      file_url_ok=False,
      provider_url_ok=False,
      urls_start_arg=0,
      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
      gs_default_api=ApiSelector.JSON,
      argparse_arguments=[
          CommandArgument.MakeZeroOrMoreCloudURLsArgument()
      ]
  )
  # Help specification. See help_provider.py for documentation.
  help_spec = Command.HelpSpec(
      help_name='rm',
      help_name_aliases=['del', 'delete', 'remove'],
      help_type='command_help',
      help_one_line_summary='Remove objects',
      help_text=_DETAILED_HELP_TEXT,
      subcommand_help_text={},
  )

  def RunCommand(self):
    """Command entry point for the rm command."""
    # self.recursion_requested is initialized in command.py (so it can be
    # checked in parent class for all commands).
    self.continue_on_error = False
    self.read_args_from_stdin = False
    self.all_versions = False
    if self.sub_opts:
      for o, unused_a in self.sub_opts:
        if o == '-a':
          self.all_versions = True
        elif o == '-f':
          self.continue_on_error = True
        elif o == '-I':
          self.read_args_from_stdin = True
        elif o == '-r' or o == '-R':
          self.recursion_requested = True
          self.all_versions = True

    if self.read_args_from_stdin:
      if self.args:
        raise CommandException('No arguments allowed with the -I flag.')
      url_strs = StdinIterator()
    else:
      if not self.args:
        raise CommandException('The rm command (without -I) expects at '
                               'least one URL.')
      url_strs = self.args

    bucket_urls_to_delete = []
    bucket_strings_to_delete = []
    if self.recursion_requested:
      bucket_fields = ['id']
      for url_str in url_strs:
        url = StorageUrlFromString(url_str)
        if url.IsBucket() or url.IsProvider():
          for blr in self.WildcardIterator(url_str).IterBuckets(
              bucket_fields=bucket_fields):
            bucket_urls_to_delete.append(blr.storage_url)
            bucket_strings_to_delete.append(url_str)

    self.preconditions = PreconditionsFromHeaders(self.headers or {})

    # Used to track if any files failed to be removed.
    self.everything_removed_okay = True

    try:
      # Expand wildcards, dirs, buckets, and bucket subdirs in URLs.
      name_expansion_iterator = NameExpansionIterator(
          self.command_name, self.debug, self.logger, self.gsutil_api,
          url_strs, self.recursion_requested, project_id=self.project_id,
          all_versions=self.all_versions,
          continue_on_error=self.continue_on_error or self.parallel_operations)

      # Perform remove requests in parallel (-m) mode, if requested, using
      # configured number of parallel processes and threads. Otherwise,
      # perform requests with sequential function calls in current process.
      self.Apply(_RemoveFuncWrapper, name_expansion_iterator,
                 _RemoveExceptionHandler,
                 fail_on_error=(not self.continue_on_error))

    # Assuming the bucket has versioning enabled, url's that don't map to
    # objects should throw an error even with all_versions, since the prior
    # round of deletes only sends objects to a history table.
    # This assumption that rm -a is only called for versioned buckets should be
    # corrected, but the fix is non-trivial.
    except CommandException as e:
      # Don't raise if there are buckets to delete -- it's valid to say:
      #   gsutil rm -r gs://some_bucket
      # if the bucket is empty.
      if not bucket_urls_to_delete and not self.continue_on_error:
        raise
      # Reset the failure count if we failed due to an empty bucket that we're
      # going to delete.
      msg = 'No URLs matched: '
      if msg in str(e):
        parts = str(e).split(msg)
        if len(parts) == 2 and parts[1] in bucket_strings_to_delete:
          ResetFailureCount()
    except ServiceException, e:
      if not self.continue_on_error:
        raise

    if not self.everything_removed_okay and not self.continue_on_error:
      raise CommandException('Some files could not be removed.')

    # If this was a gsutil rm -r command covering any bucket subdirs,
    # remove any dir_$folder$ objects (which are created by various web UI
    # tools to simulate folders).
    if self.recursion_requested:
      had_previous_failures = GetFailureCount() > 0
      folder_object_wildcards = []
      for url_str in url_strs:
        url = StorageUrlFromString(url_str)
        if url.IsObject():
          folder_object_wildcards.append('%s**_$folder$' % url_str)
      if folder_object_wildcards:
        self.continue_on_error = True
        try:
          name_expansion_iterator = NameExpansionIterator(
              self.command_name, self.debug,
              self.logger, self.gsutil_api,
              folder_object_wildcards, self.recursion_requested,
              project_id=self.project_id,
              all_versions=self.all_versions)
          # When we're removing folder objects, always continue on error
          self.Apply(_RemoveFuncWrapper, name_expansion_iterator,
                     _RemoveFoldersExceptionHandler,
                     fail_on_error=False)
        except CommandException as e:
          # Ignore exception from name expansion due to an absent folder file.
          if not e.reason.startswith('No URLs matched:'):
            raise
        if not had_previous_failures:
          ResetFailureCount()

    # Now that all data has been deleted, delete any bucket URLs.
    for url in bucket_urls_to_delete:
      self.logger.info('Removing %s...', url)

      @Retry(NotEmptyException, tries=3, timeout_secs=1)
      def BucketDeleteWithRetry():
        self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme)

      BucketDeleteWithRetry()

    return 0

  def RemoveFunc(self, name_expansion_result, thread_state=None):
    gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)

    exp_src_url = name_expansion_result.expanded_storage_url
    self.logger.info('Removing %s...', exp_src_url)
    gsutil_api.DeleteObject(
        exp_src_url.bucket_name, exp_src_url.object_name,
        preconditions=self.preconditions, generation=exp_src_url.generation,
        provider=exp_src_url.scheme)