| # -*- coding: utf-8 -*- |
| # Copyright 2012 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Implementation of setmeta command for setting cloud object metadata.""" |
| |
| from __future__ import absolute_import |
| |
| from gslib.cloud_api import AccessDeniedException |
| from gslib.cloud_api import PreconditionException |
| from gslib.cloud_api import Preconditions |
| from gslib.command import Command |
| from gslib.command_argument import CommandArgument |
| from gslib.cs_api_map import ApiSelector |
| from gslib.exception import CommandException |
| from gslib.name_expansion import NameExpansionIterator |
| from gslib.storage_url import StorageUrlFromString |
| from gslib.translation_helper import CopyObjectMetadata |
| from gslib.translation_helper import ObjectMetadataFromHeaders |
| from gslib.translation_helper import PreconditionsFromHeaders |
| from gslib.util import GetCloudApiInstance |
| from gslib.util import NO_MAX |
| from gslib.util import Retry |
| |
| |
| _SYNOPSIS = """ |
| gsutil setmeta -h [header:value|header] ... url... |
| """ |
| |
| _DETAILED_HELP_TEXT = (""" |
| <B>SYNOPSIS</B> |
| """ + _SYNOPSIS + """ |
| |
| |
| <B>DESCRIPTION</B> |
| The gsutil setmeta command allows you to set or remove the metadata on one |
| or more objects. It takes one or more header arguments followed by one or |
| more URLs, where each header argument is in one of two forms: |
| |
| - if you specify header:value, it will set the given header on all |
| named objects. |
| |
| - if you specify header (with no value), it will remove the given header |
| from all named objects. |
| |
| For example, the following command would set the Content-Type and |
| Cache-Control and remove the Content-Disposition on the specified objects: |
| |
| gsutil setmeta -h "Content-Type:text/html" \\ |
| -h "Cache-Control:public, max-age=3600" \\ |
| -h "Content-Disposition" gs://bucket/*.html |
| |
| If you have a large number of objects to update you might want to use the |
| gsutil -m option, to perform a parallel (multi-threaded/multi-processing) |
| update: |
| |
| gsutil -m setmeta -h "Content-Type:text/html" \\ |
| -h "Cache-Control:public, max-age=3600" \\ |
| -h "Content-Disposition" gs://bucket/*.html |
| |
| You can also use the setmeta command to set custom metadata on an object: |
| |
| gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object |
| |
| See "gsutil help metadata" for details about how you can set metadata |
| while uploading objects, what metadata fields can be set and the meaning of |
| these fields, use of custom metadata, and how to view currently set metadata. |
| |
| NOTE: By default, publicly readable objects are served with a Cache-Control |
| header allowing such objects to be cached for 3600 seconds. For more details |
| about this default behavior see the CACHE-CONTROL section of |
| "gsutil help metadata". If you need to ensure that updates become visible |
| immediately, you should set a Cache-Control header of "Cache-Control:private, |
| max-age=0, no-transform" on such objects. You can do this with the command: |
| |
| gsutil setmeta -h "Content-Type:text/html" \\ |
| -h "Cache-Control:private, max-age=0, no-transform" gs://bucket/*.html |
| |
| The setmeta command reads each object's current generation and metageneration |
| and uses those as preconditions unless they are otherwise specified by |
| top-level arguments. For example: |
| |
| gsutil -h "x-goog-if-metageneration-match:2" setmeta |
| -h "x-goog-meta-icecreamflavor:vanilla" |
| |
| will set the icecreamflavor:vanilla metadata if the current live object has a |
| metageneration of 2. |
| |
| <B>OPTIONS</B> |
| -h Specifies a header:value to be added, or header to be removed, |
| from each named object. |
| """) |
| |
| # Setmeta assumes a header-like model which doesn't line up with the JSON way |
| # of doing things. This list comes from functionality that was supported by |
| # gsutil3 at the time gsutil4 was released. |
| SETTABLE_FIELDS = ['cache-control', 'content-disposition', |
| 'content-encoding', 'content-language', |
| 'content-md5', 'content-type'] |
| |
| |
| def _SetMetadataExceptionHandler(cls, e): |
| """Exception handler that maintains state about post-completion status.""" |
| cls.logger.error(e) |
| cls.everything_set_okay = False |
| |
| |
| def _SetMetadataFuncWrapper(cls, name_expansion_result, thread_state=None): |
| cls.SetMetadataFunc(name_expansion_result, thread_state=thread_state) |
| |
| |
| class SetMetaCommand(Command): |
| """Implementation of gsutil setmeta command.""" |
| |
| # Command specification. See base class for documentation. |
| command_spec = Command.CreateCommandSpec( |
| 'setmeta', |
| command_name_aliases=['setheader'], |
| usage_synopsis=_SYNOPSIS, |
| min_args=1, |
| max_args=NO_MAX, |
| supported_sub_args='h:rR', |
| file_url_ok=False, |
| provider_url_ok=False, |
| urls_start_arg=1, |
| gs_api_support=[ApiSelector.XML, ApiSelector.JSON], |
| gs_default_api=ApiSelector.JSON, |
| argparse_arguments=[ |
| CommandArgument.MakeZeroOrMoreCloudURLsArgument() |
| ] |
| ) |
| # Help specification. See help_provider.py for documentation. |
| help_spec = Command.HelpSpec( |
| help_name='setmeta', |
| help_name_aliases=['setheader'], |
| help_type='command_help', |
| help_one_line_summary='Set metadata on already uploaded objects', |
| help_text=_DETAILED_HELP_TEXT, |
| subcommand_help_text={}, |
| ) |
| |
| def RunCommand(self): |
| """Command entry point for the setmeta command.""" |
| headers = [] |
| if self.sub_opts: |
| for o, a in self.sub_opts: |
| if o == '-h': |
| if 'x-goog-acl' in a or 'x-amz-acl' in a: |
| raise CommandException( |
| 'gsutil setmeta no longer allows canned ACLs. Use gsutil acl ' |
| 'set ... to set canned ACLs.') |
| headers.append(a) |
| |
| (metadata_minus, metadata_plus) = self._ParseMetadataHeaders(headers) |
| |
| self.metadata_change = metadata_plus |
| for header in metadata_minus: |
| self.metadata_change[header] = '' |
| |
| if len(self.args) == 1 and not self.recursion_requested: |
| url = StorageUrlFromString(self.args[0]) |
| if not (url.IsCloudUrl() and url.IsObject()): |
| raise CommandException('URL (%s) must name an object' % self.args[0]) |
| |
| # Used to track if any objects' metadata failed to be set. |
| self.everything_set_okay = True |
| |
| self.preconditions = PreconditionsFromHeaders(self.headers) |
| |
| name_expansion_iterator = NameExpansionIterator( |
| self.command_name, self.debug, self.logger, self.gsutil_api, |
| self.args, self.recursion_requested, all_versions=self.all_versions, |
| continue_on_error=self.parallel_operations) |
| |
| try: |
| # Perform requests in parallel (-m) mode, if requested, using |
| # configured number of parallel processes and threads. Otherwise, |
| # perform requests with sequential function calls in current process. |
| self.Apply(_SetMetadataFuncWrapper, name_expansion_iterator, |
| _SetMetadataExceptionHandler, fail_on_error=True) |
| except AccessDeniedException as e: |
| if e.status == 403: |
| self._WarnServiceAccounts() |
| raise |
| |
| if not self.everything_set_okay: |
| raise CommandException('Metadata for some objects could not be set.') |
| |
| return 0 |
| |
| @Retry(PreconditionException, tries=3, timeout_secs=1) |
| def SetMetadataFunc(self, name_expansion_result, thread_state=None): |
| """Sets metadata on an object. |
| |
| Args: |
| name_expansion_result: NameExpansionResult describing target object. |
| thread_state: gsutil Cloud API instance to use for the operation. |
| """ |
| gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) |
| |
| exp_src_url = name_expansion_result.expanded_storage_url |
| self.logger.info('Setting metadata on %s...', exp_src_url) |
| |
| fields = ['generation', 'metadata', 'metageneration'] |
| cloud_obj_metadata = gsutil_api.GetObjectMetadata( |
| exp_src_url.bucket_name, exp_src_url.object_name, |
| generation=exp_src_url.generation, provider=exp_src_url.scheme, |
| fields=fields) |
| |
| preconditions = Preconditions( |
| gen_match=self.preconditions.gen_match, |
| meta_gen_match=self.preconditions.meta_gen_match) |
| if preconditions.gen_match is None: |
| preconditions.gen_match = cloud_obj_metadata.generation |
| if preconditions.meta_gen_match is None: |
| preconditions.meta_gen_match = cloud_obj_metadata.metageneration |
| |
| # Patch handles the patch semantics for most metadata, but we need to |
| # merge the custom metadata field manually. |
| patch_obj_metadata = ObjectMetadataFromHeaders(self.metadata_change) |
| |
| api = gsutil_api.GetApiSelector(provider=exp_src_url.scheme) |
| # For XML we only want to patch through custom metadata that has |
| # changed. For JSON we need to build the complete set. |
| if api == ApiSelector.XML: |
| pass |
| elif api == ApiSelector.JSON: |
| CopyObjectMetadata(patch_obj_metadata, cloud_obj_metadata, |
| override=True) |
| patch_obj_metadata = cloud_obj_metadata |
| # Patch body does not need the object generation and metageneration. |
| patch_obj_metadata.generation = None |
| patch_obj_metadata.metageneration = None |
| |
| gsutil_api.PatchObjectMetadata( |
| exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, |
| generation=exp_src_url.generation, preconditions=preconditions, |
| provider=exp_src_url.scheme) |
| |
| def _ParseMetadataHeaders(self, headers): |
| """Validates and parses metadata changes from the headers argument. |
| |
| Args: |
| headers: Header dict to validate and parse. |
| |
| Returns: |
| (metadata_plus, metadata_minus): Tuple of header sets to add and remove. |
| """ |
| metadata_minus = set() |
| cust_metadata_minus = set() |
| metadata_plus = {} |
| cust_metadata_plus = {} |
| # Build a count of the keys encountered from each plus and minus arg so we |
| # can check for dupe field specs. |
| num_metadata_plus_elems = 0 |
| num_cust_metadata_plus_elems = 0 |
| num_metadata_minus_elems = 0 |
| num_cust_metadata_minus_elems = 0 |
| |
| for md_arg in headers: |
| parts = md_arg.split(':') |
| if len(parts) not in (1, 2): |
| raise CommandException( |
| 'Invalid argument: must be either header or header:value (%s)' % |
| md_arg) |
| if len(parts) == 2: |
| (header, value) = parts |
| else: |
| (header, value) = (parts[0], None) |
| _InsistAsciiHeader(header) |
| # Translate headers to lowercase to match the casing assumed by our |
| # sanity-checking operations. |
| header = header.lower() |
| if value: |
| if _IsCustomMeta(header): |
| # Allow non-ASCII data for custom metadata fields. |
| cust_metadata_plus[header] = value |
| num_cust_metadata_plus_elems += 1 |
| else: |
| # Don't unicode encode other fields because that would perturb their |
| # content (e.g., adding %2F's into the middle of a Cache-Control |
| # value). |
| _InsistAsciiHeaderValue(header, value) |
| value = str(value) |
| metadata_plus[header] = value |
| num_metadata_plus_elems += 1 |
| else: |
| if _IsCustomMeta(header): |
| cust_metadata_minus.add(header) |
| num_cust_metadata_minus_elems += 1 |
| else: |
| metadata_minus.add(header) |
| num_metadata_minus_elems += 1 |
| |
| if (num_metadata_plus_elems != len(metadata_plus) |
| or num_cust_metadata_plus_elems != len(cust_metadata_plus) |
| or num_metadata_minus_elems != len(metadata_minus) |
| or num_cust_metadata_minus_elems != len(cust_metadata_minus) |
| or metadata_minus.intersection(set(metadata_plus.keys()))): |
| raise CommandException('Each header must appear at most once.') |
| other_than_base_fields = (set(metadata_plus.keys()) |
| .difference(SETTABLE_FIELDS)) |
| other_than_base_fields.update( |
| metadata_minus.difference(SETTABLE_FIELDS)) |
| for f in other_than_base_fields: |
| # This check is overly simple; it would be stronger to check, for each |
| # URL argument, whether f.startswith the |
| # provider metadata_prefix, but here we just parse the spec |
| # once, before processing any of the URLs. This means we will not |
| # detect if the user tries to set an x-goog-meta- field on an another |
| # provider's object, for example. |
| if not _IsCustomMeta(f): |
| raise CommandException( |
| 'Invalid or disallowed header (%s).\nOnly these fields (plus ' |
| 'x-goog-meta-* fields) can be set or unset:\n%s' % ( |
| f, sorted(list(SETTABLE_FIELDS)))) |
| metadata_plus.update(cust_metadata_plus) |
| metadata_minus.update(cust_metadata_minus) |
| return (metadata_minus, metadata_plus) |
| |
| |
| def _InsistAscii(string, message): |
| if not all(ord(c) < 128 for c in string): |
| raise CommandException(message) |
| |
| |
| def _InsistAsciiHeader(header): |
| _InsistAscii(header, 'Invalid non-ASCII header (%s).' % header) |
| |
| |
| def _InsistAsciiHeaderValue(header, value): |
| _InsistAscii( |
| value, ('Invalid non-ASCII value (%s) was provided for header %s.' |
| % (value, header))) |
| |
| |
| def _IsCustomMeta(header): |
| return header.startswith('x-goog-meta-') or header.startswith('x-amz-meta-') |
