| # -*- coding: utf-8 -*- |
| # Copyright 2014 Google Inc. All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Helper functions for progress callbacks.""" |
| |
| import logging |
| import sys |
| |
| from gslib.util import MakeHumanReadable |
| from gslib.util import UTF8 |
| |
| # Default upper and lower bounds for progress callback frequency. |
| _START_BYTES_PER_CALLBACK = 1024*64 |
| _MAX_BYTES_PER_CALLBACK = 1024*1024*100 |
| |
| # Max width of URL to display in progress indicator. Wide enough to allow |
| # 15 chars for x/y display on an 80 char wide terminal. |
| MAX_PROGRESS_INDICATOR_COLUMNS = 65 |
| |
| |
| class ProgressCallbackWithBackoff(object): |
| """Makes progress callbacks with exponential backoff to a maximum value. |
| |
| This prevents excessive log message output. |
| """ |
| |
| def __init__(self, total_size, callback_func, |
| start_bytes_per_callback=_START_BYTES_PER_CALLBACK, |
| max_bytes_per_callback=_MAX_BYTES_PER_CALLBACK, |
| calls_per_exponent=10): |
| """Initializes the callback with backoff. |
| |
| Args: |
| total_size: Total bytes to process. If this is None, size is not known |
| at the outset. |
| callback_func: Func of (int: processed_so_far, int: total_bytes) |
| used to make callbacks. |
| start_bytes_per_callback: Lower bound of bytes per callback. |
| max_bytes_per_callback: Upper bound of bytes per callback. |
| calls_per_exponent: Number of calls to make before reducing rate. |
| """ |
| self._bytes_per_callback = start_bytes_per_callback |
| self._callback_func = callback_func |
| self._calls_per_exponent = calls_per_exponent |
| self._max_bytes_per_callback = max_bytes_per_callback |
| self._total_size = total_size |
| |
| self._bytes_processed_since_callback = 0 |
| self._callbacks_made = 0 |
| self._total_bytes_processed = 0 |
| |
| def Progress(self, bytes_processed): |
| """Tracks byte processing progress, making a callback if necessary.""" |
| self._bytes_processed_since_callback += bytes_processed |
| if (self._bytes_processed_since_callback > self._bytes_per_callback or |
| (self._total_bytes_processed + self._bytes_processed_since_callback >= |
| self._total_size and self._total_size is not None)): |
| self._total_bytes_processed += self._bytes_processed_since_callback |
| # TODO: We check if >= total_size and truncate because JSON uploads count |
| # headers+metadata during their send progress. If the size is unknown, |
| # we can't do this and the progress message will make it appear that we |
| # send more than the original stream. |
| if self._total_size is not None: |
| bytes_sent = min(self._total_bytes_processed, self._total_size) |
| else: |
| bytes_sent = self._total_bytes_processed |
| self._callback_func(bytes_sent, self._total_size) |
| self._bytes_processed_since_callback = 0 |
| self._callbacks_made += 1 |
| |
| if self._callbacks_made > self._calls_per_exponent: |
| self._bytes_per_callback = min(self._bytes_per_callback * 2, |
| self._max_bytes_per_callback) |
| self._callbacks_made = 0 |
| |
| |
| def ConstructAnnounceText(operation_name, url_string): |
| """Constructs announce text for ongoing operations on url_to_display. |
| |
| This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS. |
| Thus, concurrent output (gsutil -m) leaves progress counters in a readable |
| (fixed) position. |
| |
| Args: |
| operation_name: String describing the operation, i.e. |
| 'Uploading' or 'Hashing'. |
| url_string: String describing the file/object being processed. |
| |
| Returns: |
| Formatted announce text for outputting operation progress. |
| """ |
| # Operation name occupies 11 characters (enough for 'Downloading'), plus a |
| # space. The rest is used for url_to_display. If a longer operation name is |
| # used, it will be truncated. We can revisit this size if we need to support |
| # a longer operation, but want to make sure the terminal output is meaningful. |
| justified_op_string = operation_name[:11].ljust(12) |
| start_len = len(justified_op_string) |
| end_len = len(': ') |
| if (start_len + len(url_string) + end_len > |
| MAX_PROGRESS_INDICATOR_COLUMNS): |
| ellipsis_len = len('...') |
| url_string = '...%s' % url_string[ |
| -(MAX_PROGRESS_INDICATOR_COLUMNS - start_len - end_len - ellipsis_len):] |
| base_announce_text = '%s%s:' % (justified_op_string, url_string) |
| format_str = '{0:%ds}' % MAX_PROGRESS_INDICATOR_COLUMNS |
| return format_str.format(base_announce_text.encode(UTF8)) |
| |
| |
| class FileProgressCallbackHandler(object): |
| """Outputs progress info for large operations like file copy or hash.""" |
| |
| def __init__(self, announce_text, logger, start_byte=0, |
| override_total_size=None): |
| """Initializes the callback handler. |
| |
| Args: |
| announce_text: String describing the operation. |
| logger: For outputting log messages. |
| start_byte: The beginning of the file component, if one is being used. |
| override_total_size: The size of the file component, if one is being used. |
| """ |
| self._announce_text = announce_text |
| self._logger = logger |
| self._start_byte = start_byte |
| self._override_total_size = override_total_size |
| # Ensures final newline is written once even if we get multiple callbacks. |
| self._last_byte_written = False |
| |
| # Function signature is in boto callback format, which cannot be changed. |
| def call(self, # pylint: disable=invalid-name |
| last_byte_processed, |
| total_size): |
| """Prints an overwriting line to stderr describing the operation progress. |
| |
| Args: |
| last_byte_processed: The last byte processed in the file. For file |
| components, this number should be in the range |
| [start_byte:start_byte + override_total_size]. |
| total_size: Total size of the ongoing operation. |
| """ |
| if not self._logger.isEnabledFor(logging.INFO) or self._last_byte_written: |
| return |
| |
| if self._override_total_size: |
| total_size = self._override_total_size |
| |
| if total_size: |
| total_size_string = '/%s' % MakeHumanReadable(total_size) |
| else: |
| total_size_string = '' |
| # Use sys.stderr.write instead of self.logger.info so progress messages |
| # output on a single continuously overwriting line. |
| # TODO: Make this work with logging.Logger. |
| sys.stderr.write('%s%s%s \r' % ( |
| self._announce_text, |
| MakeHumanReadable(last_byte_processed - self._start_byte), |
| total_size_string)) |
| if total_size and last_byte_processed - self._start_byte == total_size: |
| self._last_byte_written = True |
| sys.stderr.write('\n') |
