""" | |
Python implementation of the io module. | |
""" | |
from __future__ import (print_function, unicode_literals) | |
import os | |
import abc | |
import codecs | |
import warnings | |
# Import thread instead of threading to reduce startup cost | |
try: | |
from thread import allocate_lock as Lock | |
except ImportError: | |
from dummy_thread import allocate_lock as Lock | |
import io | |
from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) | |
from errno import EINTR | |
__metaclass__ = type | |
# open() uses st_blksize whenever we can | |
DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes | |
# NOTE: Base classes defined here are registered with the "official" ABCs | |
# defined in io.py. We don't use real inheritance though, because we don't | |
# want to inherit the C implementations. | |
class BlockingIOError(IOError): | |
"""Exception raised when I/O would block on a non-blocking I/O stream.""" | |
def __init__(self, errno, strerror, characters_written=0): | |
super(IOError, self).__init__(errno, strerror) | |
if not isinstance(characters_written, (int, long)): | |
raise TypeError("characters_written must be a integer") | |
self.characters_written = characters_written | |
def open(file, mode="r", buffering=-1, | |
encoding=None, errors=None, | |
newline=None, closefd=True): | |
r"""Open file and return a stream. Raise IOError upon failure. | |
file is either a text or byte string giving the name (and the path | |
if the file isn't in the current working directory) of the file to | |
be opened or an integer file descriptor of the file to be | |
wrapped. (If a file descriptor is given, it is closed when the | |
returned I/O object is closed, unless closefd is set to False.) | |
mode is an optional string that specifies the mode in which the file | |
is opened. It defaults to 'r' which means open for reading in text | |
mode. Other common values are 'w' for writing (truncating the file if | |
it already exists), and 'a' for appending (which on some Unix systems, | |
means that all writes append to the end of the file regardless of the | |
current seek position). In text mode, if encoding is not specified the | |
encoding used is platform dependent. (For reading and writing raw | |
bytes use binary mode and leave encoding unspecified.) The available | |
modes are: | |
========= =============================================================== | |
Character Meaning | |
--------- --------------------------------------------------------------- | |
'r' open for reading (default) | |
'w' open for writing, truncating the file first | |
'a' open for writing, appending to the end of the file if it exists | |
'b' binary mode | |
't' text mode (default) | |
'+' open a disk file for updating (reading and writing) | |
'U' universal newline mode (for backwards compatibility; unneeded | |
for new code) | |
========= =============================================================== | |
The default mode is 'rt' (open for reading text). For binary random | |
access, the mode 'w+b' opens and truncates the file to 0 bytes, while | |
'r+b' opens the file without truncation. | |
Python distinguishes between files opened in binary and text modes, | |
even when the underlying operating system doesn't. Files opened in | |
binary mode (appending 'b' to the mode argument) return contents as | |
bytes objects without any decoding. In text mode (the default, or when | |
't' is appended to the mode argument), the contents of the file are | |
returned as strings, the bytes having been first decoded using a | |
platform-dependent encoding or using the specified encoding if given. | |
buffering is an optional integer used to set the buffering policy. | |
Pass 0 to switch buffering off (only allowed in binary mode), 1 to select | |
line buffering (only usable in text mode), and an integer > 1 to indicate | |
the size of a fixed-size chunk buffer. When no buffering argument is | |
given, the default buffering policy works as follows: | |
* Binary files are buffered in fixed-size chunks; the size of the buffer | |
is chosen using a heuristic trying to determine the underlying device's | |
"block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. | |
On many systems, the buffer will typically be 4096 or 8192 bytes long. | |
* "Interactive" text files (files for which isatty() returns True) | |
use line buffering. Other text files use the policy described above | |
for binary files. | |
encoding is the name of the encoding used to decode or encode the | |
file. This should only be used in text mode. The default encoding is | |
platform dependent, but any encoding supported by Python can be | |
passed. See the codecs module for the list of supported encodings. | |
errors is an optional string that specifies how encoding errors are to | |
be handled---this argument should not be used in binary mode. Pass | |
'strict' to raise a ValueError exception if there is an encoding error | |
(the default of None has the same effect), or pass 'ignore' to ignore | |
errors. (Note that ignoring encoding errors can lead to data loss.) | |
See the documentation for codecs.register for a list of the permitted | |
encoding error strings. | |
newline controls how universal newlines works (it only applies to text | |
mode). It can be None, '', '\n', '\r', and '\r\n'. It works as | |
follows: | |
* On input, if newline is None, universal newlines mode is | |
enabled. Lines in the input can end in '\n', '\r', or '\r\n', and | |
these are translated into '\n' before being returned to the | |
caller. If it is '', universal newline mode is enabled, but line | |
endings are returned to the caller untranslated. If it has any of | |
the other legal values, input lines are only terminated by the given | |
string, and the line ending is returned to the caller untranslated. | |
* On output, if newline is None, any '\n' characters written are | |
translated to the system default line separator, os.linesep. If | |
newline is '', no translation takes place. If newline is any of the | |
other legal values, any '\n' characters written are translated to | |
the given string. | |
If closefd is False, the underlying file descriptor will be kept open | |
when the file is closed. This does not work when a file name is given | |
and must be True in that case. | |
open() returns a file object whose type depends on the mode, and | |
through which the standard file operations such as reading and writing | |
are performed. When open() is used to open a file in a text mode ('w', | |
'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open | |
a file in a binary mode, the returned class varies: in read binary | |
mode, it returns a BufferedReader; in write binary and append binary | |
modes, it returns a BufferedWriter, and in read/write mode, it returns | |
a BufferedRandom. | |
It is also possible to use a string or bytearray as a file for both | |
reading and writing. For strings StringIO can be used like a file | |
opened in a text mode, and for bytes a BytesIO can be used like a file | |
opened in a binary mode. | |
""" | |
if not isinstance(file, (basestring, int, long)): | |
raise TypeError("invalid file: %r" % file) | |
if not isinstance(mode, basestring): | |
raise TypeError("invalid mode: %r" % mode) | |
if not isinstance(buffering, (int, long)): | |
raise TypeError("invalid buffering: %r" % buffering) | |
if encoding is not None and not isinstance(encoding, basestring): | |
raise TypeError("invalid encoding: %r" % encoding) | |
if errors is not None and not isinstance(errors, basestring): | |
raise TypeError("invalid errors: %r" % errors) | |
modes = set(mode) | |
if modes - set("arwb+tU") or len(mode) > len(modes): | |
raise ValueError("invalid mode: %r" % mode) | |
reading = "r" in modes | |
writing = "w" in modes | |
appending = "a" in modes | |
updating = "+" in modes | |
text = "t" in modes | |
binary = "b" in modes | |
if "U" in modes: | |
if writing or appending: | |
raise ValueError("can't use U and writing mode at once") | |
reading = True | |
if text and binary: | |
raise ValueError("can't have text and binary mode at once") | |
if reading + writing + appending > 1: | |
raise ValueError("can't have read/write/append mode at once") | |
if not (reading or writing or appending): | |
raise ValueError("must have exactly one of read/write/append mode") | |
if binary and encoding is not None: | |
raise ValueError("binary mode doesn't take an encoding argument") | |
if binary and errors is not None: | |
raise ValueError("binary mode doesn't take an errors argument") | |
if binary and newline is not None: | |
raise ValueError("binary mode doesn't take a newline argument") | |
raw = FileIO(file, | |
(reading and "r" or "") + | |
(writing and "w" or "") + | |
(appending and "a" or "") + | |
(updating and "+" or ""), | |
closefd) | |
line_buffering = False | |
if buffering == 1 or buffering < 0 and raw.isatty(): | |
buffering = -1 | |
line_buffering = True | |
if buffering < 0: | |
buffering = DEFAULT_BUFFER_SIZE | |
try: | |
bs = os.fstat(raw.fileno()).st_blksize | |
except (os.error, AttributeError): | |
pass | |
else: | |
if bs > 1: | |
buffering = bs | |
if buffering < 0: | |
raise ValueError("invalid buffering size") | |
if buffering == 0: | |
if binary: | |
return raw | |
raise ValueError("can't have unbuffered text I/O") | |
if updating: | |
buffer = BufferedRandom(raw, buffering) | |
elif writing or appending: | |
buffer = BufferedWriter(raw, buffering) | |
elif reading: | |
buffer = BufferedReader(raw, buffering) | |
else: | |
raise ValueError("unknown mode: %r" % mode) | |
if binary: | |
return buffer | |
text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering) | |
text.mode = mode | |
return text | |
class DocDescriptor: | |
"""Helper for builtins.open.__doc__ | |
""" | |
def __get__(self, obj, typ): | |
return ( | |
"open(file, mode='r', buffering=-1, encoding=None, " | |
"errors=None, newline=None, closefd=True)\n\n" + | |
open.__doc__) | |
class OpenWrapper: | |
"""Wrapper for builtins.open | |
Trick so that open won't become a bound method when stored | |
as a class variable (as dbm.dumb does). | |
See initstdio() in Python/pythonrun.c. | |
""" | |
__doc__ = DocDescriptor() | |
def __new__(cls, *args, **kwargs): | |
return open(*args, **kwargs) | |
class UnsupportedOperation(ValueError, IOError): | |
pass | |
class IOBase: | |
__metaclass__ = abc.ABCMeta | |
"""The abstract base class for all I/O classes, acting on streams of | |
bytes. There is no public constructor. | |
This class provides dummy implementations for many methods that | |
derived classes can override selectively; the default implementations | |
represent a file that cannot be read, written or seeked. | |
Even though IOBase does not declare read, readinto, or write because | |
their signatures will vary, implementations and clients should | |
consider those methods part of the interface. Also, implementations | |
may raise a IOError when operations they do not support are called. | |
The basic type used for binary data read from or written to a file is | |
bytes. bytearrays are accepted too, and in some cases (such as | |
readinto) needed. Text I/O classes work with str data. | |
Note that calling any method (even inquiries) on a closed stream is | |
undefined. Implementations may raise IOError in this case. | |
IOBase (and its subclasses) support the iterator protocol, meaning | |
that an IOBase object can be iterated over yielding the lines in a | |
stream. | |
IOBase also supports the :keyword:`with` statement. In this example, | |
fp is closed after the suite of the with statement is complete: | |
with open('spam.txt', 'r') as fp: | |
fp.write('Spam and eggs!') | |
""" | |
### Internal ### | |
def _unsupported(self, name): | |
"""Internal: raise an exception for unsupported operations.""" | |
raise UnsupportedOperation("%s.%s() not supported" % | |
(self.__class__.__name__, name)) | |
### Positioning ### | |
def seek(self, pos, whence=0): | |
"""Change stream position. | |
Change the stream position to byte offset offset. offset is | |
interpreted relative to the position indicated by whence. Values | |
for whence are: | |
* 0 -- start of stream (the default); offset should be zero or positive | |
* 1 -- current stream position; offset may be negative | |
* 2 -- end of stream; offset is usually negative | |
Return the new absolute position. | |
""" | |
self._unsupported("seek") | |
def tell(self): | |
"""Return current stream position.""" | |
return self.seek(0, 1) | |
def truncate(self, pos=None): | |
"""Truncate file to size bytes. | |
Size defaults to the current IO position as reported by tell(). Return | |
the new size. | |
""" | |
self._unsupported("truncate") | |
### Flush and close ### | |
def flush(self): | |
"""Flush write buffers, if applicable. | |
This is not implemented for read-only and non-blocking streams. | |
""" | |
self._checkClosed() | |
# XXX Should this return the number of bytes written??? | |
__closed = False | |
def close(self): | |
"""Flush and close the IO object. | |
This method has no effect if the file is already closed. | |
""" | |
if not self.__closed: | |
self.flush() | |
self.__closed = True | |
def __del__(self): | |
"""Destructor. Calls close().""" | |
# The try/except block is in case this is called at program | |
# exit time, when it's possible that globals have already been | |
# deleted, and then the close() call might fail. Since | |
# there's nothing we can do about such failures and they annoy | |
# the end users, we suppress the traceback. | |
try: | |
self.close() | |
except: | |
pass | |
### Inquiries ### | |
def seekable(self): | |
"""Return whether object supports random access. | |
If False, seek(), tell() and truncate() will raise IOError. | |
This method may need to do a test seek(). | |
""" | |
return False | |
def _checkSeekable(self, msg=None): | |
"""Internal: raise an IOError if file is not seekable | |
""" | |
if not self.seekable(): | |
raise IOError("File or stream is not seekable." | |
if msg is None else msg) | |
def readable(self): | |
"""Return whether object was opened for reading. | |
If False, read() will raise IOError. | |
""" | |
return False | |
def _checkReadable(self, msg=None): | |
"""Internal: raise an IOError if file is not readable | |
""" | |
if not self.readable(): | |
raise IOError("File or stream is not readable." | |
if msg is None else msg) | |
def writable(self): | |
"""Return whether object was opened for writing. | |
If False, write() and truncate() will raise IOError. | |
""" | |
return False | |
def _checkWritable(self, msg=None): | |
"""Internal: raise an IOError if file is not writable | |
""" | |
if not self.writable(): | |
raise IOError("File or stream is not writable." | |
if msg is None else msg) | |
@property | |
def closed(self): | |
"""closed: bool. True iff the file has been closed. | |
For backwards compatibility, this is a property, not a predicate. | |
""" | |
return self.__closed | |
def _checkClosed(self, msg=None): | |
"""Internal: raise an ValueError if file is closed | |
""" | |
if self.closed: | |
raise ValueError("I/O operation on closed file." | |
if msg is None else msg) | |
### Context manager ### | |
def __enter__(self): | |
"""Context management protocol. Returns self.""" | |
self._checkClosed() | |
return self | |
def __exit__(self, *args): | |
"""Context management protocol. Calls close()""" | |
self.close() | |
### Lower-level APIs ### | |
# XXX Should these be present even if unimplemented? | |
def fileno(self): | |
"""Returns underlying file descriptor if one exists. | |
An IOError is raised if the IO object does not use a file descriptor. | |
""" | |
self._unsupported("fileno") | |
def isatty(self): | |
"""Return whether this is an 'interactive' stream. | |
Return False if it can't be determined. | |
""" | |
self._checkClosed() | |
return False | |
### Readline[s] and writelines ### | |
def readline(self, limit=-1): | |
r"""Read and return a line from the stream. | |
If limit is specified, at most limit bytes will be read. | |
The line terminator is always b'\n' for binary files; for text | |
files, the newlines argument to open can be used to select the line | |
terminator(s) recognized. | |
""" | |
# For backwards compatibility, a (slowish) readline(). | |
if hasattr(self, "peek"): | |
def nreadahead(): | |
readahead = self.peek(1) | |
if not readahead: | |
return 1 | |
n = (readahead.find(b"\n") + 1) or len(readahead) | |
if limit >= 0: | |
n = min(n, limit) | |
return n | |
else: | |
def nreadahead(): | |
return 1 | |
if limit is None: | |
limit = -1 | |
elif not isinstance(limit, (int, long)): | |
raise TypeError("limit must be an integer") | |
res = bytearray() | |
while limit < 0 or len(res) < limit: | |
b = self.read(nreadahead()) | |
if not b: | |
break | |
res += b | |
if res.endswith(b"\n"): | |
break | |
return bytes(res) | |
def __iter__(self): | |
self._checkClosed() | |
return self | |
def next(self): | |
line = self.readline() | |
if not line: | |
raise StopIteration | |
return line | |
def readlines(self, hint=None): | |
"""Return a list of lines from the stream. | |
hint can be specified to control the number of lines read: no more | |
lines will be read if the total size (in bytes/characters) of all | |
lines so far exceeds hint. | |
""" | |
if hint is not None and not isinstance(hint, (int, long)): | |
raise TypeError("integer or None expected") | |
if hint is None or hint <= 0: | |
return list(self) | |
n = 0 | |
lines = [] | |
for line in self: | |
lines.append(line) | |
n += len(line) | |
if n >= hint: | |
break | |
return lines | |
def writelines(self, lines): | |
self._checkClosed() | |
for line in lines: | |
self.write(line) | |
io.IOBase.register(IOBase) | |
class RawIOBase(IOBase): | |
"""Base class for raw binary I/O.""" | |
# The read() method is implemented by calling readinto(); derived | |
# classes that want to support read() only need to implement | |
# readinto() as a primitive operation. In general, readinto() can be | |
# more efficient than read(). | |
# (It would be tempting to also provide an implementation of | |
# readinto() in terms of read(), in case the latter is a more suitable | |
# primitive operation, but that would lead to nasty recursion in case | |
# a subclass doesn't implement either.) | |
def read(self, n=-1): | |
"""Read and return up to n bytes. | |
Returns an empty bytes object on EOF, or None if the object is | |
set not to block and has no data to read. | |
""" | |
if n is None: | |
n = -1 | |
if n < 0: | |
return self.readall() | |
b = bytearray(n.__index__()) | |
n = self.readinto(b) | |
if n is None: | |
return None | |
del b[n:] | |
return bytes(b) | |
def readall(self): | |
"""Read until EOF, using multiple read() call.""" | |
res = bytearray() | |
while True: | |
data = self.read(DEFAULT_BUFFER_SIZE) | |
if not data: | |
break | |
res += data | |
if res: | |
return bytes(res) | |
else: | |
# b'' or None | |
return data | |
def readinto(self, b): | |
"""Read up to len(b) bytes into b. | |
Returns number of bytes read (0 for EOF), or None if the object | |
is set not to block and has no data to read. | |
""" | |
self._unsupported("readinto") | |
def write(self, b): | |
"""Write the given buffer to the IO stream. | |
Returns the number of bytes written, which may be less than len(b). | |
""" | |
self._unsupported("write") | |
io.RawIOBase.register(RawIOBase) | |
from _io import FileIO | |
RawIOBase.register(FileIO) | |
class BufferedIOBase(IOBase): | |
"""Base class for buffered IO objects. | |
The main difference with RawIOBase is that the read() method | |
supports omitting the size argument, and does not have a default | |
implementation that defers to readinto(). | |
In addition, read(), readinto() and write() may raise | |
BlockingIOError if the underlying raw stream is in non-blocking | |
mode and not ready; unlike their raw counterparts, they will never | |
return None. | |
A typical implementation should not inherit from a RawIOBase | |
implementation, but wrap one. | |
""" | |
def read(self, n=None): | |
"""Read and return up to n bytes. | |
If the argument is omitted, None, or negative, reads and | |
returns all data until EOF. | |
If the argument is positive, and the underlying raw stream is | |
not 'interactive', multiple raw reads may be issued to satisfy | |
the byte count (unless EOF is reached first). But for | |
interactive raw streams (XXX and for pipes?), at most one raw | |
read will be issued, and a short result does not imply that | |
EOF is imminent. | |
Returns an empty bytes array on EOF. | |
Raises BlockingIOError if the underlying raw stream has no | |
data at the moment. | |
""" | |
self._unsupported("read") | |
def read1(self, n=None): | |
"""Read up to n bytes with at most one read() system call.""" | |
self._unsupported("read1") | |
def readinto(self, b): | |
"""Read up to len(b) bytes into b. | |
Like read(), this may issue multiple reads to the underlying raw | |
stream, unless the latter is 'interactive'. | |
Returns the number of bytes read (0 for EOF). | |
Raises BlockingIOError if the underlying raw stream has no | |
data at the moment. | |
""" | |
# XXX This ought to work with anything that supports the buffer API | |
data = self.read(len(b)) | |
n = len(data) | |
try: | |
b[:n] = data | |
except TypeError as err: | |
import array | |
if not isinstance(b, array.array): | |
raise err | |
b[:n] = array.array(b'b', data) | |
return n | |
def write(self, b): | |
"""Write the given buffer to the IO stream. | |
Return the number of bytes written, which is never less than | |
len(b). | |
Raises BlockingIOError if the buffer is full and the | |
underlying raw stream cannot accept more data at the moment. | |
""" | |
self._unsupported("write") | |
def detach(self): | |
""" | |
Separate the underlying raw stream from the buffer and return it. | |
After the raw stream has been detached, the buffer is in an unusable | |
state. | |
""" | |
self._unsupported("detach") | |
io.BufferedIOBase.register(BufferedIOBase) | |
class _BufferedIOMixin(BufferedIOBase): | |
"""A mixin implementation of BufferedIOBase with an underlying raw stream. | |
This passes most requests on to the underlying raw stream. It | |
does *not* provide implementations of read(), readinto() or | |
write(). | |
""" | |
def __init__(self, raw): | |
self._raw = raw | |
### Positioning ### | |
def seek(self, pos, whence=0): | |
new_position = self.raw.seek(pos, whence) | |
if new_position < 0: | |
raise IOError("seek() returned an invalid position") | |
return new_position | |
def tell(self): | |
pos = self.raw.tell() | |
if pos < 0: | |
raise IOError("tell() returned an invalid position") | |
return pos | |
def truncate(self, pos=None): | |
# Flush the stream. We're mixing buffered I/O with lower-level I/O, | |
# and a flush may be necessary to synch both views of the current | |
# file state. | |
self.flush() | |
if pos is None: | |
pos = self.tell() | |
# XXX: Should seek() be used, instead of passing the position | |
# XXX directly to truncate? | |
return self.raw.truncate(pos) | |
### Flush and close ### | |
def flush(self): | |
if self.closed: | |
raise ValueError("flush of closed file") | |
self.raw.flush() | |
def close(self): | |
if self.raw is not None and not self.closed: | |
self.flush() | |
self.raw.close() | |
def detach(self): | |
if self.raw is None: | |
raise ValueError("raw stream already detached") | |
self.flush() | |
raw = self._raw | |
self._raw = None | |
return raw | |
### Inquiries ### | |
def seekable(self): | |
return self.raw.seekable() | |
def readable(self): | |
return self.raw.readable() | |
def writable(self): | |
return self.raw.writable() | |
@property | |
def raw(self): | |
return self._raw | |
@property | |
def closed(self): | |
return self.raw.closed | |
@property | |
def name(self): | |
return self.raw.name | |
@property | |
def mode(self): | |
return self.raw.mode | |
def __repr__(self): | |
clsname = self.__class__.__name__ | |
try: | |
name = self.name | |
except AttributeError: | |
return "<_pyio.{0}>".format(clsname) | |
else: | |
return "<_pyio.{0} name={1!r}>".format(clsname, name) | |
### Lower-level APIs ### | |
def fileno(self): | |
return self.raw.fileno() | |
def isatty(self): | |
return self.raw.isatty() | |
class BytesIO(BufferedIOBase): | |
"""Buffered I/O implementation using an in-memory bytes buffer.""" | |
def __init__(self, initial_bytes=None): | |
buf = bytearray() | |
if initial_bytes is not None: | |
buf.extend(initial_bytes) | |
self._buffer = buf | |
self._pos = 0 | |
def __getstate__(self): | |
if self.closed: | |
raise ValueError("__getstate__ on closed file") | |
return self.__dict__.copy() | |
def getvalue(self): | |
"""Return the bytes value (contents) of the buffer | |
""" | |
if self.closed: | |
raise ValueError("getvalue on closed file") | |
return bytes(self._buffer) | |
def read(self, n=None): | |
if self.closed: | |
raise ValueError("read from closed file") | |
if n is None: | |
n = -1 | |
if not isinstance(n, (int, long)): | |
raise TypeError("integer argument expected, got {0!r}".format( | |
type(n))) | |
if n < 0: | |
n = len(self._buffer) | |
if len(self._buffer) <= self._pos: | |
return b"" | |
newpos = min(len(self._buffer), self._pos + n) | |
b = self._buffer[self._pos : newpos] | |
self._pos = newpos | |
return bytes(b) | |
def read1(self, n): | |
"""This is the same as read. | |
""" | |
return self.read(n) | |
def write(self, b): | |
if self.closed: | |
raise ValueError("write to closed file") | |
if isinstance(b, unicode): | |
raise TypeError("can't write unicode to binary stream") | |
n = len(b) | |
if n == 0: | |
return 0 | |
pos = self._pos | |
if pos > len(self._buffer): | |
# Inserts null bytes between the current end of the file | |
# and the new write position. | |
padding = b'\x00' * (pos - len(self._buffer)) | |
self._buffer += padding | |
self._buffer[pos:pos + n] = b | |
self._pos += n | |
return n | |
def seek(self, pos, whence=0): | |
if self.closed: | |
raise ValueError("seek on closed file") | |
try: | |
pos.__index__ | |
except AttributeError: | |
raise TypeError("an integer is required") | |
if whence == 0: | |
if pos < 0: | |
raise ValueError("negative seek position %r" % (pos,)) | |
self._pos = pos | |
elif whence == 1: | |
self._pos = max(0, self._pos + pos) | |
elif whence == 2: | |
self._pos = max(0, len(self._buffer) + pos) | |
else: | |
raise ValueError("invalid whence value") | |
return self._pos | |
def tell(self): | |
if self.closed: | |
raise ValueError("tell on closed file") | |
return self._pos | |
def truncate(self, pos=None): | |
if self.closed: | |
raise ValueError("truncate on closed file") | |
if pos is None: | |
pos = self._pos | |
else: | |
try: | |
pos.__index__ | |
except AttributeError: | |
raise TypeError("an integer is required") | |
if pos < 0: | |
raise ValueError("negative truncate position %r" % (pos,)) | |
del self._buffer[pos:] | |
return pos | |
def readable(self): | |
return True | |
def writable(self): | |
return True | |
def seekable(self): | |
return True | |
class BufferedReader(_BufferedIOMixin): | |
"""BufferedReader(raw[, buffer_size]) | |
A buffer for a readable, sequential BaseRawIO object. | |
The constructor creates a BufferedReader for the given readable raw | |
stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE | |
is used. | |
""" | |
def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): | |
"""Create a new buffered reader using the given readable raw IO object. | |
""" | |
if not raw.readable(): | |
raise IOError('"raw" argument must be readable.') | |
_BufferedIOMixin.__init__(self, raw) | |
if buffer_size <= 0: | |
raise ValueError("invalid buffer size") | |
self.buffer_size = buffer_size | |
self._reset_read_buf() | |
self._read_lock = Lock() | |
def _reset_read_buf(self): | |
self._read_buf = b"" | |
self._read_pos = 0 | |
def read(self, n=None): | |
"""Read n bytes. | |
Returns exactly n bytes of data unless the underlying raw IO | |
stream reaches EOF or if the call would block in non-blocking | |
mode. If n is negative, read until EOF or until read() would | |
block. | |
""" | |
if n is not None and n < -1: | |
raise ValueError("invalid number of bytes to read") | |
with self._read_lock: | |
return self._read_unlocked(n) | |
def _read_unlocked(self, n=None): | |
nodata_val = b"" | |
empty_values = (b"", None) | |
buf = self._read_buf | |
pos = self._read_pos | |
# Special case for when the number of bytes to read is unspecified. | |
if n is None or n == -1: | |
self._reset_read_buf() | |
chunks = [buf[pos:]] # Strip the consumed bytes. | |
current_size = 0 | |
while True: | |
# Read until EOF or until read() would block. | |
try: | |
chunk = self.raw.read() | |
except IOError as e: | |
if e.errno != EINTR: | |
raise | |
continue | |
if chunk in empty_values: | |
nodata_val = chunk | |
break | |
current_size += len(chunk) | |
chunks.append(chunk) | |
return b"".join(chunks) or nodata_val | |
# The number of bytes to read is specified, return at most n bytes. | |
avail = len(buf) - pos # Length of the available buffered data. | |
if n <= avail: | |
# Fast path: the data to read is fully buffered. | |
self._read_pos += n | |
return buf[pos:pos+n] | |
# Slow path: read from the stream until enough bytes are read, | |
# or until an EOF occurs or until read() would block. | |
chunks = [buf[pos:]] | |
wanted = max(self.buffer_size, n) | |
while avail < n: | |
try: | |
chunk = self.raw.read(wanted) | |
except IOError as e: | |
if e.errno != EINTR: | |
raise | |
continue | |
if chunk in empty_values: | |
nodata_val = chunk | |
break | |
avail += len(chunk) | |
chunks.append(chunk) | |
# n is more then avail only when an EOF occurred or when | |
# read() would have blocked. | |
n = min(n, avail) | |
out = b"".join(chunks) | |
self._read_buf = out[n:] # Save the extra data in the buffer. | |
self._read_pos = 0 | |
return out[:n] if out else nodata_val | |
def peek(self, n=0): | |
"""Returns buffered bytes without advancing the position. | |
The argument indicates a desired minimal number of bytes; we | |
do at most one raw read to satisfy it. We never return more | |
than self.buffer_size. | |
""" | |
with self._read_lock: | |
return self._peek_unlocked(n) | |
def _peek_unlocked(self, n=0): | |
want = min(n, self.buffer_size) | |
have = len(self._read_buf) - self._read_pos | |
if have < want or have <= 0: | |
to_read = self.buffer_size - have | |
while True: | |
try: | |
current = self.raw.read(to_read) | |
except IOError as e: | |
if e.errno != EINTR: | |
raise | |
continue | |
break | |
if current: | |
self._read_buf = self._read_buf[self._read_pos:] + current | |
self._read_pos = 0 | |
return self._read_buf[self._read_pos:] | |
def read1(self, n): | |
"""Reads up to n bytes, with at most one read() system call.""" | |
# Returns up to n bytes. If at least one byte is buffered, we | |
# only return buffered bytes. Otherwise, we do one raw read. | |
if n < 0: | |
raise ValueError("number of bytes to read must be positive") | |
if n == 0: | |
return b"" | |
with self._read_lock: | |
self._peek_unlocked(1) | |
return self._read_unlocked( | |
min(n, len(self._read_buf) - self._read_pos)) | |
def tell(self): | |
return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos | |
def seek(self, pos, whence=0): | |
if not (0 <= whence <= 2): | |
raise ValueError("invalid whence value") | |
with self._read_lock: | |
if whence == 1: | |
pos -= len(self._read_buf) - self._read_pos | |
pos = _BufferedIOMixin.seek(self, pos, whence) | |
self._reset_read_buf() | |
return pos | |
class BufferedWriter(_BufferedIOMixin): | |
"""A buffer for a writeable sequential RawIO object. | |
The constructor creates a BufferedWriter for the given writeable raw | |
stream. If the buffer_size is not given, it defaults to | |
DEFAULT_BUFFER_SIZE. | |
""" | |
_warning_stack_offset = 2 | |
def __init__(self, raw, | |
buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): | |
if not raw.writable(): | |
raise IOError('"raw" argument must be writable.') | |
_BufferedIOMixin.__init__(self, raw) | |
if buffer_size <= 0: | |
raise ValueError("invalid buffer size") | |
if max_buffer_size is not None: | |
warnings.warn("max_buffer_size is deprecated", DeprecationWarning, | |
self._warning_stack_offset) | |
self.buffer_size = buffer_size | |
self._write_buf = bytearray() | |
self._write_lock = Lock() | |
def write(self, b): | |
if self.closed: | |
raise ValueError("write to closed file") | |
if isinstance(b, unicode): | |
raise TypeError("can't write unicode to binary stream") | |
with self._write_lock: | |
# XXX we can implement some more tricks to try and avoid | |
# partial writes | |
if len(self._write_buf) > self.buffer_size: | |
# We're full, so let's pre-flush the buffer | |
try: | |
self._flush_unlocked() | |
except BlockingIOError as e: | |
# We can't accept anything else. | |
# XXX Why not just let the exception pass through? | |
raise BlockingIOError(e.errno, e.strerror, 0) | |
before = len(self._write_buf) | |
self._write_buf.extend(b) | |
written = len(self._write_buf) - before | |
if len(self._write_buf) > self.buffer_size: | |
try: | |
self._flush_unlocked() | |
except BlockingIOError as e: | |
if len(self._write_buf) > self.buffer_size: | |
# We've hit the buffer_size. We have to accept a partial | |
# write and cut back our buffer. | |
overage = len(self._write_buf) - self.buffer_size | |
written -= overage | |
self._write_buf = self._write_buf[:self.buffer_size] | |
raise BlockingIOError(e.errno, e.strerror, written) | |
return written | |
def truncate(self, pos=None): | |
with self._write_lock: | |
self._flush_unlocked() | |
if pos is None: | |
pos = self.raw.tell() | |
return self.raw.truncate(pos) | |
def flush(self): | |
with self._write_lock: | |
self._flush_unlocked() | |
def _flush_unlocked(self): | |
if self.closed: | |
raise ValueError("flush of closed file") | |
written = 0 | |
try: | |
while self._write_buf: | |
try: | |
n = self.raw.write(self._write_buf) | |
except IOError as e: | |
if e.errno != EINTR: | |
raise | |
continue | |
if n > len(self._write_buf) or n < 0: | |
raise IOError("write() returned incorrect number of bytes") | |
del self._write_buf[:n] | |
written += n | |
except BlockingIOError as e: | |
n = e.characters_written | |
del self._write_buf[:n] | |
written += n | |
raise BlockingIOError(e.errno, e.strerror, written) | |
def tell(self): | |
return _BufferedIOMixin.tell(self) + len(self._write_buf) | |
def seek(self, pos, whence=0): | |
if not (0 <= whence <= 2): | |
raise ValueError("invalid whence") | |
with self._write_lock: | |
self._flush_unlocked() | |
return _BufferedIOMixin.seek(self, pos, whence) | |
class BufferedRWPair(BufferedIOBase): | |
"""A buffered reader and writer object together. | |
A buffered reader object and buffered writer object put together to | |
form a sequential IO object that can read and write. This is typically | |
used with a socket or two-way pipe. | |
reader and writer are RawIOBase objects that are readable and | |
writeable respectively. If the buffer_size is omitted it defaults to | |
DEFAULT_BUFFER_SIZE. | |
""" | |
# XXX The usefulness of this (compared to having two separate IO | |
# objects) is questionable. | |
def __init__(self, reader, writer, | |
buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): | |
"""Constructor. | |
The arguments are two RawIO instances. | |
""" | |
if max_buffer_size is not None: | |
warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2) | |
if not reader.readable(): | |
raise IOError('"reader" argument must be readable.') | |
if not writer.writable(): | |
raise IOError('"writer" argument must be writable.') | |
self.reader = BufferedReader(reader, buffer_size) | |
self.writer = BufferedWriter(writer, buffer_size) | |
def read(self, n=None): | |
if n is None: | |
n = -1 | |
return self.reader.read(n) | |
def readinto(self, b): | |
return self.reader.readinto(b) | |
def write(self, b): | |
return self.writer.write(b) | |
def peek(self, n=0): | |
return self.reader.peek(n) | |
def read1(self, n): | |
return self.reader.read1(n) | |
def readable(self): | |
return self.reader.readable() | |
def writable(self): | |
return self.writer.writable() | |
def flush(self): | |
return self.writer.flush() | |
def close(self): | |
self.writer.close() | |
self.reader.close() | |
def isatty(self): | |
return self.reader.isatty() or self.writer.isatty() | |
@property | |
def closed(self): | |
return self.writer.closed | |
class BufferedRandom(BufferedWriter, BufferedReader): | |
"""A buffered interface to random access streams. | |
The constructor creates a reader and writer for a seekable stream, | |
raw, given in the first argument. If the buffer_size is omitted it | |
defaults to DEFAULT_BUFFER_SIZE. | |
""" | |
_warning_stack_offset = 3 | |
def __init__(self, raw, | |
buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): | |
raw._checkSeekable() | |
BufferedReader.__init__(self, raw, buffer_size) | |
BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size) | |
def seek(self, pos, whence=0): | |
if not (0 <= whence <= 2): | |
raise ValueError("invalid whence") | |
self.flush() | |
if self._read_buf: | |
# Undo read ahead. | |
with self._read_lock: | |
self.raw.seek(self._read_pos - len(self._read_buf), 1) | |
# First do the raw seek, then empty the read buffer, so that | |
# if the raw seek fails, we don't lose buffered data forever. | |
pos = self.raw.seek(pos, whence) | |
with self._read_lock: | |
self._reset_read_buf() | |
if pos < 0: | |
raise IOError("seek() returned invalid position") | |
return pos | |
def tell(self): | |
if self._write_buf: | |
return BufferedWriter.tell(self) | |
else: | |
return BufferedReader.tell(self) | |
def truncate(self, pos=None): | |
if pos is None: | |
pos = self.tell() | |
# Use seek to flush the read buffer. | |
return BufferedWriter.truncate(self, pos) | |
def read(self, n=None): | |
if n is None: | |
n = -1 | |
self.flush() | |
return BufferedReader.read(self, n) | |
def readinto(self, b): | |
self.flush() | |
return BufferedReader.readinto(self, b) | |
def peek(self, n=0): | |
self.flush() | |
return BufferedReader.peek(self, n) | |
def read1(self, n): | |
self.flush() | |
return BufferedReader.read1(self, n) | |
def write(self, b): | |
if self._read_buf: | |
# Undo readahead | |
with self._read_lock: | |
self.raw.seek(self._read_pos - len(self._read_buf), 1) | |
self._reset_read_buf() | |
return BufferedWriter.write(self, b) | |
class TextIOBase(IOBase): | |
"""Base class for text I/O. | |
This class provides a character and line based interface to stream | |
I/O. There is no readinto method because Python's character strings | |
are immutable. There is no public constructor. | |
""" | |
def read(self, n=-1): | |
"""Read at most n characters from stream. | |
Read from underlying buffer until we have n characters or we hit EOF. | |
If n is negative or omitted, read until EOF. | |
""" | |
self._unsupported("read") | |
def write(self, s): | |
"""Write string s to stream.""" | |
self._unsupported("write") | |
def truncate(self, pos=None): | |
"""Truncate size to pos.""" | |
self._unsupported("truncate") | |
def readline(self): | |
"""Read until newline or EOF. | |
Returns an empty string if EOF is hit immediately. | |
""" | |
self._unsupported("readline") | |
def detach(self): | |
""" | |
Separate the underlying buffer from the TextIOBase and return it. | |
After the underlying buffer has been detached, the TextIO is in an | |
unusable state. | |
""" | |
self._unsupported("detach") | |
@property | |
def encoding(self): | |
"""Subclasses should override.""" | |
return None | |
@property | |
def newlines(self): | |
"""Line endings translated so far. | |
Only line endings translated during reading are considered. | |
Subclasses should override. | |
""" | |
return None | |
@property | |
def errors(self): | |
"""Error setting of the decoder or encoder. | |
Subclasses should override.""" | |
return None | |
io.TextIOBase.register(TextIOBase) | |
class IncrementalNewlineDecoder(codecs.IncrementalDecoder): | |
r"""Codec used when reading a file in universal newlines mode. It wraps | |
another incremental decoder, translating \r\n and \r into \n. It also | |
records the types of newlines encountered. When used with | |
translate=False, it ensures that the newline sequence is returned in | |
one piece. | |
""" | |
def __init__(self, decoder, translate, errors='strict'): | |
codecs.IncrementalDecoder.__init__(self, errors=errors) | |
self.translate = translate | |
self.decoder = decoder | |
self.seennl = 0 | |
self.pendingcr = False | |
def decode(self, input, final=False): | |
# decode input (with the eventual \r from a previous pass) | |
if self.decoder is None: | |
output = input | |
else: | |
output = self.decoder.decode(input, final=final) | |
if self.pendingcr and (output or final): | |
output = "\r" + output | |
self.pendingcr = False | |
# retain last \r even when not translating data: | |
# then readline() is sure to get \r\n in one pass | |
if output.endswith("\r") and not final: | |
output = output[:-1] | |
self.pendingcr = True | |
# Record which newlines are read | |
crlf = output.count('\r\n') | |
cr = output.count('\r') - crlf | |
lf = output.count('\n') - crlf | |
self.seennl |= (lf and self._LF) | (cr and self._CR) \ | |
| (crlf and self._CRLF) | |
if self.translate: | |
if crlf: | |
output = output.replace("\r\n", "\n") | |
if cr: | |
output = output.replace("\r", "\n") | |
return output | |
def getstate(self): | |
if self.decoder is None: | |
buf = b"" | |
flag = 0 | |
else: | |
buf, flag = self.decoder.getstate() | |
flag <<= 1 | |
if self.pendingcr: | |
flag |= 1 | |
return buf, flag | |
def setstate(self, state): | |
buf, flag = state | |
self.pendingcr = bool(flag & 1) | |
if self.decoder is not None: | |
self.decoder.setstate((buf, flag >> 1)) | |
def reset(self): | |
self.seennl = 0 | |
self.pendingcr = False | |
if self.decoder is not None: | |
self.decoder.reset() | |
_LF = 1 | |
_CR = 2 | |
_CRLF = 4 | |
@property | |
def newlines(self): | |
return (None, | |
"\n", | |
"\r", | |
("\r", "\n"), | |
"\r\n", | |
("\n", "\r\n"), | |
("\r", "\r\n"), | |
("\r", "\n", "\r\n") | |
)[self.seennl] | |
class TextIOWrapper(TextIOBase): | |
r"""Character and line based layer over a BufferedIOBase object, buffer. | |
encoding gives the name of the encoding that the stream will be | |
decoded or encoded with. It defaults to locale.getpreferredencoding. | |
errors determines the strictness of encoding and decoding (see the | |
codecs.register) and defaults to "strict". | |
newline can be None, '', '\n', '\r', or '\r\n'. It controls the | |
handling of line endings. If it is None, universal newlines is | |
enabled. With this enabled, on input, the lines endings '\n', '\r', | |
or '\r\n' are translated to '\n' before being returned to the | |
caller. Conversely, on output, '\n' is translated to the system | |
default line seperator, os.linesep. If newline is any other of its | |
legal values, that newline becomes the newline when the file is read | |
and it is returned untranslated. On output, '\n' is converted to the | |
newline. | |
If line_buffering is True, a call to flush is implied when a call to | |
write contains a newline character. | |
""" | |
_CHUNK_SIZE = 2048 | |
def __init__(self, buffer, encoding=None, errors=None, newline=None, | |
line_buffering=False): | |
if newline is not None and not isinstance(newline, basestring): | |
raise TypeError("illegal newline type: %r" % (type(newline),)) | |
if newline not in (None, "", "\n", "\r", "\r\n"): | |
raise ValueError("illegal newline value: %r" % (newline,)) | |
if encoding is None: | |
try: | |
import locale | |
except ImportError: | |
# Importing locale may fail if Python is being built | |
encoding = "ascii" | |
else: | |
encoding = locale.getpreferredencoding() | |
if not isinstance(encoding, basestring): | |
raise ValueError("invalid encoding: %r" % encoding) | |
if errors is None: | |
errors = "strict" | |
else: | |
if not isinstance(errors, basestring): | |
raise ValueError("invalid errors: %r" % errors) | |
self._buffer = buffer | |
self._line_buffering = line_buffering | |
self._encoding = encoding | |
self._errors = errors | |
self._readuniversal = not newline | |
self._readtranslate = newline is None | |
self._readnl = newline | |
self._writetranslate = newline != '' | |
self._writenl = newline or os.linesep | |
self._encoder = None | |
self._decoder = None | |
self._decoded_chars = '' # buffer for text returned from decoder | |
self._decoded_chars_used = 0 # offset into _decoded_chars for read() | |
self._snapshot = None # info for reconstructing decoder state | |
self._seekable = self._telling = self.buffer.seekable() | |
if self._seekable and self.writable(): | |
position = self.buffer.tell() | |
if position != 0: | |
try: | |
self._get_encoder().setstate(0) | |
except LookupError: | |
# Sometimes the encoder doesn't exist | |
pass | |
# self._snapshot is either None, or a tuple (dec_flags, next_input) | |
# where dec_flags is the second (integer) item of the decoder state | |
# and next_input is the chunk of input bytes that comes next after the | |
# snapshot point. We use this to reconstruct decoder states in tell(). | |
# Naming convention: | |
# - "bytes_..." for integer variables that count input bytes | |
# - "chars_..." for integer variables that count decoded characters | |
def __repr__(self): | |
try: | |
name = self.name | |
except AttributeError: | |
return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding) | |
else: | |
return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format( | |
name, self.encoding) | |
@property | |
def encoding(self): | |
return self._encoding | |
@property | |
def errors(self): | |
return self._errors | |
@property | |
def line_buffering(self): | |
return self._line_buffering | |
@property | |
def buffer(self): | |
return self._buffer | |
def seekable(self): | |
return self._seekable | |
def readable(self): | |
return self.buffer.readable() | |
def writable(self): | |
return self.buffer.writable() | |
def flush(self): | |
self.buffer.flush() | |
self._telling = self._seekable | |
def close(self): | |
if self.buffer is not None and not self.closed: | |
self.flush() | |
self.buffer.close() | |
@property | |
def closed(self): | |
return self.buffer.closed | |
@property | |
def name(self): | |
return self.buffer.name | |
def fileno(self): | |
return self.buffer.fileno() | |
def isatty(self): | |
return self.buffer.isatty() | |
def write(self, s): | |
if self.closed: | |
raise ValueError("write to closed file") | |
if not isinstance(s, unicode): | |
raise TypeError("can't write %s to text stream" % | |
s.__class__.__name__) | |
length = len(s) | |
haslf = (self._writetranslate or self._line_buffering) and "\n" in s | |
if haslf and self._writetranslate and self._writenl != "\n": | |
s = s.replace("\n", self._writenl) | |
encoder = self._encoder or self._get_encoder() | |
# XXX What if we were just reading? | |
b = encoder.encode(s) | |
self.buffer.write(b) | |
if self._line_buffering and (haslf or "\r" in s): | |
self.flush() | |
self._snapshot = None | |
if self._decoder: | |
self._decoder.reset() | |
return length | |
def _get_encoder(self): | |
make_encoder = codecs.getincrementalencoder(self._encoding) | |
self._encoder = make_encoder(self._errors) | |
return self._encoder | |
def _get_decoder(self): | |
make_decoder = codecs.getincrementaldecoder(self._encoding) | |
decoder = make_decoder(self._errors) | |
if self._readuniversal: | |
decoder = IncrementalNewlineDecoder(decoder, self._readtranslate) | |
self._decoder = decoder | |
return decoder | |
# The following three methods implement an ADT for _decoded_chars. | |
# Text returned from the decoder is buffered here until the client | |
# requests it by calling our read() or readline() method. | |
def _set_decoded_chars(self, chars): | |
"""Set the _decoded_chars buffer.""" | |
self._decoded_chars = chars | |
self._decoded_chars_used = 0 | |
def _get_decoded_chars(self, n=None): | |
"""Advance into the _decoded_chars buffer.""" | |
offset = self._decoded_chars_used | |
if n is None: | |
chars = self._decoded_chars[offset:] | |
else: | |
chars = self._decoded_chars[offset:offset + n] | |
self._decoded_chars_used += len(chars) | |
return chars | |
def _rewind_decoded_chars(self, n): | |
"""Rewind the _decoded_chars buffer.""" | |
if self._decoded_chars_used < n: | |
raise AssertionError("rewind decoded_chars out of bounds") | |
self._decoded_chars_used -= n | |
def _read_chunk(self): | |
""" | |
Read and decode the next chunk of data from the BufferedReader. | |
""" | |
# The return value is True unless EOF was reached. The decoded | |
# string is placed in self._decoded_chars (replacing its previous | |
# value). The entire input chunk is sent to the decoder, though | |
# some of it may remain buffered in the decoder, yet to be | |
# converted. | |
if self._decoder is None: | |
raise ValueError("no decoder") | |
if self._telling: | |
# To prepare for tell(), we need to snapshot a point in the | |
# file where the decoder's input buffer is empty. | |
dec_buffer, dec_flags = self._decoder.getstate() | |
# Given this, we know there was a valid snapshot point | |
# len(dec_buffer) bytes ago with decoder state (b'', dec_flags). | |
# Read a chunk, decode it, and put the result in self._decoded_chars. | |
input_chunk = self.buffer.read1(self._CHUNK_SIZE) | |
eof = not input_chunk | |
self._set_decoded_chars(self._decoder.decode(input_chunk, eof)) | |
if self._telling: | |
# At the snapshot point, len(dec_buffer) bytes before the read, | |
# the next input to be decoded is dec_buffer + input_chunk. | |
self._snapshot = (dec_flags, dec_buffer + input_chunk) | |
return not eof | |
def _pack_cookie(self, position, dec_flags=0, | |
bytes_to_feed=0, need_eof=0, chars_to_skip=0): | |
# The meaning of a tell() cookie is: seek to position, set the | |
# decoder flags to dec_flags, read bytes_to_feed bytes, feed them | |
# into the decoder with need_eof as the EOF flag, then skip | |
# chars_to_skip characters of the decoded result. For most simple | |
# decoders, tell() will often just give a byte offset in the file. | |
return (position | (dec_flags<<64) | (bytes_to_feed<<128) | | |
(chars_to_skip<<192) | bool(need_eof)<<256) | |
def _unpack_cookie(self, bigint): | |
rest, position = divmod(bigint, 1<<64) | |
rest, dec_flags = divmod(rest, 1<<64) | |
rest, bytes_to_feed = divmod(rest, 1<<64) | |
need_eof, chars_to_skip = divmod(rest, 1<<64) | |
return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip | |
def tell(self): | |
if not self._seekable: | |
raise IOError("underlying stream is not seekable") | |
if not self._telling: | |
raise IOError("telling position disabled by next() call") | |
self.flush() | |
position = self.buffer.tell() | |
decoder = self._decoder | |
if decoder is None or self._snapshot is None: | |
if self._decoded_chars: | |
# This should never happen. | |
raise AssertionError("pending decoded text") | |
return position | |
# Skip backward to the snapshot point (see _read_chunk). | |
dec_flags, next_input = self._snapshot | |
position -= len(next_input) | |
# How many decoded characters have been used up since the snapshot? | |
chars_to_skip = self._decoded_chars_used | |
if chars_to_skip == 0: | |
# We haven't moved from the snapshot point. | |
return self._pack_cookie(position, dec_flags) | |
# Starting from the snapshot position, we will walk the decoder | |
# forward until it gives us enough decoded characters. | |
saved_state = decoder.getstate() | |
try: | |
# Note our initial start point. | |
decoder.setstate((b'', dec_flags)) | |
start_pos = position | |
start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 | |
need_eof = 0 | |
# Feed the decoder one byte at a time. As we go, note the | |
# nearest "safe start point" before the current location | |
# (a point where the decoder has nothing buffered, so seek() | |
# can safely start from there and advance to this location). | |
for next_byte in next_input: | |
bytes_fed += 1 | |
chars_decoded += len(decoder.decode(next_byte)) | |
dec_buffer, dec_flags = decoder.getstate() | |
if not dec_buffer and chars_decoded <= chars_to_skip: | |
# Decoder buffer is empty, so this is a safe start point. | |
start_pos += bytes_fed | |
chars_to_skip -= chars_decoded | |
start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 | |
if chars_decoded >= chars_to_skip: | |
break | |
else: | |
# We didn't get enough decoded data; signal EOF to get more. | |
chars_decoded += len(decoder.decode(b'', final=True)) | |
need_eof = 1 | |
if chars_decoded < chars_to_skip: | |
raise IOError("can't reconstruct logical file position") | |
# The returned cookie corresponds to the last safe start point. | |
return self._pack_cookie( | |
start_pos, start_flags, bytes_fed, need_eof, chars_to_skip) | |
finally: | |
decoder.setstate(saved_state) | |
def truncate(self, pos=None): | |
self.flush() | |
if pos is None: | |
pos = self.tell() | |
return self.buffer.truncate(pos) | |
def detach(self): | |
if self.buffer is None: | |
raise ValueError("buffer is already detached") | |
self.flush() | |
buffer = self._buffer | |
self._buffer = None | |
return buffer | |
def seek(self, cookie, whence=0): | |
if self.closed: | |
raise ValueError("tell on closed file") | |
if not self._seekable: | |
raise IOError("underlying stream is not seekable") | |
if whence == 1: # seek relative to current position | |
if cookie != 0: | |
raise IOError("can't do nonzero cur-relative seeks") | |
# Seeking to the current position should attempt to | |
# sync the underlying buffer with the current position. | |
whence = 0 | |
cookie = self.tell() | |
if whence == 2: # seek relative to end of file | |
if cookie != 0: | |
raise IOError("can't do nonzero end-relative seeks") | |
self.flush() | |
position = self.buffer.seek(0, 2) | |
self._set_decoded_chars('') | |
self._snapshot = None | |
if self._decoder: | |
self._decoder.reset() | |
return position | |
if whence != 0: | |
raise ValueError("invalid whence (%r, should be 0, 1 or 2)" % | |
(whence,)) | |
if cookie < 0: | |
raise ValueError("negative seek position %r" % (cookie,)) | |
self.flush() | |
# The strategy of seek() is to go back to the safe start point | |
# and replay the effect of read(chars_to_skip) from there. | |
start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \ | |
self._unpack_cookie(cookie) | |
# Seek back to the safe start point. | |
self.buffer.seek(start_pos) | |
self._set_decoded_chars('') | |
self._snapshot = None | |
# Restore the decoder to its state from the safe start point. | |
if cookie == 0 and self._decoder: | |
self._decoder.reset() | |
elif self._decoder or dec_flags or chars_to_skip: | |
self._decoder = self._decoder or self._get_decoder() | |
self._decoder.setstate((b'', dec_flags)) | |
self._snapshot = (dec_flags, b'') | |
if chars_to_skip: | |
# Just like _read_chunk, feed the decoder and save a snapshot. | |
input_chunk = self.buffer.read(bytes_to_feed) | |
self._set_decoded_chars( | |
self._decoder.decode(input_chunk, need_eof)) | |
self._snapshot = (dec_flags, input_chunk) | |
# Skip chars_to_skip of the decoded characters. | |
if len(self._decoded_chars) < chars_to_skip: | |
raise IOError("can't restore logical file position") | |
self._decoded_chars_used = chars_to_skip | |
# Finally, reset the encoder (merely useful for proper BOM handling) | |
try: | |
encoder = self._encoder or self._get_encoder() | |
except LookupError: | |
# Sometimes the encoder doesn't exist | |
pass | |
else: | |
if cookie != 0: | |
encoder.setstate(0) | |
else: | |
encoder.reset() | |
return cookie | |
def read(self, n=None): | |
self._checkReadable() | |
if n is None: | |
n = -1 | |
decoder = self._decoder or self._get_decoder() | |
try: | |
n.__index__ | |
except AttributeError: | |
raise TypeError("an integer is required") | |
if n < 0: | |
# Read everything. | |
result = (self._get_decoded_chars() + | |
decoder.decode(self.buffer.read(), final=True)) | |
self._set_decoded_chars('') | |
self._snapshot = None | |
return result | |
else: | |
# Keep reading chunks until we have n characters to return. | |
eof = False | |
result = self._get_decoded_chars(n) | |
while len(result) < n and not eof: | |
eof = not self._read_chunk() | |
result += self._get_decoded_chars(n - len(result)) | |
return result | |
def next(self): | |
self._telling = False | |
line = self.readline() | |
if not line: | |
self._snapshot = None | |
self._telling = self._seekable | |
raise StopIteration | |
return line | |
def readline(self, limit=None): | |
if self.closed: | |
raise ValueError("read from closed file") | |
if limit is None: | |
limit = -1 | |
elif not isinstance(limit, (int, long)): | |
raise TypeError("limit must be an integer") | |
# Grab all the decoded text (we will rewind any extra bits later). | |
line = self._get_decoded_chars() | |
start = 0 | |
# Make the decoder if it doesn't already exist. | |
if not self._decoder: | |
self._get_decoder() | |
pos = endpos = None | |
while True: | |
if self._readtranslate: | |
# Newlines are already translated, only search for \n | |
pos = line.find('\n', start) | |
if pos >= 0: | |
endpos = pos + 1 | |
break | |
else: | |
start = len(line) | |
elif self._readuniversal: | |
# Universal newline search. Find any of \r, \r\n, \n | |
# The decoder ensures that \r\n are not split in two pieces | |
# In C we'd look for these in parallel of course. | |
nlpos = line.find("\n", start) | |
crpos = line.find("\r", start) | |
if crpos == -1: | |
if nlpos == -1: | |
# Nothing found | |
start = len(line) | |
else: | |
# Found \n | |
endpos = nlpos + 1 | |
break | |
elif nlpos == -1: | |
# Found lone \r | |
endpos = crpos + 1 | |
break | |
elif nlpos < crpos: | |
# Found \n | |
endpos = nlpos + 1 | |
break | |
elif nlpos == crpos + 1: | |
# Found \r\n | |
endpos = crpos + 2 | |
break | |
else: | |
# Found \r | |
endpos = crpos + 1 | |
break | |
else: | |
# non-universal | |
pos = line.find(self._readnl) | |
if pos >= 0: | |
endpos = pos + len(self._readnl) | |
break | |
if limit >= 0 and len(line) >= limit: | |
endpos = limit # reached length limit | |
break | |
# No line ending seen yet - get more data' | |
while self._read_chunk(): | |
if self._decoded_chars: | |
break | |
if self._decoded_chars: | |
line += self._get_decoded_chars() | |
else: | |
# end of file | |
self._set_decoded_chars('') | |
self._snapshot = None | |
return line | |
if limit >= 0 and endpos > limit: | |
endpos = limit # don't exceed limit | |
# Rewind _decoded_chars to just after the line ending we found. | |
self._rewind_decoded_chars(len(line) - endpos) | |
return line[:endpos] | |
@property | |
def newlines(self): | |
return self._decoder.newlines if self._decoder else None | |
class StringIO(TextIOWrapper): | |
"""Text I/O implementation using an in-memory buffer. | |
The initial_value argument sets the value of object. The newline | |
argument is like the one of TextIOWrapper's constructor. | |
""" | |
def __init__(self, initial_value="", newline="\n"): | |
super(StringIO, self).__init__(BytesIO(), | |
encoding="utf-8", | |
errors="strict", | |
newline=newline) | |
# Issue #5645: make universal newlines semantics the same as in the | |
# C version, even under Windows. | |
if newline is None: | |
self._writetranslate = False | |
if initial_value: | |
if not isinstance(initial_value, unicode): | |
initial_value = unicode(initial_value) | |
self.write(initial_value) | |
self.seek(0) | |
def getvalue(self): | |
self.flush() | |
return self.buffer.getvalue().decode(self._encoding, self._errors) | |
def __repr__(self): | |
# TextIOWrapper tells the encoding in its repr. In StringIO, | |
# that's a implementation detail. | |
return object.__repr__(self) | |
@property | |
def errors(self): | |
return None | |
@property | |
def encoding(self): | |
return None | |
def detach(self): | |
# This doesn't make sense on StringIO. | |
self._unsupported("detach") |