| """A dumb and slow but simple dbm clone. |
| |
| For database spam, spam.dir contains the index (a text file), |
| spam.bak *may* contain a backup of the index (also a text file), |
| while spam.dat contains the data (a binary file). |
| |
| XXX TO DO: |
| |
| - seems to contain a bug when updating... |
| |
| - reclaim free space (currently, space once occupied by deleted or expanded |
| items is never reused) |
| |
| - support concurrent access (currently, if two processes take turns making |
| updates, they can mess up the index) |
| |
| - support efficient access to large databases (currently, the whole index |
| is read when the database is opened, and some updates rewrite the whole index) |
| |
| - support opening for read-only (flag = 'm') |
| |
| """ |
| |
| import os as _os |
| import __builtin__ |
| import UserDict |
| |
| _open = __builtin__.open |
| |
| _BLOCKSIZE = 512 |
| |
| error = IOError # For anydbm |
| |
| class _Database(UserDict.DictMixin): |
| |
| # The on-disk directory and data files can remain in mutually |
| # inconsistent states for an arbitrarily long time (see comments |
| # at the end of __setitem__). This is only repaired when _commit() |
| # gets called. One place _commit() gets called is from __del__(), |
| # and if that occurs at program shutdown time, module globals may |
| # already have gotten rebound to None. Since it's crucial that |
| # _commit() finish successfully, we can't ignore shutdown races |
| # here, and _commit() must not reference any globals. |
| _os = _os # for _commit() |
| _open = _open # for _commit() |
| |
| def __init__(self, filebasename, mode): |
| self._mode = mode |
| |
| # The directory file is a text file. Each line looks like |
| # "%r, (%d, %d)\n" % (key, pos, siz) |
| # where key is the string key, pos is the offset into the dat |
| # file of the associated value's first byte, and siz is the number |
| # of bytes in the associated value. |
| self._dirfile = filebasename + _os.extsep + 'dir' |
| |
| # The data file is a binary file pointed into by the directory |
| # file, and holds the values associated with keys. Each value |
| # begins at a _BLOCKSIZE-aligned byte offset, and is a raw |
| # binary 8-bit string value. |
| self._datfile = filebasename + _os.extsep + 'dat' |
| self._bakfile = filebasename + _os.extsep + 'bak' |
| |
| # The index is an in-memory dict, mirroring the directory file. |
| self._index = None # maps keys to (pos, siz) pairs |
| |
| # Mod by Jack: create data file if needed |
| try: |
| f = _open(self._datfile, 'r') |
| except IOError: |
| f = _open(self._datfile, 'w') |
| self._chmod(self._datfile) |
| f.close() |
| self._update() |
| |
| # Read directory file into the in-memory index dict. |
| def _update(self): |
| self._index = {} |
| try: |
| f = _open(self._dirfile) |
| except IOError: |
| pass |
| else: |
| for line in f: |
| line = line.rstrip() |
| key, pos_and_siz_pair = eval(line) |
| self._index[key] = pos_and_siz_pair |
| f.close() |
| |
| # Write the index dict to the directory file. The original directory |
| # file (if any) is renamed with a .bak extension first. If a .bak |
| # file currently exists, it's deleted. |
| def _commit(self): |
| # CAUTION: It's vital that _commit() succeed, and _commit() can |
| # be called from __del__(). Therefore we must never reference a |
| # global in this routine. |
| if self._index is None: |
| return # nothing to do |
| |
| try: |
| self._os.unlink(self._bakfile) |
| except self._os.error: |
| pass |
| |
| try: |
| self._os.rename(self._dirfile, self._bakfile) |
| except self._os.error: |
| pass |
| |
| f = self._open(self._dirfile, 'w') |
| self._chmod(self._dirfile) |
| for key, pos_and_siz_pair in self._index.iteritems(): |
| f.write("%r, %r\n" % (key, pos_and_siz_pair)) |
| f.close() |
| |
| sync = _commit |
| |
| def __getitem__(self, key): |
| pos, siz = self._index[key] # may raise KeyError |
| f = _open(self._datfile, 'rb') |
| f.seek(pos) |
| dat = f.read(siz) |
| f.close() |
| return dat |
| |
| # Append val to the data file, starting at a _BLOCKSIZE-aligned |
| # offset. The data file is first padded with NUL bytes (if needed) |
| # to get to an aligned offset. Return pair |
| # (starting offset of val, len(val)) |
| def _addval(self, val): |
| f = _open(self._datfile, 'rb+') |
| f.seek(0, 2) |
| pos = int(f.tell()) |
| npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE |
| f.write('\0'*(npos-pos)) |
| pos = npos |
| f.write(val) |
| f.close() |
| return (pos, len(val)) |
| |
| # Write val to the data file, starting at offset pos. The caller |
| # is responsible for ensuring that there's enough room starting at |
| # pos to hold val, without overwriting some other value. Return |
| # pair (pos, len(val)). |
| def _setval(self, pos, val): |
| f = _open(self._datfile, 'rb+') |
| f.seek(pos) |
| f.write(val) |
| f.close() |
| return (pos, len(val)) |
| |
| # key is a new key whose associated value starts in the data file |
| # at offset pos and with length siz. Add an index record to |
| # the in-memory index dict, and append one to the directory file. |
| def _addkey(self, key, pos_and_siz_pair): |
| self._index[key] = pos_and_siz_pair |
| f = _open(self._dirfile, 'a') |
| self._chmod(self._dirfile) |
| f.write("%r, %r\n" % (key, pos_and_siz_pair)) |
| f.close() |
| |
| def __setitem__(self, key, val): |
| if not type(key) == type('') == type(val): |
| raise TypeError, "keys and values must be strings" |
| if key not in self._index: |
| self._addkey(key, self._addval(val)) |
| else: |
| # See whether the new value is small enough to fit in the |
| # (padded) space currently occupied by the old value. |
| pos, siz = self._index[key] |
| oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE |
| newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE |
| if newblocks <= oldblocks: |
| self._index[key] = self._setval(pos, val) |
| else: |
| # The new value doesn't fit in the (padded) space used |
| # by the old value. The blocks used by the old value are |
| # forever lost. |
| self._index[key] = self._addval(val) |
| |
| # Note that _index may be out of synch with the directory |
| # file now: _setval() and _addval() don't update the directory |
| # file. This also means that the on-disk directory and data |
| # files are in a mutually inconsistent state, and they'll |
| # remain that way until _commit() is called. Note that this |
| # is a disaster (for the database) if the program crashes |
| # (so that _commit() never gets called). |
| |
| def __delitem__(self, key): |
| # The blocks used by the associated value are lost. |
| del self._index[key] |
| # XXX It's unclear why we do a _commit() here (the code always |
| # XXX has, so I'm not changing it). _setitem__ doesn't try to |
| # XXX keep the directory file in synch. Why should we? Or |
| # XXX why shouldn't __setitem__? |
| self._commit() |
| |
| def keys(self): |
| return self._index.keys() |
| |
| def has_key(self, key): |
| return key in self._index |
| |
| def __contains__(self, key): |
| return key in self._index |
| |
| def iterkeys(self): |
| return self._index.iterkeys() |
| __iter__ = iterkeys |
| |
| def __len__(self): |
| return len(self._index) |
| |
| def close(self): |
| self._commit() |
| self._index = self._datfile = self._dirfile = self._bakfile = None |
| |
| __del__ = close |
| |
| def _chmod (self, file): |
| if hasattr(self._os, 'chmod'): |
| self._os.chmod(file, self._mode) |
| |
| |
| def open(file, flag=None, mode=0666): |
| """Open the database file, filename, and return corresponding object. |
| |
| The flag argument, used to control how the database is opened in the |
| other DBM implementations, is ignored in the dumbdbm module; the |
| database is always opened for update, and will be created if it does |
| not exist. |
| |
| The optional mode argument is the UNIX mode of the file, used only when |
| the database has to be created. It defaults to octal code 0666 (and |
| will be modified by the prevailing umask). |
| |
| """ |
| # flag argument is currently ignored |
| |
| # Modify mode depending on the umask |
| try: |
| um = _os.umask(0) |
| _os.umask(um) |
| except AttributeError: |
| pass |
| else: |
| # Turn off any bits that are set in the umask |
| mode = mode & (~um) |
| |
| return _Database(file, mode) |
