WebKitTools/Scripts/webkitpy/autoinstall.py - platform/external/webkit - Git at Google

 # Copyright (c) 2009, Daniel Krech All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #  * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright
 # notice, this list of conditions and the following disclaimer in the
 # documentation and/or other materials provided with the distribution.
 #
 #  * Neither the name of the Daniel Krech nor the names of its
 # contributors may be used to endorse or promote products derived from
 # this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 """\
 package loader for auto installing Python packages.

 A package loader in the spirit of Zero Install that can be used to
 inject dependencies into the import process.


 To install::

     easy_install -U autoinstall

       or

     download, unpack, python setup.py install

       or

     try the bootstrap loader. See below.


 To use::

     # You can bind any package name to a URL pointing to something
     # that can be imported using the zipimporter.

     autoinstall.bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg")

     import pymarc

     print pymarc.__version__, pymarc.__file__


 Changelog::

 - added support for non top level packages.
 - cache files now use filename part from URL.
 - applied patch from Eric Seidel <eseidel@google.com> to add support
 for loading modules where the module is not at the root of the .zip
 file.


 TODO::

 - a description of the intended use case
 - address other issues pointed out in:

     http://mail.python.org/pipermail/python-dev/2008-March/077926.html

 Scribbles::

 pull vs. push
 user vs. system
 web vs. filesystem
 auto vs. manual

 manage development sandboxes

 optional interfaces...

     def get_data(pathname) -> string with file data.

     Return the data associated with 'pathname'. Raise IOError if
     the file wasn't found.");

     def is_package,
     "is_package(fullname) -> bool.

     Return True if the module specified by fullname is a package.
     Raise ZipImportError is the module couldn't be found.");

     def get_code,
     "get_code(fullname) -> code object.

     Return the code object for the specified module. Raise ZipImportError
     is the module couldn't be found.");

     def get_source,
     "get_source(fullname) -> source string.

     Return the source code for the specified module. Raise ZipImportError
     is the module couldn't be found, return None if the archive does
     contain the module, but has no source for it.");


 Autoinstall can also be bootstraped with the nascent package loader
 bootstrap module. For example::

     #  or via the bootstrap
     # loader.

     try:
         _version = "0.2"
         import autoinstall
         if autoinstall.__version__ != _version:
             raise ImportError("A different version than expected found.")
     except ImportError, e:
         # http://svn.python.org/projects/sandbox/trunk/bootstrap/bootstrap.py
         import bootstrap
         pypi = "http://pypi.python.org"
         dir = "packages/source/a/autoinstall"
         url = "%s/%s/autoinstall-%s.tar.gz" % (pypi, dir, _version)
         bootstrap.main((url,))
         import autoinstall

 References::

   http://0install.net/
   http://www.python.org/dev/peps/pep-0302/
   http://svn.python.org/projects/sandbox/trunk/import_in_py
   http://0install.net/injector-find.html
   http://roscidus.com/desktop/node/903

 """

 # To allow use of the "with" keyword for Python 2.5 users.
 from __future__ import with_statement

 __version__ = "0.2"
 __docformat__ = "restructuredtext en"

 import os
 import new
 import sys
 import urllib
 import logging
 import tempfile
 import zipimport

 _logger = logging.getLogger(__name__)


 _importer = None

 def _getImporter():
     global _importer
     if _importer is None:
         _importer = Importer()
         sys.meta_path.append(_importer)
     return _importer

 def bind(package_name, url, zip_subpath=None):
     """bind a top level package name to a URL.

     The package name should be a package name and the url should be a
     url to something that can be imported using the zipimporter.

     Optional zip_subpath parameter allows searching for modules
     below the root level of the zip file.
     """
     _getImporter().bind(package_name, url, zip_subpath)


 class Cache(object):

     def __init__(self, directory=None):
         if directory is None:
             # Default to putting the cache directory in the same directory
             # as this file.
             containing_directory = os.path.dirname(__file__)
             directory = os.path.join(containing_directory, "autoinstall.cache.d");

         self.directory = directory
         try:
             if not os.path.exists(self.directory):
                 self._create_cache_directory()
         except Exception, err:
             _logger.exception(err)
             self.cache_directry = tempfile.mkdtemp()
         _logger.info("Using cache directory '%s'." % self.directory)

     def _create_cache_directory(self):
         _logger.debug("Creating cache directory '%s'." % self.directory)
         os.mkdir(self.directory)
         readme_path = os.path.join(self.directory, "README")
         with open(readme_path, "w") as f:
             f.write("This directory was auto-generated by '%s'.\n"
                     "It is safe to delete.\n" % __file__)

     def get(self, url):
         _logger.info("Getting '%s' from cache." % url)
         filename = url.rsplit("/")[-1]

         # so that source url is significant in determining cache hits
         d = os.path.join(self.directory, "%s" % hash(url))
         if not os.path.exists(d):
             os.mkdir(d)

         filename = os.path.join(d, filename)

         if os.path.exists(filename):
             _logger.debug("... already cached in file '%s'." % filename)
         else:
             _logger.debug("... not in cache. Caching in '%s'." % filename)
             stream = file(filename, "wb")
             self.download(url, stream)
             stream.close()
         return filename

     def download(self, url, stream):
         _logger.info("Downloading: %s" % url)
         try:
             netstream = urllib.urlopen(url)
             code = 200
             if hasattr(netstream, "getcode"):
                 code = netstream.getcode()
             if not 200 <= code < 300:
                 raise ValueError("HTTP Error code %s" % code)
         except Exception, err:
             _logger.exception(err)

         BUFSIZE = 2**13  # 8KB
         size = 0
         while True:
             data = netstream.read(BUFSIZE)
             if not data:
                 break
             stream.write(data)
             size += len(data)
         netstream.close()
         _logger.info("Downloaded %d bytes." % size)


 class Importer(object):

     def __init__(self):
         self.packages = {}
         self.__cache = None

     def __get_store(self):
         return self.__store
     store = property(__get_store)

     def _get_cache(self):
         if self.__cache is None:
             self.__cache = Cache()
         return self.__cache
     def _set_cache(self, cache):
         self.__cache = cache
     cache = property(_get_cache, _set_cache)

     def find_module(self, fullname, path=None):
         """-> self or None.

         Search for a module specified by 'fullname'. 'fullname' must be
         the fully qualified (dotted) module name. It returns the
         zipimporter instance itself if the module was found, or None if
         it wasn't. The optional 'path' argument is ignored -- it's
         there for compatibility with the importer protocol.");
         """
         _logger.debug("find_module(%s, path=%s)" % (fullname, path))

         if fullname in self.packages:
             (url, zip_subpath) = self.packages[fullname]
             filename = self.cache.get(url)
             zip_path = "%s/%s" % (filename, zip_subpath) if zip_subpath else filename
             _logger.debug("fullname: %s url: %s path: %s zip_path: %s" % (fullname, url, path, zip_path))
             try:
                 loader = zipimport.zipimporter(zip_path)
                 _logger.debug("returning: %s" % loader)
             except Exception, e:
                 _logger.exception(e)
                 return None
             return loader
         return None

     def bind(self, package_name, url, zip_subpath):
         _logger.info("binding: %s -> %s subpath: %s" % (package_name, url, zip_subpath))
         self.packages[package_name] = (url, zip_subpath)


 if __name__=="__main__":
     import logging
     #logging.basicConfig()
     logger = logging.getLogger()

     console = logging.StreamHandler()
     console.setLevel(logging.DEBUG)
     # set a format which is simpler for console use
     formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
     # tell the handler to use this format
     console.setFormatter(formatter)
     # add the handler to the root logger
     logger.addHandler(console)
     logger.setLevel(logging.INFO)

     bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg")

     import pymarc

     print pymarc.__version__, pymarc.__file__

     assert pymarc.__version__=="2.1"

     d = _getImporter().cache.directory
     assert d in pymarc.__file__, "'%s' not found in pymarc.__file__ (%s)" % (d, pymarc.__file__)

     # Can now also bind to non top level packages. The packages
     # leading up to the package being bound will need to be defined
     # however.
     #
     # bind("rdf.plugins.stores.memory",
     #      "http://pypi.python.org/packages/2.5/r/rdf.plugins.stores.memeory/rdf.plugins.stores.memory-0.9a-py2.5.egg")
     #
     # from rdf.plugins.stores.memory import Memory
	# Copyright (c) 2009, Daniel Krech All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions are
	# met:
	#
	# * Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	#
	# * Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution.
	#
	# * Neither the name of the Daniel Krech nor the names of its
	# contributors may be used to endorse or promote products derived from
	# this software without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	"""\
	package loader for auto installing Python packages.

	A package loader in the spirit of Zero Install that can be used to
	inject dependencies into the import process.


	To install::

	easy_install -U autoinstall

	or

	download, unpack, python setup.py install

	or

	try the bootstrap loader. See below.


	To use::

	# You can bind any package name to a URL pointing to something
	# that can be imported using the zipimporter.

	autoinstall.bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg")

	import pymarc

	print pymarc.__version__, pymarc.__file__


	Changelog::

	- added support for non top level packages.
	- cache files now use filename part from URL.
	- applied patch from Eric Seidel <eseidel@google.com> to add support
	for loading modules where the module is not at the root of the .zip
	file.


	TODO::

	- a description of the intended use case
	- address other issues pointed out in:

	http://mail.python.org/pipermail/python-dev/2008-March/077926.html

	Scribbles::

	pull vs. push
	user vs. system
	web vs. filesystem
	auto vs. manual

	manage development sandboxes

	optional interfaces...

	def get_data(pathname) -> string with file data.

	Return the data associated with 'pathname'. Raise IOError if
	the file wasn't found.");

	def is_package,
	"is_package(fullname) -> bool.

	Return True if the module specified by fullname is a package.
	Raise ZipImportError is the module couldn't be found.");

	def get_code,
	"get_code(fullname) -> code object.

	Return the code object for the specified module. Raise ZipImportError
	is the module couldn't be found.");

	def get_source,
	"get_source(fullname) -> source string.

	Return the source code for the specified module. Raise ZipImportError
	is the module couldn't be found, return None if the archive does
	contain the module, but has no source for it.");


	Autoinstall can also be bootstraped with the nascent package loader
	bootstrap module. For example::

	# or via the bootstrap
	# loader.

	try:
	_version = "0.2"
	import autoinstall
	if autoinstall.__version__ != _version:
	raise ImportError("A different version than expected found.")
	except ImportError, e:
	# http://svn.python.org/projects/sandbox/trunk/bootstrap/bootstrap.py
	import bootstrap
	pypi = "http://pypi.python.org"
	dir = "packages/source/a/autoinstall"
	url = "%s/%s/autoinstall-%s.tar.gz" % (pypi, dir, _version)
	bootstrap.main((url,))
	import autoinstall

	References::

	http://0install.net/
	http://www.python.org/dev/peps/pep-0302/
	http://svn.python.org/projects/sandbox/trunk/import_in_py
	http://0install.net/injector-find.html
	http://roscidus.com/desktop/node/903

	"""

	# To allow use of the "with" keyword for Python 2.5 users.
	from __future__ import with_statement

	__version__ = "0.2"
	__docformat__ = "restructuredtext en"

	import os
	import new
	import sys
	import urllib
	import logging
	import tempfile
	import zipimport

	_logger = logging.getLogger(__name__)


	_importer = None

	def _getImporter():
	global _importer
	if _importer is None:
	_importer = Importer()
	sys.meta_path.append(_importer)
	return _importer

	def bind(package_name, url, zip_subpath=None):
	"""bind a top level package name to a URL.

	The package name should be a package name and the url should be a
	url to something that can be imported using the zipimporter.

	Optional zip_subpath parameter allows searching for modules
	below the root level of the zip file.
	"""
	_getImporter().bind(package_name, url, zip_subpath)


	class Cache(object):

	def __init__(self, directory=None):
	if directory is None:
	# Default to putting the cache directory in the same directory
	# as this file.
	containing_directory = os.path.dirname(__file__)
	directory = os.path.join(containing_directory, "autoinstall.cache.d");

	self.directory = directory
	try:
	if not os.path.exists(self.directory):
	self._create_cache_directory()
	except Exception, err:
	_logger.exception(err)
	self.cache_directry = tempfile.mkdtemp()
	_logger.info("Using cache directory '%s'." % self.directory)

	def _create_cache_directory(self):
	_logger.debug("Creating cache directory '%s'." % self.directory)
	os.mkdir(self.directory)
	readme_path = os.path.join(self.directory, "README")
	with open(readme_path, "w") as f:
	f.write("This directory was auto-generated by '%s'.\n"
	"It is safe to delete.\n" % __file__)

	def get(self, url):
	_logger.info("Getting '%s' from cache." % url)
	filename = url.rsplit("/")[-1]

	# so that source url is significant in determining cache hits
	d = os.path.join(self.directory, "%s" % hash(url))
	if not os.path.exists(d):
	os.mkdir(d)

	filename = os.path.join(d, filename)

	if os.path.exists(filename):
	_logger.debug("... already cached in file '%s'." % filename)
	else:
	_logger.debug("... not in cache. Caching in '%s'." % filename)
	stream = file(filename, "wb")
	self.download(url, stream)
	stream.close()
	return filename

	def download(self, url, stream):
	_logger.info("Downloading: %s" % url)
	try:
	netstream = urllib.urlopen(url)
	code = 200
	if hasattr(netstream, "getcode"):
	code = netstream.getcode()
	if not 200 <= code < 300:
	raise ValueError("HTTP Error code %s" % code)
	except Exception, err:
	_logger.exception(err)

	BUFSIZE = 2**13 # 8KB
	size = 0
	while True:
	data = netstream.read(BUFSIZE)
	if not data:
	break
	stream.write(data)
	size += len(data)
	netstream.close()
	_logger.info("Downloaded %d bytes." % size)


	class Importer(object):

	def __init__(self):
	self.packages = {}
	self.__cache = None

	def __get_store(self):
	return self.__store
	store = property(__get_store)

	def _get_cache(self):
	if self.__cache is None:
	self.__cache = Cache()
	return self.__cache
	def _set_cache(self, cache):
	self.__cache = cache
	cache = property(_get_cache, _set_cache)

	def find_module(self, fullname, path=None):
	"""-> self or None.

	Search for a module specified by 'fullname'. 'fullname' must be
	the fully qualified (dotted) module name. It returns the
	zipimporter instance itself if the module was found, or None if
	it wasn't. The optional 'path' argument is ignored -- it's
	there for compatibility with the importer protocol.");
	"""
	_logger.debug("find_module(%s, path=%s)" % (fullname, path))

	if fullname in self.packages:
	(url, zip_subpath) = self.packages[fullname]
	filename = self.cache.get(url)
	zip_path = "%s/%s" % (filename, zip_subpath) if zip_subpath else filename
	_logger.debug("fullname: %s url: %s path: %s zip_path: %s" % (fullname, url, path, zip_path))
	try:
	loader = zipimport.zipimporter(zip_path)
	_logger.debug("returning: %s" % loader)
	except Exception, e:
	_logger.exception(e)
	return None
	return loader
	return None

	def bind(self, package_name, url, zip_subpath):
	_logger.info("binding: %s -> %s subpath: %s" % (package_name, url, zip_subpath))
	self.packages[package_name] = (url, zip_subpath)


	if __name__=="__main__":
	import logging
	#logging.basicConfig()
	logger = logging.getLogger()

	console = logging.StreamHandler()
	console.setLevel(logging.DEBUG)
	# set a format which is simpler for console use
	formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
	# tell the handler to use this format
	console.setFormatter(formatter)
	# add the handler to the root logger
	logger.addHandler(console)
	logger.setLevel(logging.INFO)

	bind("pymarc", "http://pypi.python.org/packages/2.5/p/pymarc/pymarc-2.1-py2.5.egg")

	import pymarc

	print pymarc.__version__, pymarc.__file__

	assert pymarc.__version__=="2.1"

	d = _getImporter().cache.directory
	assert d in pymarc.__file__, "'%s' not found in pymarc.__file__ (%s)" % (d, pymarc.__file__)

	# Can now also bind to non top level packages. The packages
	# leading up to the package being bound will need to be defined
	# however.
	#
	# bind("rdf.plugins.stores.memory",
	# "http://pypi.python.org/packages/2.5/r/rdf.plugins.stores.memeory/rdf.plugins.stores.memory-0.9a-py2.5.egg")
	#
	# from rdf.plugins.stores.memory import Memory