Merge "libselinux: Fix location of product_seapp_contexts"
diff --git a/libselinux/src/Makefile b/libselinux/src/Makefile
index 977b5c8..8891086 100644
--- a/libselinux/src/Makefile
+++ b/libselinux/src/Makefile
@@ -64,7 +64,8 @@
 EXTRA_CFLAGS = -fipa-pure-const -Wlogical-op -Wpacked-bitfield-compat -Wsync-nand \
 	-Wcoverage-mismatch -Wcpp -Wformat-contains-nul -Wnormalized=nfc -Wsuggest-attribute=const \
 	-Wsuggest-attribute=noreturn -Wsuggest-attribute=pure -Wtrampolines -Wjump-misses-init \
-	-Wno-suggest-attribute=pure -Wno-suggest-attribute=const -Wp,-D_FORTIFY_SOURCE
+	-Wno-suggest-attribute=pure -Wno-suggest-attribute=const -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 \
+	-Wstrict-overflow=5
 else
 EXTRA_CFLAGS = -Wunused-command-line-argument
 endif
diff --git a/libselinux/utils/Makefile b/libselinux/utils/Makefile
index d06ffd6..3615063 100644
--- a/libselinux/utils/Makefile
+++ b/libselinux/utils/Makefile
@@ -30,10 +30,10 @@
           -Wformat-extra-args -Wformat-zero-length -Wformat=2 -Wmultichar \
           -Woverflow -Wpointer-to-int-cast -Wpragmas \
           -Wno-missing-field-initializers -Wno-sign-compare \
-          -Wno-format-nonliteral -Wframe-larger-than=$(MAX_STACK_SIZE) -Wp,-D_FORTIFY_SOURCE \
+          -Wno-format-nonliteral -Wframe-larger-than=$(MAX_STACK_SIZE) -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 \
           -fstack-protector-all --param=ssp-buffer-size=4 -fexceptions \
           -fasynchronous-unwind-tables -fdiagnostics-show-option -funit-at-a-time \
-          -Werror -Wno-aggregate-return -Wno-redundant-decls \
+          -Werror -Wno-aggregate-return -Wno-redundant-decls -Wstrict-overflow=5 \
           $(EXTRA_CFLAGS)
 
 LD_SONAME_FLAGS=-soname,$(LIBSO),-z,defs,-z,relro
diff --git a/libsemanage/src/genhomedircon.c b/libsemanage/src/genhomedircon.c
index 3e61b51..591941f 100644
--- a/libsemanage/src/genhomedircon.c
+++ b/libsemanage/src/genhomedircon.c
@@ -361,7 +361,11 @@
 
 	errno = 0;
 	setpwent();
-	while ((pwbuf = getpwent()) != NULL) {
+	while (1) {
+		errno = 0;
+		pwbuf = getpwent();
+		if (pwbuf == NULL)
+			break;
 		if (pwbuf->pw_uid < minuid || pwbuf->pw_uid > maxuid)
 			continue;
 		if (!semanage_list_find(shells, pwbuf->pw_shell))
@@ -403,7 +407,6 @@
 		}
 		free(path);
 		path = NULL;
-		errno = 0;
 	}
 
 	if (errno) {
@@ -1101,7 +1104,11 @@
 	}
 
 	setpwent();
-	while ((pw = getpwent()) != NULL) {
+	while (1) {
+		errno = 0;
+		pw = getpwent();
+		if (pw == NULL)
+			break;
 		// skip users who also have this group as their
 		// primary group
 		if (lfind(pw->pw_name, group->gr_mem, &nmembers,
diff --git a/libsemanage/utils/semanage_migrate_store b/libsemanage/utils/semanage_migrate_store
index b789d04..018b1a3 100755
--- a/libsemanage/utils/semanage_migrate_store
+++ b/libsemanage/utils/semanage_migrate_store
@@ -8,282 +8,289 @@
 import sys
 from optparse import OptionParser
 
-import ctypes
-
-sepol = ctypes.cdll.LoadLibrary('libsepol.so.1')
 
 try:
-	import selinux
-	import semanage
+    import selinux
+    import semanage
 except ImportError:
-	print("You must install libselinux-python and libsemanage-python before running this tool", file=sys.stderr)
-	exit(1)
+    print("You must install libselinux-python and libsemanage-python before running this tool", file=sys.stderr)
+    exit(1)
 
 
 def copy_file(src, dst):
-	if DEBUG:
-		print("copying %s to %s" % (src, dst))
-	try:
-		shutil.copy(src, dst)
-	except OSError as the_err:
-		(err, strerr) = the_err.args
-		print("Could not copy %s to %s, %s" %(src, dst, strerr), file=sys.stderr)
-		exit(1)
+    if DEBUG:
+        print("copying %s to %s" % (src, dst))
+    try:
+        shutil.copy(src, dst)
+    except OSError as the_err:
+        (err, strerr) = the_err.args
+        print("Could not copy %s to %s, %s" % (src, dst, strerr), file=sys.stderr)
+        exit(1)
 
 
 def create_dir(dst, mode):
-	if DEBUG: print("Making directory %s" % dst)
-	try:
-		os.makedirs(dst, mode)
-	except OSError as the_err:
-		(err, stderr) = the_err.args
-		if err == errno.EEXIST:
-			pass
-		else:
-			print("Error creating %s" % dst, file=sys.stderr)
-			exit(1)
+    if DEBUG:
+        print("Making directory %s" % dst)
+    try:
+        os.makedirs(dst, mode)
+    except OSError as the_err:
+        (err, stderr) = the_err.args
+        if err == errno.EEXIST:
+            pass
+        else:
+            print("Error creating %s" % dst, file=sys.stderr)
+            exit(1)
 
 
 def create_file(dst):
-	if DEBUG: print("Making file %s" % dst)
-	try:
-		open(dst, 'a').close()
-	except OSError as the_err:
-		(err, stderr) = the_err.args
-		print("Error creating %s" % dst, file=sys.stderr)
-		exit(1)
+    if DEBUG:
+        print("Making file %s" % dst)
+    try:
+        open(dst, 'a').close()
+    except OSError as the_err:
+        (err, stderr) = the_err.args
+        print("Error creating %s" % dst, file=sys.stderr)
+        exit(1)
 
 
 def copy_module(store, name, base):
-	if DEBUG: print("Install module %s" % name)
-	(file, ext) = os.path.splitext(name)
-	if ext != ".pp":
-		# Stray non-pp file in modules directory, skip
-		print("warning: %s has invalid extension, skipping" % name, file=sys.stderr)
-		return
-	try:
-		if base:
-			root = oldstore_path(store)
-		else:
-			root = oldmodules_path(store)
+    if DEBUG:
+        print("Install module %s" % name)
+    (file, ext) = os.path.splitext(name)
+    if ext != ".pp":
+        # Stray non-pp file in modules directory, skip
+        print("warning: %s has invalid extension, skipping" % name, file=sys.stderr)
+        return
+    try:
+        if base:
+            root = oldstore_path(store)
+        else:
+            root = oldmodules_path(store)
 
-		bottomdir = bottomdir_path(store)
+        bottomdir = bottomdir_path(store)
 
-		os.mkdir("%s/%s" % (bottomdir, file))
+        os.mkdir("%s/%s" % (bottomdir, file))
 
-		copy_file(os.path.join(root, name), "%s/%s/hll" % (bottomdir, file))
+        copy_file(os.path.join(root, name), "%s/%s/hll" % (bottomdir, file))
 
-		# This is the ext file that will eventually be used to choose a compiler
-		efile = open("%s/%s/lang_ext" % (bottomdir, file), "w+", 0o600)
-		efile.write("pp")
-		efile.close()
+        # This is the ext file that will eventually be used to choose a compiler
+        efile = open("%s/%s/lang_ext" % (bottomdir, file), "w+", 0o600)
+        efile.write("pp")
+        efile.close()
 
-	except:
-		print("Error installing module %s" % name, file=sys.stderr)
-		exit(1)
+    except (IOError, OSError):
+        print("Error installing module %s" % name, file=sys.stderr)
+        exit(1)
 
 
 def disable_module(file, name, disabledmodules):
-	if DEBUG: print("Disabling %s" % name)
-	(disabledname, disabledext) = os.path.splitext(file)
-	create_file("%s/%s" % (disabledmodules, disabledname))
+    if DEBUG:
+        print("Disabling %s" % name)
+    (disabledname, disabledext) = os.path.splitext(file)
+    create_file("%s/%s" % (disabledmodules, disabledname))
+
 
 def migrate_store(store):
+    oldstore = oldstore_path(store)
+    oldmodules = oldmodules_path(store)
+    disabledmodules = disabledmodules_path(store)
+    newstore = newstore_path(store)
+    newmodules = newmodules_path(store)
+    bottomdir = bottomdir_path(store)
 
-	oldstore = oldstore_path(store);
-	oldmodules = oldmodules_path(store);
-	disabledmodules = disabledmodules_path(store);
-	newstore = newstore_path(store);
-	newmodules = newmodules_path(store);
-	bottomdir = bottomdir_path(store);
+    print("Migrating from %s to %s" % (oldstore, newstore))
 
-	print("Migrating from %s to %s" % (oldstore, newstore))
+    # Build up new directory structure
+    create_dir("%s/%s" % (newroot_path(), store), 0o755)
+    create_dir(newstore, 0o700)
+    create_dir(newmodules, 0o700)
+    create_dir(bottomdir, 0o700)
+    create_dir(disabledmodules, 0o700)
 
-	# Build up new directory structure
-	create_dir("%s/%s" % (newroot_path(), store), 0o755)
-	create_dir(newstore, 0o700)
-	create_dir(newmodules, 0o700)
-	create_dir(bottomdir, 0o700)
-	create_dir(disabledmodules, 0o700)
+    # Special case for base since it was in a different location
+    copy_module(store, "base.pp", 1)
 
-	# Special case for base since it was in a different location
-	copy_module(store, "base.pp", 1)
+    # Dir structure built, start copying files
+    for root, dirs, files in os.walk(oldstore):
+        if root == oldstore:
+            # This is the top level directory, need to move
+            for name in files:
+                # Check to see if it is in TOPPATHS and copy if so
+                if name in TOPPATHS:
+                    if name == "seusers":
+                        newname = "seusers.local"
+                    else:
+                        newname = name
+                    copy_file(os.path.join(root, name), os.path.join(newstore, newname))
 
-	# Dir structure built, start copying files
-	for root, dirs, files in os.walk(oldstore):
-		if root == oldstore:
-			# This is the top level directory, need to move
-			for name in files:
-				# Check to see if it is in TOPPATHS and copy if so
-				if name in TOPPATHS:
-					if name == "seusers":
-						newname = "seusers.local"
-					else:
-						newname = name
-					copy_file(os.path.join(root, name), os.path.join(newstore, newname))
+        elif root == oldmodules:
+            # This should be the modules directory
+            for name in files:
+                (file, ext) = os.path.splitext(name)
+                if name == "base.pp":
+                    print("Error installing module %s, name conflicts with base" % name, file=sys.stderr)
+                    exit(1)
+                elif ext == ".disabled":
+                    disable_module(file, name, disabledmodules)
+                else:
+                    copy_module(store, name, 0)
 
-		elif root == oldmodules:
-			# This should be the modules directory
-			for name in files:
-				(file, ext) = os.path.splitext(name)
-				if name == "base.pp":
-					print("Error installing module %s, name conflicts with base" % name, file=sys.stderr)
-					exit(1)
-				elif ext == ".disabled":
-					disable_module(file, name, disabledmodules)
-				else:
-					copy_module(store, name, 0)
 
 def rebuild_policy():
-	# Ok, the modules are loaded, lets try to rebuild the policy
-	print("Attempting to rebuild policy from %s" % newroot_path())
+    # Ok, the modules are loaded, lets try to rebuild the policy
+    print("Attempting to rebuild policy from %s" % newroot_path())
 
-	curstore = selinux.selinux_getpolicytype()[1]
+    curstore = selinux.selinux_getpolicytype()[1]
 
-	handle = semanage.semanage_handle_create()
-	if not handle:
-		print("Could not create semanage handle", file=sys.stderr)
-		exit(1)
+    handle = semanage.semanage_handle_create()
+    if not handle:
+        print("Could not create semanage handle", file=sys.stderr)
+        exit(1)
 
-	semanage.semanage_select_store(handle, curstore, semanage.SEMANAGE_CON_DIRECT)
+    semanage.semanage_select_store(handle, curstore, semanage.SEMANAGE_CON_DIRECT)
 
-	if not semanage.semanage_is_managed(handle):
-		semanage.semanage_handle_destroy(handle)
-		print("SELinux policy is not managed or store cannot be accessed.", file=sys.stderr)
-		exit(1)
+    if not semanage.semanage_is_managed(handle):
+        semanage.semanage_handle_destroy(handle)
+        print("SELinux policy is not managed or store cannot be accessed.", file=sys.stderr)
+        exit(1)
 
-	rc = semanage.semanage_access_check(handle)
-	if rc < semanage.SEMANAGE_CAN_WRITE:
-		semanage.semanage_handle_destroy(handle)
-		print("Cannot write to policy store.", file=sys.stderr)
-		exit(1)
+    rc = semanage.semanage_access_check(handle)
+    if rc < semanage.SEMANAGE_CAN_WRITE:
+        semanage.semanage_handle_destroy(handle)
+        print("Cannot write to policy store.", file=sys.stderr)
+        exit(1)
 
-	rc = semanage.semanage_connect(handle)
-	if rc < 0:
-		semanage.semanage_handle_destroy(handle)
-		print("Could not establish semanage connection", file=sys.stderr)
-		exit(1)
+    rc = semanage.semanage_connect(handle)
+    if rc < 0:
+        semanage.semanage_handle_destroy(handle)
+        print("Could not establish semanage connection", file=sys.stderr)
+        exit(1)
 
-	semanage.semanage_set_rebuild(handle, 1)
+    semanage.semanage_set_rebuild(handle, 1)
 
-	rc = semanage.semanage_begin_transaction(handle)
-	if rc < 0:
-		semanage.semanage_handle_destroy(handle)
-		print("Could not begin transaction", file=sys.stderr)
-		exit(1)
+    rc = semanage.semanage_begin_transaction(handle)
+    if rc < 0:
+        semanage.semanage_handle_destroy(handle)
+        print("Could not begin transaction", file=sys.stderr)
+        exit(1)
 
-	rc = semanage.semanage_commit(handle)
-	if rc < 0:
-		print("Could not commit transaction", file=sys.stderr)
+    rc = semanage.semanage_commit(handle)
+    if rc < 0:
+        print("Could not commit transaction", file=sys.stderr)
 
-	semanage.semanage_handle_destroy(handle)
+    semanage.semanage_handle_destroy(handle)
 
 
 def oldroot_path():
-	return "%s/etc/selinux" % ROOT
+    return "%s/etc/selinux" % ROOT
+
 
 def oldstore_path(store):
-	return "%s/%s/modules/active" % (oldroot_path(), store)
+    return "%s/%s/modules/active" % (oldroot_path(), store)
+
 
 def oldmodules_path(store):
-	return "%s/modules" % oldstore_path(store)
+    return "%s/modules" % oldstore_path(store)
+
 
 def disabledmodules_path(store):
-	return "%s/disabled" % newmodules_path(store)
+    return "%s/disabled" % newmodules_path(store)
+
 
 def newroot_path():
-	return "%s%s" % (ROOT, PATH)
+    return "%s%s" % (ROOT, PATH)
+
 
 def newstore_path(store):
-	return "%s/%s/active" % (newroot_path(), store)
+    return "%s/%s/active" % (newroot_path(), store)
+
 
 def newmodules_path(store):
-	return "%s/modules" % newstore_path(store)
+    return "%s/modules" % newstore_path(store)
+
 
 def bottomdir_path(store):
-	return "%s/%s" % (newmodules_path(store), PRIORITY)
+    return "%s/%s" % (newmodules_path(store), PRIORITY)
 
 
 if __name__ == "__main__":
 
-	parser = OptionParser()
-	parser.add_option("-p", "--priority", dest="priority", default="100",
-			  help="Set priority of modules in new store (default: 100)")
-	parser.add_option("-s", "--store", dest="store", default=None,
-			  help="Store to read from and write to")
-	parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False,
-			  help="Output debug information")
-	parser.add_option("-c", "--clean", dest="clean", action="store_true", default=False,
-			  help="Clean old modules directory after migrate (default: no)")
-	parser.add_option("-n", "--norebuild", dest="norebuild", action="store_true", default=False,
-			  help="Disable rebuilding policy after migration (default: no)")
-	parser.add_option("-P", "--path", dest="path",
-			  help="Set path for the policy store (default: /var/lib/selinux)")
-	parser.add_option("-r", "--root", dest="root",
-			  help="Set an alternative root for the migration (default: /)")
+    parser = OptionParser()
+    parser.add_option("-p", "--priority", dest="priority", default="100",
+                      help="Set priority of modules in new store (default: 100)")
+    parser.add_option("-s", "--store", dest="store", default=None,
+                      help="Store to read from and write to")
+    parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False,
+                      help="Output debug information")
+    parser.add_option("-c", "--clean", dest="clean", action="store_true", default=False,
+                      help="Clean old modules directory after migrate (default: no)")
+    parser.add_option("-n", "--norebuild", dest="norebuild", action="store_true", default=False,
+                      help="Disable rebuilding policy after migration (default: no)")
+    parser.add_option("-P", "--path", dest="path",
+                      help="Set path for the policy store (default: /var/lib/selinux)")
+    parser.add_option("-r", "--root", dest="root",
+                      help="Set an alternative root for the migration (default: /)")
 
-	(options, args) = parser.parse_args()
+    (options, args) = parser.parse_args()
 
-	DEBUG = options.debug
-	PRIORITY = options.priority
-	TYPE = options.store
-	CLEAN = options.clean
-	NOREBUILD = options.norebuild
-	PATH = options.path
-	if PATH is None:
-		PATH = "/var/lib/selinux"
+    DEBUG = options.debug
+    PRIORITY = options.priority
+    TYPE = options.store
+    CLEAN = options.clean
+    NOREBUILD = options.norebuild
+    PATH = options.path
+    if PATH is None:
+        PATH = "/var/lib/selinux"
 
-	ROOT = options.root
-	if ROOT is None:
-		ROOT = ""
+    ROOT = options.root
+    if ROOT is None:
+        ROOT = ""
 
-	# List of paths that go in the active 'root'
-	TOPPATHS = [
-		"commit_num",
-		"ports.local",
-		"interfaces.local",
-		"nodes.local",
-		"booleans.local",
-		"file_contexts.local",
-		"seusers",
-		"users.local",
-		"users_extra",
-		"users_extra.local",
-		"disable_dontaudit",
-		"preserve_tunables",
-		"policy.kern",
-		"file_contexts",
-		"homedir_template",
-		"pkeys.local",
-		"ibendports.local"]
+    # List of paths that go in the active 'root'
+    TOPPATHS = [
+        "commit_num",
+        "ports.local",
+        "interfaces.local",
+        "nodes.local",
+        "booleans.local",
+        "file_contexts.local",
+        "seusers",
+        "users.local",
+        "users_extra",
+        "users_extra.local",
+        "disable_dontaudit",
+        "preserve_tunables",
+        "policy.kern",
+        "file_contexts",
+        "homedir_template",
+        "pkeys.local",
+        "ibendports.local"]
 
+    create_dir(newroot_path(), 0o755)
 
-	create_dir(newroot_path(), 0o755)
+    stores = None
+    if TYPE is not None:
+        stores = [TYPE]
+    else:
+        stores = os.listdir(oldroot_path())
 
-	stores = None
-	if TYPE is not None:
-		stores = [TYPE]
-	else:
-		stores = os.listdir(oldroot_path())
+    # find stores in oldroot and migrate them to newroot if necessary
+    for store in stores:
+        if not os.path.isdir(oldmodules_path(store)):
+            # already migrated or not an selinux store
+            continue
 
-	# find stores in oldroot and migrate them to newroot if necessary
-	for store in stores:
-		if not os.path.isdir(oldmodules_path(store)):
-			# already migrated or not an selinux store
-			continue
+        if os.path.isdir(newstore_path(store)):
+            # store has already been migrated, but old modules dir still exits
+            print("warning: Policy type %s has already been migrated, but modules still exist in the old store. Skipping store." % store, file=sys.stderr)
+            continue
 
-		if os.path.isdir(newstore_path(store)):
-			# store has already been migrated, but old modules dir still exits
-			print("warning: Policy type %s has already been migrated, but modules still exist in the old store. Skipping store." % store, file=sys.stderr)
-			continue
+        migrate_store(store)
 
-		migrate_store(store)
+        if CLEAN is True:
+            def remove_error(function, path, execinfo):
+                print("warning: Unable to remove old store modules directory %s. Cleaning failed." % oldmodules_path(store), file=sys.stderr)
+            shutil.rmtree(oldmodules_path(store), onerror=remove_error)
 
-		if CLEAN is True:
-			def remove_error(function, path, execinfo):
-				print("warning: Unable to remove old store modules directory %s. Cleaning failed." % oldmodules_path(store), file=sys.stderr)
-			shutil.rmtree(oldmodules_path(store), onerror=remove_error)
-
-	if NOREBUILD is False:
-		rebuild_policy()
-
+    if NOREBUILD is False:
+        rebuild_policy()
diff --git a/mcstrans/src/mcscolor.c b/mcstrans/src/mcscolor.c
index cc6174b..6ea1aa9 100644
--- a/mcstrans/src/mcscolor.c
+++ b/mcstrans/src/mcscolor.c
@@ -292,7 +292,7 @@
 	size_t result_size = (N_COLOR * CHARS_PER_COLOR) + 1;
 	int rc = -1;
 
-	if (!color_str || !*color_str) {
+	if (!color_str || *color_str) {
 		return -1;
 	}
 
diff --git a/python/audit2allow/audit2allow b/python/audit2allow/audit2allow
index 195f151..18fe0a5 100755
--- a/python/audit2allow/audit2allow
+++ b/python/audit2allow/audit2allow
@@ -242,7 +242,10 @@
 
     def __output_audit2why(self):
         import selinux
-        import sepolicy
+        try:
+            import sepolicy
+        except (ImportError, ValueError):
+            sepolicy = None
         for i in self.__parser.avc_msgs:
             rc = i.type
             data = i.data
@@ -262,11 +265,13 @@
                 if len(data) > 1:
                     print("\tOne of the following booleans was set incorrectly.")
                     for b in data:
-                        print("\tDescription:\n\t%s\n" % sepolicy.boolean_desc(b[0]))
+                        if sepolicy is not None:
+                            print("\tDescription:\n\t%s\n" % sepolicy.boolean_desc(b[0]))
                         print("\tAllow access by executing:\n\t# setsebool -P %s %d" % (b[0], b[1]))
                 else:
                     print("\tThe boolean %s was set incorrectly. " % (data[0][0]))
-                    print("\tDescription:\n\t%s\n" % sepolicy.boolean_desc(data[0][0]))
+                    if sepolicy is not None:
+                        print("\tDescription:\n\t%s\n" % sepolicy.boolean_desc(data[0][0]))
                     print("\tAllow access by executing:\n\t# setsebool -P %s %d" % (data[0][0], data[0][1]))
                 continue
 
diff --git a/python/audit2allow/sepolgen-ifgen b/python/audit2allow/sepolgen-ifgen
index acf9638..e3f67d4 100644
--- a/python/audit2allow/sepolgen-ifgen
+++ b/python/audit2allow/sepolgen-ifgen
@@ -96,7 +96,7 @@
     ret = subprocess.Popen([ATTR_HELPER, policy_path, outfile.name], stdout=fd).wait()
     fd.close()
     if ret != 0:
-        sys.stderr.write("could not run attribute helper")
+        sys.stderr.write("could not run attribute helper\n")
         return None
 
     attrs = interfaces.AttributeSet()
@@ -135,8 +135,7 @@
     try:
         headers = refparser.parse_headers(options.headers, output=log, debug=options.debug)
     except ValueError as e:
-        print("error parsing headers")
-        print(str(e))
+        sys.stderr.write("error parsing headers: %s\n" % e)
         return 1
 
     if_set = interfaces.InterfaceSet(output=log)
diff --git a/python/semanage/semanage b/python/semanage/semanage
index a192fac..49add51 100644
--- a/python/semanage/semanage
+++ b/python/semanage/semanage
@@ -73,7 +73,7 @@
 usage_boolean = "semanage boolean [-h] [-n] [-N] [-S STORE] ["
 usage_boolean_dict = {' --modify': ('(', '--on', '|', '--off', ')', 'boolean'), ' --list': ('-C',), '  --extract': ('',), ' --deleteall': ('',)}
 
-import sepolicy
+
 
 
 class CheckRole(argparse.Action):
@@ -82,7 +82,12 @@
         newval = getattr(namespace, self.dest)
         if not newval:
             newval = []
-        roles = sepolicy.get_all_roles()
+        try:
+            # sepolicy tries to load the SELinux policy and raises ValueError if it fails.
+            import sepolicy
+            roles = sepolicy.get_all_roles()
+        except ValueError:
+            roles = []
         for v in value.split():
             if v not in roles:
                 raise ValueError("%s must be an SELinux role:\nValid roles: %s" % (v, ", ".join(roles)))
diff --git a/python/semanage/seobject.py b/python/semanage/seobject.py
index efec0a5..556d3ba 100644
--- a/python/semanage/seobject.py
+++ b/python/semanage/seobject.py
@@ -260,6 +260,8 @@
         if self.store == "" or self.store == localstore:
             self.mylog = logger()
         else:
+            sepolicy.load_store_policy(self.store)
+            selinux.selinux_set_policy_root("%s%s" % (selinux.selinux_path(), self.store))
             self.mylog = nulllogger()
 
     def set_reload(self, load):
@@ -1043,13 +1045,15 @@
 
 
 class portRecords(semanageRecords):
-    try:
-        valid_types = list(list(sepolicy.info(sepolicy.ATTRIBUTE, "port_type"))[0]["types"])
-    except RuntimeError:
-        valid_types = []
+
+    valid_types = []
 
     def __init__(self, args = None):
         semanageRecords.__init__(self, args)
+        try:
+            self.valid_types = list(list(sepolicy.info(sepolicy.ATTRIBUTE, "port_type"))[0]["types"])
+        except RuntimeError:
+            pass
 
     def __genkey(self, port, proto):
         if proto == "tcp":
@@ -1321,14 +1325,16 @@
             print(rec)
 
 class ibpkeyRecords(semanageRecords):
-    try:
-        q = setools.TypeQuery(setools.SELinuxPolicy(sepolicy.get_installed_policy()), attrs=["ibpkey_type"])
-        valid_types = sorted(str(t) for t in q.results())
-    except:
-        valid_types = []
+
+    valid_types = []
 
     def __init__(self, args = None):
         semanageRecords.__init__(self, args)
+        try:
+            q = setools.TypeQuery(setools.SELinuxPolicy(sepolicy.get_store_policy(self.store)), attrs=["ibpkey_type"])
+            self.valid_types = sorted(str(t) for t in q.results())
+        except:
+            pass
 
     def __genkey(self, pkey, subnet_prefix):
         if subnet_prefix == "":
@@ -1579,14 +1585,16 @@
             print(rec)
 
 class ibendportRecords(semanageRecords):
-    try:
-        q = setools.TypeQuery(setools.SELinuxPolicy(sepolicy.get_installed_policy()), attrs=["ibendport_type"])
-        valid_types = set(str(t) for t in q.results())
-    except:
-        valid_types = []
+
+    valid_types = []
 
     def __init__(self, args = None):
         semanageRecords.__init__(self, args)
+        try:
+            q = setools.TypeQuery(setools.SELinuxPolicy(sepolicy.get_store_policy(self.store)), attrs=["ibendport_type"])
+            self.valid_types = set(str(t) for t in q.results())
+        except:
+            pass
 
     def __genkey(self, ibendport, ibdev_name):
         if ibdev_name == "":
@@ -1823,14 +1831,16 @@
             print(rec)
 
 class nodeRecords(semanageRecords):
-    try:
-        valid_types = list(list(sepolicy.info(sepolicy.ATTRIBUTE, "node_type"))[0]["types"])
-    except RuntimeError:
-        valid_types = []
+
+    valid_types = []
 
     def __init__(self, args = None):
         semanageRecords.__init__(self, args)
         self.protocol = ["ipv4", "ipv6"]
+        try:
+            self.valid_types = list(list(sepolicy.info(sepolicy.ATTRIBUTE, "node_type"))[0]["types"])
+        except RuntimeError:
+            pass
 
     def validate(self, addr, mask, protocol):
         newaddr = addr
@@ -2264,14 +2274,17 @@
 
 
 class fcontextRecords(semanageRecords):
-    try:
-        valid_types = list(list(sepolicy.info(sepolicy.ATTRIBUTE, "file_type"))[0]["types"])
-        valid_types += list(list(sepolicy.info(sepolicy.ATTRIBUTE, "device_node"))[0]["types"])
-    except RuntimeError:
-        valid_types = []
+
+    valid_types = []
 
     def __init__(self, args = None):
         semanageRecords.__init__(self, args)
+        try:
+            self.valid_types = list(list(sepolicy.info(sepolicy.ATTRIBUTE, "file_type"))[0]["types"])
+            self.valid_types += list(list(sepolicy.info(sepolicy.ATTRIBUTE, "device_node"))[0]["types"])
+        except RuntimeError:
+            pass
+
         self.equiv = {}
         self.equiv_dist = {}
         self.equal_ind = False
diff --git a/python/sepolgen/src/sepolgen/defaults.py b/python/sepolgen/src/sepolgen/defaults.py
index 199acfa..533a904 100644
--- a/python/sepolgen/src/sepolgen/defaults.py
+++ b/python/sepolgen/src/sepolgen/defaults.py
@@ -32,12 +32,13 @@
         self.config_pathname = pathname
         ignore = re.compile(r"^\s*(?:#.+)?$")
         consider = re.compile(r"^\s*(\w+)\s*=\s*(.+?)\s*$")
-        for lineno, line in enumerate(open(pathname)):
-            if ignore.match(line): continue
-            mo = consider.match(line)
-            if not mo:
-                raise ValueError("%s:%d: line is not in key = value format" % (pathname, lineno+1))
-            self.config[mo.group(1)] = mo.group(2)
+        with open(pathname, "r") as fd:
+            for lineno, line in enumerate(fd):
+                if ignore.match(line): continue
+                mo = consider.match(line)
+                if not mo:
+                    raise ValueError("%s:%d: line is not in key = value format" % (pathname, lineno+1))
+                self.config[mo.group(1)] = mo.group(2)
 
     # We're only exporting one useful function, so why not be a function
     def __call__(self, testfilename, pathset="SELINUX_DEVEL_PATH"):
diff --git a/python/sepolgen/src/sepolgen/lex.py b/python/sepolgen/src/sepolgen/lex.py
index c13acef..f95bcdb 100644
--- a/python/sepolgen/src/sepolgen/lex.py
+++ b/python/sepolgen/src/sepolgen/lex.py
@@ -1,207 +1,258 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # ply: lex.py
 #
-# Author: David M. Beazley (dave@dabeaz.com)
+# Copyright (C) 2001-2018
+# David M. Beazley (Dabeaz LLC)
+# All rights reserved.
 #
-# Copyright (C) 2001-2006, David M. Beazley
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
 #
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-# 
-# See the file COPYING for a complete copy of the LGPL.
-#-----------------------------------------------------------------------------
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the David Beazley or Dabeaz LLC may be used to
+#   endorse or promote products derived from this software without
+#  specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
 
-__version__ = "2.2"
+__version__    = '3.11'
+__tabversion__ = '3.10'
 
-import re, sys, types
+import re
+import sys
+import types
+import copy
+import os
+import inspect
 
-from . import util
-import collections
-
-
-# Regular expression used to match valid token names
-_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$')
-
-# Available instance types.  This is used when parsers are defined by a class.
-# In Python3 the InstanceType and ObjectType are no more, they've passed, ceased
-# to be, they are ex-classes along with old-style classes
-
+# This tuple contains known string types
 try:
-   _INSTANCETYPE = (types.InstanceType, types.ObjectType)
+    # Python 2.6
+    StringTypes = (types.StringType, types.UnicodeType)
 except AttributeError:
-   _INSTANCETYPE = object
+    # Python 3.0
+    StringTypes = (str, bytes)
+
+# This regular expression is used to match valid token names
+_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$')
 
 # Exception thrown when invalid token encountered and no default error
 # handler is defined.
 class LexError(Exception):
-    def __init__(self,message,s):
-         self.args = (message,)
-         self.text = s
+    def __init__(self, message, s):
+        self.args = (message,)
+        self.text = s
 
-# Token class
+
+# Token class.  This class is used to represent the tokens produced.
 class LexToken(object):
     def __str__(self):
-        return "LexToken(%s,%r,%d,%d)" % (self.type,self.value,self.lineno,self.lexpos)
+        return 'LexToken(%s,%r,%d,%d)' % (self.type, self.value, self.lineno, self.lexpos)
+
     def __repr__(self):
         return str(self)
-    def skip(self,n):
-        self.lexer.skip(n)
+
+
+# This object is a stand-in for a logging object created by the
+# logging module.
+
+class PlyLogger(object):
+    def __init__(self, f):
+        self.f = f
+
+    def critical(self, msg, *args, **kwargs):
+        self.f.write((msg % args) + '\n')
+
+    def warning(self, msg, *args, **kwargs):
+        self.f.write('WARNING: ' + (msg % args) + '\n')
+
+    def error(self, msg, *args, **kwargs):
+        self.f.write('ERROR: ' + (msg % args) + '\n')
+
+    info = critical
+    debug = critical
+
+
+# Null logger is used when no output is generated. Does nothing.
+class NullLogger(object):
+    def __getattribute__(self, name):
+        return self
+
+    def __call__(self, *args, **kwargs):
+        return self
+
 
 # -----------------------------------------------------------------------------
-# Lexer class
+#                        === Lexing Engine ===
 #
-# This class encapsulates all of the methods and data associated with a lexer.
+# The following Lexer class implements the lexer runtime.   There are only
+# a few public methods and attributes:
 #
 #    input()          -  Store a new string in the lexer
 #    token()          -  Get the next token
+#    clone()          -  Clone the lexer
+#
+#    lineno           -  Current line number
+#    lexpos           -  Current position in the input string
 # -----------------------------------------------------------------------------
 
 class Lexer:
     def __init__(self):
-        self.lexre = None             # Master regular expression. This is a list of 
-                                      # tuples (re,findex) where re is a compiled
+        self.lexre = None             # Master regular expression. This is a list of
+                                      # tuples (re, findex) where re is a compiled
                                       # regular expression and findex is a list
                                       # mapping regex group numbers to rules
         self.lexretext = None         # Current regular expression strings
         self.lexstatere = {}          # Dictionary mapping lexer states to master regexs
         self.lexstateretext = {}      # Dictionary mapping lexer states to regex strings
-        self.lexstate = "INITIAL"     # Current lexer state
+        self.lexstaterenames = {}     # Dictionary mapping lexer states to symbol names
+        self.lexstate = 'INITIAL'     # Current lexer state
         self.lexstatestack = []       # Stack of lexer states
         self.lexstateinfo = None      # State information
         self.lexstateignore = {}      # Dictionary of ignored characters for each state
         self.lexstateerrorf = {}      # Dictionary of error functions for each state
+        self.lexstateeoff = {}        # Dictionary of eof functions for each state
         self.lexreflags = 0           # Optional re compile flags
         self.lexdata = None           # Actual input data (as a string)
         self.lexpos = 0               # Current position in input text
         self.lexlen = 0               # Length of the input text
         self.lexerrorf = None         # Error rule (if any)
+        self.lexeoff = None           # EOF rule (if any)
         self.lextokens = None         # List of valid tokens
-        self.lexignore = ""           # Ignored characters
-        self.lexliterals = ""         # Literal characters that can be passed through
+        self.lexignore = ''           # Ignored characters
+        self.lexliterals = ''         # Literal characters that can be passed through
         self.lexmodule = None         # Module
         self.lineno = 1               # Current line number
-        self.lexdebug = 0             # Debugging mode
-        self.lexoptimize = 0          # Optimized mode
+        self.lexoptimize = False      # Optimized mode
 
-    def clone(self,object=None):
-        c = Lexer()
-        c.lexstatere = self.lexstatere
-        c.lexstateinfo = self.lexstateinfo
-        c.lexstateretext = self.lexstateretext
-        c.lexstate = self.lexstate
-        c.lexstatestack = self.lexstatestack
-        c.lexstateignore = self.lexstateignore
-        c.lexstateerrorf = self.lexstateerrorf
-        c.lexreflags = self.lexreflags
-        c.lexdata = self.lexdata
-        c.lexpos = self.lexpos
-        c.lexlen = self.lexlen
-        c.lextokens = self.lextokens
-        c.lexdebug = self.lexdebug
-        c.lineno = self.lineno
-        c.lexoptimize = self.lexoptimize
-        c.lexliterals = self.lexliterals
-        c.lexmodule   = self.lexmodule
+    def clone(self, object=None):
+        c = copy.copy(self)
 
         # If the object parameter has been supplied, it means we are attaching the
         # lexer to a new object.  In this case, we have to rebind all methods in
         # the lexstatere and lexstateerrorf tables.
 
         if object:
-            newtab = { }
+            newtab = {}
             for key, ritem in self.lexstatere.items():
                 newre = []
                 for cre, findex in ritem:
-                     newfindex = []
-                     for f in findex:
-                         if not f or not f[0]:
-                             newfindex.append(f)
-                             continue
-                         newfindex.append((getattr(object,f[0].__name__),f[1]))
-                newre.append((cre,newfindex))
+                    newfindex = []
+                    for f in findex:
+                        if not f or not f[0]:
+                            newfindex.append(f)
+                            continue
+                        newfindex.append((getattr(object, f[0].__name__), f[1]))
+                newre.append((cre, newfindex))
                 newtab[key] = newre
             c.lexstatere = newtab
-            c.lexstateerrorf = { }
+            c.lexstateerrorf = {}
             for key, ef in self.lexstateerrorf.items():
-                c.lexstateerrorf[key] = getattr(object,ef.__name__)
+                c.lexstateerrorf[key] = getattr(object, ef.__name__)
             c.lexmodule = object
-
-        # Set up other attributes
-        c.begin(c.lexstate)
         return c
 
     # ------------------------------------------------------------
     # writetab() - Write lexer information to a table file
     # ------------------------------------------------------------
-    def writetab(self,tabfile):
-        tf = open(tabfile+".py","w")
-        tf.write("# %s.py. This file automatically created by PLY (version %s). Don't edit!\n" % (tabfile,__version__))
-        tf.write("_lextokens    = %s\n" % repr(self.lextokens))
-        tf.write("_lexreflags   = %s\n" % repr(self.lexreflags))
-        tf.write("_lexliterals  = %s\n" % repr(self.lexliterals))
-        tf.write("_lexstateinfo = %s\n" % repr(self.lexstateinfo))
-        
-        tabre = { }
-        for key, lre in self.lexstatere.items():
-             titem = []
-             for i in range(len(lre)):
-                  titem.append((self.lexstateretext[key][i],_funcs_to_names(lre[i][1])))
-             tabre[key] = titem
+    def writetab(self, lextab, outputdir=''):
+        if isinstance(lextab, types.ModuleType):
+            raise IOError("Won't overwrite existing lextab module")
+        basetabmodule = lextab.split('.')[-1]
+        filename = os.path.join(outputdir, basetabmodule) + '.py'
+        with open(filename, 'w') as tf:
+            tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__))
+            tf.write('_tabversion   = %s\n' % repr(__tabversion__))
+            tf.write('_lextokens    = set(%s)\n' % repr(tuple(sorted(self.lextokens))))
+            tf.write('_lexreflags   = %s\n' % repr(int(self.lexreflags)))
+            tf.write('_lexliterals  = %s\n' % repr(self.lexliterals))
+            tf.write('_lexstateinfo = %s\n' % repr(self.lexstateinfo))
 
-        tf.write("_lexstatere   = %s\n" % repr(tabre))
-        tf.write("_lexstateignore = %s\n" % repr(self.lexstateignore))
+            # Rewrite the lexstatere table, replacing function objects with function names
+            tabre = {}
+            for statename, lre in self.lexstatere.items():
+                titem = []
+                for (pat, func), retext, renames in zip(lre, self.lexstateretext[statename], self.lexstaterenames[statename]):
+                    titem.append((retext, _funcs_to_names(func, renames)))
+                tabre[statename] = titem
 
-        taberr = { }
-        for key, ef in self.lexstateerrorf.items():
-             if ef:
-                  taberr[key] = ef.__name__
-             else:
-                  taberr[key] = None
-        tf.write("_lexstateerrorf = %s\n" % repr(taberr))
-        tf.close()
+            tf.write('_lexstatere   = %s\n' % repr(tabre))
+            tf.write('_lexstateignore = %s\n' % repr(self.lexstateignore))
+
+            taberr = {}
+            for statename, ef in self.lexstateerrorf.items():
+                taberr[statename] = ef.__name__ if ef else None
+            tf.write('_lexstateerrorf = %s\n' % repr(taberr))
+
+            tabeof = {}
+            for statename, ef in self.lexstateeoff.items():
+                tabeof[statename] = ef.__name__ if ef else None
+            tf.write('_lexstateeoff = %s\n' % repr(tabeof))
 
     # ------------------------------------------------------------
     # readtab() - Read lexer information from a tab file
     # ------------------------------------------------------------
-    def readtab(self,tabfile,fdict):
-        exec("import %s as lextab" % tabfile)
+    def readtab(self, tabfile, fdict):
+        if isinstance(tabfile, types.ModuleType):
+            lextab = tabfile
+        else:
+            exec('import %s' % tabfile)
+            lextab = sys.modules[tabfile]
+
+        if getattr(lextab, '_tabversion', '0.0') != __tabversion__:
+            raise ImportError('Inconsistent PLY version')
+
         self.lextokens      = lextab._lextokens
         self.lexreflags     = lextab._lexreflags
         self.lexliterals    = lextab._lexliterals
+        self.lextokens_all  = self.lextokens | set(self.lexliterals)
         self.lexstateinfo   = lextab._lexstateinfo
         self.lexstateignore = lextab._lexstateignore
-        self.lexstatere     = { }
-        self.lexstateretext = { }
-        for key,lre in lextab._lexstatere.items():
-             titem = []
-             txtitem = []
-             for i in range(len(lre)):
-                  titem.append((re.compile(lre[i][0],lextab._lexreflags),_names_to_funcs(lre[i][1],fdict)))
-                  txtitem.append(lre[i][0])
-             self.lexstatere[key] = titem
-             self.lexstateretext[key] = txtitem
-        self.lexstateerrorf = { }
-        for key,ef in lextab._lexstateerrorf.items():
-             self.lexstateerrorf[key] = fdict[ef]
+        self.lexstatere     = {}
+        self.lexstateretext = {}
+        for statename, lre in lextab._lexstatere.items():
+            titem = []
+            txtitem = []
+            for pat, func_name in lre:
+                titem.append((re.compile(pat, lextab._lexreflags), _names_to_funcs(func_name, fdict)))
+
+            self.lexstatere[statename] = titem
+            self.lexstateretext[statename] = txtitem
+
+        self.lexstateerrorf = {}
+        for statename, ef in lextab._lexstateerrorf.items():
+            self.lexstateerrorf[statename] = fdict[ef]
+
+        self.lexstateeoff = {}
+        for statename, ef in lextab._lexstateeoff.items():
+            self.lexstateeoff[statename] = fdict[ef]
+
         self.begin('INITIAL')
-         
+
     # ------------------------------------------------------------
     # input() - Push a new string into the lexer
     # ------------------------------------------------------------
-    def input(self,s):
-        if not (isinstance(s,util.bytes_type) or isinstance(s, util.string_type)):
-            raise ValueError("Expected a string")
+    def input(self, s):
+        # Pull off the first character to see if s looks like a string
+        c = s[:1]
+        if not isinstance(c, StringTypes):
+            raise ValueError('Expected a string')
         self.lexdata = s
         self.lexpos = 0
         self.lexlen = len(s)
@@ -209,19 +260,20 @@
     # ------------------------------------------------------------
     # begin() - Changes the lexing state
     # ------------------------------------------------------------
-    def begin(self,state):
+    def begin(self, state):
         if state not in self.lexstatere:
-            raise ValueError("Undefined state")
+            raise ValueError('Undefined state')
         self.lexre = self.lexstatere[state]
         self.lexretext = self.lexstateretext[state]
-        self.lexignore = self.lexstateignore.get(state,"")
-        self.lexerrorf = self.lexstateerrorf.get(state,None)
+        self.lexignore = self.lexstateignore.get(state, '')
+        self.lexerrorf = self.lexstateerrorf.get(state, None)
+        self.lexeoff = self.lexstateeoff.get(state, None)
         self.lexstate = state
 
     # ------------------------------------------------------------
     # push_state() - Changes the lexing state and saves old on stack
     # ------------------------------------------------------------
-    def push_state(self,state):
+    def push_state(self, state):
         self.lexstatestack.append(self.lexstate)
         self.begin(state)
 
@@ -240,11 +292,11 @@
     # ------------------------------------------------------------
     # skip() - Skip ahead n characters
     # ------------------------------------------------------------
-    def skip(self,n):
+    def skip(self, n):
         self.lexpos += n
 
     # ------------------------------------------------------------
-    # token() - Return the next token from the Lexer
+    # opttoken() - Return the next token from the Lexer
     #
     # Note: This function has been carefully implemented to be as fast
     # as possible.  Don't make changes unless you really know what
@@ -264,48 +316,51 @@
                 continue
 
             # Look for a regular expression match
-            for lexre,lexindexfunc in self.lexre:
-                m = lexre.match(lexdata,lexpos)
-                if not m: continue
-
-                # Set last match in lexer so that rules can access it if they want
-                self.lexmatch = m
+            for lexre, lexindexfunc in self.lexre:
+                m = lexre.match(lexdata, lexpos)
+                if not m:
+                    continue
 
                 # Create a token for return
                 tok = LexToken()
                 tok.value = m.group()
                 tok.lineno = self.lineno
                 tok.lexpos = lexpos
-                tok.lexer = self
 
-                lexpos = m.end()
                 i = m.lastindex
-                func,tok.type = lexindexfunc[i]
-                self.lexpos = lexpos
+                func, tok.type = lexindexfunc[i]
 
                 if not func:
-                   # If no token type was set, it's an ignored token
-                   if tok.type: return tok      
-                   break
+                    # If no token type was set, it's an ignored token
+                    if tok.type:
+                        self.lexpos = m.end()
+                        return tok
+                    else:
+                        lexpos = m.end()
+                        break
 
-                # if func not callable, it means it's an ignored token                
-                if not isinstance(func, collections.Callable):
-                   break 
+                lexpos = m.end()
 
                 # If token is processed by a function, call it
+
+                tok.lexer = self      # Set additional attributes useful in token rules
+                self.lexmatch = m
+                self.lexpos = lexpos
+
                 newtok = func(tok)
-                
+
                 # Every function must return a token, if nothing, we just move to next token
-                if not newtok: 
-                    lexpos = self.lexpos        # This is here in case user has updated lexpos.
+                if not newtok:
+                    lexpos    = self.lexpos         # This is here in case user has updated lexpos.
+                    lexignore = self.lexignore      # This is here in case there was a state change
                     break
-                
+
                 # Verify type of the token.  If not in the token map, raise an error
                 if not self.lexoptimize:
-                    if newtok.type not in self.lextokens:
+                    if newtok.type not in self.lextokens_all:
                         raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % (
                             func.__code__.co_filename, func.__code__.co_firstlineno,
-                            func.__name__, newtok.type),lexdata[lexpos:])
+                            func.__name__, newtok.type), lexdata[lexpos:])
 
                 return newtok
             else:
@@ -314,18 +369,17 @@
                     tok = LexToken()
                     tok.value = lexdata[lexpos]
                     tok.lineno = self.lineno
-                    tok.lexer = self
                     tok.type = tok.value
                     tok.lexpos = lexpos
                     self.lexpos = lexpos + 1
                     return tok
-        
+
                 # No match. Call t_error() if defined.
                 if self.lexerrorf:
                     tok = LexToken()
                     tok.value = self.lexdata[lexpos:]
                     tok.lineno = self.lineno
-                    tok.type = "error"
+                    tok.type = 'error'
                     tok.lexer = self
                     tok.lexpos = lexpos
                     self.lexpos = lexpos
@@ -334,56 +388,70 @@
                         # Error method didn't change text position at all. This is an error.
                         raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:])
                     lexpos = self.lexpos
-                    if not newtok: continue
+                    if not newtok:
+                        continue
                     return newtok
 
                 self.lexpos = lexpos
-                raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos],lexpos), lexdata[lexpos:])
+                raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:])
+
+        if self.lexeoff:
+            tok = LexToken()
+            tok.type = 'eof'
+            tok.value = ''
+            tok.lineno = self.lineno
+            tok.lexpos = lexpos
+            tok.lexer = self
+            self.lexpos = lexpos
+            newtok = self.lexeoff(tok)
+            return newtok
 
         self.lexpos = lexpos + 1
         if self.lexdata is None:
-             raise RuntimeError("No input string given with input()")
+            raise RuntimeError('No input string given with input()')
         return None
-        
+
+    # Iterator interface
+    def __iter__(self):
+        return self
+
+    def next(self):
+        t = self.token()
+        if t is None:
+            raise StopIteration
+        return t
+
+    __next__ = next
+
 # -----------------------------------------------------------------------------
-# _validate_file()
+#                           ==== Lex Builder ===
 #
-# This checks to see if there are duplicated t_rulename() functions or strings
-# in the parser input file.  This is done using a simple regular expression
-# match on each line in the filename.
+# The functions and classes below are used to collect lexing information
+# and build a Lexer object from it.
 # -----------------------------------------------------------------------------
 
-def _validate_file(filename):
-    import os.path
-    base,ext = os.path.splitext(filename)
-    if ext != '.py': return 1        # No idea what the file is. Return OK
+# -----------------------------------------------------------------------------
+# _get_regex(func)
+#
+# Returns the regular expression assigned to a function either as a doc string
+# or as a .regex attribute attached by the @TOKEN decorator.
+# -----------------------------------------------------------------------------
+def _get_regex(func):
+    return getattr(func, 'regex', func.__doc__)
 
-    try:
-        f = open(filename)
-        lines = f.readlines()
-        f.close()
-    except IOError:
-        return 1                       # Oh well
-
-    fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(')
-    sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=')
-    counthash = { }
-    linen = 1
-    noerror = 1
-    for l in lines:
-        m = fre.match(l)
-        if not m:
-            m = sre.match(l)
-        if m:
-            name = m.group(1)
-            prev = counthash.get(name)
-            if not prev:
-                counthash[name] = linen
-            else:
-                print("%s:%d: Rule %s redefined. Previously defined on line %d" % (filename,linen,name,prev))
-                noerror = 0
-        linen += 1
-    return noerror
+# -----------------------------------------------------------------------------
+# get_caller_module_dict()
+#
+# This function returns a dictionary containing all of the symbols defined within
+# a caller further down the call stack.  This is used to get the environment
+# associated with the yacc() call if none was provided.
+# -----------------------------------------------------------------------------
+def get_caller_module_dict(levels):
+    f = sys._getframe(levels)
+    ldict = f.f_globals.copy()
+    if f.f_globals != f.f_locals:
+        ldict.update(f.f_locals)
+    return ldict
 
 # -----------------------------------------------------------------------------
 # _funcs_to_names()
@@ -391,14 +459,13 @@
 # Given a list of regular expression functions, this converts it to a list
 # suitable for output to a table file
 # -----------------------------------------------------------------------------
-
-def _funcs_to_names(funclist):
+def _funcs_to_names(funclist, namelist):
     result = []
-    for f in funclist:
-         if f and f[0]:
-             result.append((f[0].__name__,f[1]))
-         else:
-             result.append(f)
+    for f, name in zip(funclist, namelist):
+        if f and f[0]:
+            result.append((name, f[1]))
+        else:
+            result.append(f)
     return result
 
 # -----------------------------------------------------------------------------
@@ -407,15 +474,14 @@
 # Given a list of regular expression function names, this converts it back to
 # functions.
 # -----------------------------------------------------------------------------
-
-def _names_to_funcs(namelist,fdict):
-     result = []
-     for n in namelist:
-          if n and n[0]:
-              result.append((fdict[n[0]],n[1]))
-          else:
-              result.append(n)
-     return result
+def _names_to_funcs(namelist, fdict):
+    result = []
+    for n in namelist:
+        if n and n[0]:
+            result.append((fdict[n[0]], n[1]))
+        else:
+            result.append(n)
+    return result
 
 # -----------------------------------------------------------------------------
 # _form_master_re()
@@ -424,35 +490,37 @@
 # form the master regular expression.  Given limitations in the Python re
 # module, it may be necessary to break the master regex into separate expressions.
 # -----------------------------------------------------------------------------
-
-def _form_master_re(relist,reflags,ldict):
-    if not relist: return []
-    regex = "|".join(relist)
+def _form_master_re(relist, reflags, ldict, toknames):
+    if not relist:
+        return []
+    regex = '|'.join(relist)
     try:
-        lexre = re.compile(regex,re.VERBOSE | reflags)
+        lexre = re.compile(regex, reflags)
 
         # Build the index to function map for the matching engine
-        lexindexfunc = [ None ] * (max(lexre.groupindex.values())+1)
-        for f,i in lexre.groupindex.items():
-            handle = ldict.get(f,None)
+        lexindexfunc = [None] * (max(lexre.groupindex.values()) + 1)
+        lexindexnames = lexindexfunc[:]
+
+        for f, i in lexre.groupindex.items():
+            handle = ldict.get(f, None)
             if type(handle) in (types.FunctionType, types.MethodType):
-                lexindexfunc[i] = (handle,handle.__name__[2:])
+                lexindexfunc[i] = (handle, toknames[f])
+                lexindexnames[i] = f
             elif handle is not None:
-                # If rule was specified as a string, we build an anonymous
-                # callback function to carry out the action
-                if f.find("ignore_") > 0:
-                    lexindexfunc[i] = (None,None)
-                    print("IGNORE", f)
+                lexindexnames[i] = f
+                if f.find('ignore_') > 0:
+                    lexindexfunc[i] = (None, None)
                 else:
-                    lexindexfunc[i] = (None, f[2:])
-         
-        return [(lexre,lexindexfunc)],[regex]
-    except Exception as e:
+                    lexindexfunc[i] = (None, toknames[f])
+
+        return [(lexre, lexindexfunc)], [regex], [lexindexnames]
+    except Exception:
         m = int(len(relist)/2)
-        if m == 0: m = 1
-        llist, lre = _form_master_re(relist[:m],reflags,ldict)
-        rlist, rre = _form_master_re(relist[m:],reflags,ldict)
-        return llist+rlist, lre+rre
+        if m == 0:
+            m = 1
+        llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames)
+        rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames)
+        return (llist+rlist), (lre+rre), (lnames+rnames)
 
 # -----------------------------------------------------------------------------
 # def _statetoken(s,names)
@@ -462,362 +530,518 @@
 # is a tuple of state names and tokenname is the name of the token.  For example,
 # calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM')
 # -----------------------------------------------------------------------------
+def _statetoken(s, names):
+    parts = s.split('_')
+    for i, part in enumerate(parts[1:], 1):
+        if part not in names and part != 'ANY':
+            break
 
-def _statetoken(s,names):
-    nonstate = 1
-    parts = s.split("_")
-    for i in range(1,len(parts)):
-         if parts[i] not in names and parts[i] != 'ANY': break
     if i > 1:
-       states = tuple(parts[1:i])
+        states = tuple(parts[1:i])
     else:
-       states = ('INITIAL',)
+        states = ('INITIAL',)
 
     if 'ANY' in states:
-       states = tuple(names.keys())
-      
-    tokenname = "_".join(parts[i:])
-    return (states,tokenname)
+        states = tuple(names)
+
+    tokenname = '_'.join(parts[i:])
+    return (states, tokenname)
+
+
+# -----------------------------------------------------------------------------
+# LexerReflect()
+#
+# This class represents information needed to build a lexer as extracted from a
+# user's input file.
+# -----------------------------------------------------------------------------
+class LexerReflect(object):
+    def __init__(self, ldict, log=None, reflags=0):
+        self.ldict      = ldict
+        self.error_func = None
+        self.tokens     = []
+        self.reflags    = reflags
+        self.stateinfo  = {'INITIAL': 'inclusive'}
+        self.modules    = set()
+        self.error      = False
+        self.log        = PlyLogger(sys.stderr) if log is None else log
+
+    # Get all of the basic information
+    def get_all(self):
+        self.get_tokens()
+        self.get_literals()
+        self.get_states()
+        self.get_rules()
+
+    # Validate all of the information
+    def validate_all(self):
+        self.validate_tokens()
+        self.validate_literals()
+        self.validate_rules()
+        return self.error
+
+    # Get the tokens map
+    def get_tokens(self):
+        tokens = self.ldict.get('tokens', None)
+        if not tokens:
+            self.log.error('No token list is defined')
+            self.error = True
+            return
+
+        if not isinstance(tokens, (list, tuple)):
+            self.log.error('tokens must be a list or tuple')
+            self.error = True
+            return
+
+        if not tokens:
+            self.log.error('tokens is empty')
+            self.error = True
+            return
+
+        self.tokens = tokens
+
+    # Validate the tokens
+    def validate_tokens(self):
+        terminals = {}
+        for n in self.tokens:
+            if not _is_identifier.match(n):
+                self.log.error("Bad token name '%s'", n)
+                self.error = True
+            if n in terminals:
+                self.log.warning("Token '%s' multiply defined", n)
+            terminals[n] = 1
+
+    # Get the literals specifier
+    def get_literals(self):
+        self.literals = self.ldict.get('literals', '')
+        if not self.literals:
+            self.literals = ''
+
+    # Validate literals
+    def validate_literals(self):
+        try:
+            for c in self.literals:
+                if not isinstance(c, StringTypes) or len(c) > 1:
+                    self.log.error('Invalid literal %s. Must be a single character', repr(c))
+                    self.error = True
+
+        except TypeError:
+            self.log.error('Invalid literals specification. literals must be a sequence of characters')
+            self.error = True
+
+    def get_states(self):
+        self.states = self.ldict.get('states', None)
+        # Build statemap
+        if self.states:
+            if not isinstance(self.states, (tuple, list)):
+                self.log.error('states must be defined as a tuple or list')
+                self.error = True
+            else:
+                for s in self.states:
+                    if not isinstance(s, tuple) or len(s) != 2:
+                        self.log.error("Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')", repr(s))
+                        self.error = True
+                        continue
+                    name, statetype = s
+                    if not isinstance(name, StringTypes):
+                        self.log.error('State name %s must be a string', repr(name))
+                        self.error = True
+                        continue
+                    if not (statetype == 'inclusive' or statetype == 'exclusive'):
+                        self.log.error("State type for state %s must be 'inclusive' or 'exclusive'", name)
+                        self.error = True
+                        continue
+                    if name in self.stateinfo:
+                        self.log.error("State '%s' already defined", name)
+                        self.error = True
+                        continue
+                    self.stateinfo[name] = statetype
+
+    # Get all of the symbols with a t_ prefix and sort them into various
+    # categories (functions, strings, error functions, and ignore characters)
+
+    def get_rules(self):
+        tsymbols = [f for f in self.ldict if f[:2] == 't_']
+
+        # Now build up a list of functions and a list of strings
+        self.toknames = {}        # Mapping of symbols to token names
+        self.funcsym  = {}        # Symbols defined as functions
+        self.strsym   = {}        # Symbols defined as strings
+        self.ignore   = {}        # Ignore strings by state
+        self.errorf   = {}        # Error functions by state
+        self.eoff     = {}        # EOF functions by state
+
+        for s in self.stateinfo:
+            self.funcsym[s] = []
+            self.strsym[s] = []
+
+        if len(tsymbols) == 0:
+            self.log.error('No rules of the form t_rulename are defined')
+            self.error = True
+            return
+
+        for f in tsymbols:
+            t = self.ldict[f]
+            states, tokname = _statetoken(f, self.stateinfo)
+            self.toknames[f] = tokname
+
+            if hasattr(t, '__call__'):
+                if tokname == 'error':
+                    for s in states:
+                        self.errorf[s] = t
+                elif tokname == 'eof':
+                    for s in states:
+                        self.eoff[s] = t
+                elif tokname == 'ignore':
+                    line = t.__code__.co_firstlineno
+                    file = t.__code__.co_filename
+                    self.log.error("%s:%d: Rule '%s' must be defined as a string", file, line, t.__name__)
+                    self.error = True
+                else:
+                    for s in states:
+                        self.funcsym[s].append((f, t))
+            elif isinstance(t, StringTypes):
+                if tokname == 'ignore':
+                    for s in states:
+                        self.ignore[s] = t
+                    if '\\' in t:
+                        self.log.warning("%s contains a literal backslash '\\'", f)
+
+                elif tokname == 'error':
+                    self.log.error("Rule '%s' must be defined as a function", f)
+                    self.error = True
+                else:
+                    for s in states:
+                        self.strsym[s].append((f, t))
+            else:
+                self.log.error('%s not defined as a function or string', f)
+                self.error = True
+
+        # Sort the functions by line number
+        for f in self.funcsym.values():
+            f.sort(key=lambda x: x[1].__code__.co_firstlineno)
+
+        # Sort the strings by regular expression length
+        for s in self.strsym.values():
+            s.sort(key=lambda x: len(x[1]), reverse=True)
+
+    # Validate all of the t_rules collected
+    def validate_rules(self):
+        for state in self.stateinfo:
+            # Validate all rules defined by functions
+
+            for fname, f in self.funcsym[state]:
+                line = f.__code__.co_firstlineno
+                file = f.__code__.co_filename
+                module = inspect.getmodule(f)
+                self.modules.add(module)
+
+                tokname = self.toknames[fname]
+                if isinstance(f, types.MethodType):
+                    reqargs = 2
+                else:
+                    reqargs = 1
+                nargs = f.__code__.co_argcount
+                if nargs > reqargs:
+                    self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__)
+                    self.error = True
+                    continue
+
+                if nargs < reqargs:
+                    self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__)
+                    self.error = True
+                    continue
+
+                if not _get_regex(f):
+                    self.log.error("%s:%d: No regular expression defined for rule '%s'", file, line, f.__name__)
+                    self.error = True
+                    continue
+
+                try:
+                    c = re.compile('(?P<%s>%s)' % (fname, _get_regex(f)), self.reflags)
+                    if c.match(''):
+                        self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file, line, f.__name__)
+                        self.error = True
+                except re.error as e:
+                    self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file, line, f.__name__, e)
+                    if '#' in _get_regex(f):
+                        self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'", file, line, f.__name__)
+                    self.error = True
+
+            # Validate all rules defined by strings
+            for name, r in self.strsym[state]:
+                tokname = self.toknames[name]
+                if tokname == 'error':
+                    self.log.error("Rule '%s' must be defined as a function", name)
+                    self.error = True
+                    continue
+
+                if tokname not in self.tokens and tokname.find('ignore_') < 0:
+                    self.log.error("Rule '%s' defined for an unspecified token %s", name, tokname)
+                    self.error = True
+                    continue
+
+                try:
+                    c = re.compile('(?P<%s>%s)' % (name, r), self.reflags)
+                    if (c.match('')):
+                        self.log.error("Regular expression for rule '%s' matches empty string", name)
+                        self.error = True
+                except re.error as e:
+                    self.log.error("Invalid regular expression for rule '%s'. %s", name, e)
+                    if '#' in r:
+                        self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'", name)
+                    self.error = True
+
+            if not self.funcsym[state] and not self.strsym[state]:
+                self.log.error("No rules defined for state '%s'", state)
+                self.error = True
+
+            # Validate the error function
+            efunc = self.errorf.get(state, None)
+            if efunc:
+                f = efunc
+                line = f.__code__.co_firstlineno
+                file = f.__code__.co_filename
+                module = inspect.getmodule(f)
+                self.modules.add(module)
+
+                if isinstance(f, types.MethodType):
+                    reqargs = 2
+                else:
+                    reqargs = 1
+                nargs = f.__code__.co_argcount
+                if nargs > reqargs:
+                    self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__)
+                    self.error = True
+
+                if nargs < reqargs:
+                    self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__)
+                    self.error = True
+
+        for module in self.modules:
+            self.validate_module(module)
+
+    # -----------------------------------------------------------------------------
+    # validate_module()
+    #
+    # This checks to see if there are duplicated t_rulename() functions or strings
+    # in the parser input file.  This is done using a simple regular expression
+    # match on each line in the source code of the given module.
+    # -----------------------------------------------------------------------------
+
+    def validate_module(self, module):
+        try:
+            lines, linen = inspect.getsourcelines(module)
+        except IOError:
+            return
+
+        fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(')
+        sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=')
+
+        counthash = {}
+        linen += 1
+        for line in lines:
+            m = fre.match(line)
+            if not m:
+                m = sre.match(line)
+            if m:
+                name = m.group(1)
+                prev = counthash.get(name)
+                if not prev:
+                    counthash[name] = linen
+                else:
+                    filename = inspect.getsourcefile(module)
+                    self.log.error('%s:%d: Rule %s redefined. Previously defined on line %d', filename, linen, name, prev)
+                    self.error = True
+            linen += 1
 
 # -----------------------------------------------------------------------------
 # lex(module)
 #
 # Build all of the regular expression rules from definitions in the supplied module
 # -----------------------------------------------------------------------------
-def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,nowarn=0):
+def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab',
+        reflags=int(re.VERBOSE), nowarn=False, outputdir=None, debuglog=None, errorlog=None):
+
+    if lextab is None:
+        lextab = 'lextab'
+
     global lexer
+
     ldict = None
-    stateinfo  = { 'INITIAL' : 'inclusive'}
-    error = 0
-    files = { }
+    stateinfo  = {'INITIAL': 'inclusive'}
     lexobj = Lexer()
-    lexobj.lexdebug = debug
     lexobj.lexoptimize = optimize
-    global token,input
+    global token, input
 
-    if nowarn: warn = 0
-    else: warn = 1
-    
-    if object: module = object
+    if errorlog is None:
+        errorlog = PlyLogger(sys.stderr)
 
+    if debug:
+        if debuglog is None:
+            debuglog = PlyLogger(sys.stderr)
+
+    # Get the module dictionary used for the lexer
+    if object:
+        module = object
+
+    # Get the module dictionary used for the parser
     if module:
-        # User supplied a module object.
-        if isinstance(module, types.ModuleType):
-            ldict = module.__dict__
-        elif isinstance(module, _INSTANCETYPE):
-            _items = [(k,getattr(module,k)) for k in dir(module)]
-            ldict = { }
-            for (i,v) in _items:
-                ldict[i] = v
-        else:
-            raise ValueError("Expected a module or instance")
-        lexobj.lexmodule = module
-        
+        _items = [(k, getattr(module, k)) for k in dir(module)]
+        ldict = dict(_items)
+        # If no __file__ attribute is available, try to obtain it from the __module__ instead
+        if '__file__' not in ldict:
+            ldict['__file__'] = sys.modules[ldict['__module__']].__file__
     else:
-        # No module given.  We might be able to get information from the caller.
-        try:
-            raise RuntimeError
-        except RuntimeError:
-            e,b,t = sys.exc_info()
-            f = t.tb_frame
-            f = f.f_back           # Walk out to our calling function
-            ldict = f.f_globals    # Grab its globals dictionary
+        ldict = get_caller_module_dict(2)
+
+    # Determine if the module is package of a package or not.
+    # If so, fix the tabmodule setting so that tables load correctly
+    pkg = ldict.get('__package__')
+    if pkg and isinstance(lextab, str):
+        if '.' not in lextab:
+            lextab = pkg + '.' + lextab
+
+    # Collect parser information from the dictionary
+    linfo = LexerReflect(ldict, log=errorlog, reflags=reflags)
+    linfo.get_all()
+    if not optimize:
+        if linfo.validate_all():
+            raise SyntaxError("Can't build lexer")
 
     if optimize and lextab:
         try:
-            lexobj.readtab(lextab,ldict)
+            lexobj.readtab(lextab, ldict)
             token = lexobj.token
             input = lexobj.input
             lexer = lexobj
             return lexobj
-        
+
         except ImportError:
             pass
-        
-    # Get the tokens, states, and literals variables (if any)
-    if (module and isinstance(module,_INSTANCETYPE)):
-        tokens   = getattr(module,"tokens",None)
-        states   = getattr(module,"states",None)
-        literals = getattr(module,"literals","")
-    else:
-        tokens   = ldict.get("tokens",None)
-        states   = ldict.get("states",None)
-        literals = ldict.get("literals","")
-        
-    if not tokens:
-        raise SyntaxError("lex: module does not define 'tokens'")
-    if not (isinstance(tokens,list) or isinstance(tokens,tuple)):
-        raise SyntaxError("lex: tokens must be a list or tuple.")
+
+    # Dump some basic debugging information
+    if debug:
+        debuglog.info('lex: tokens   = %r', linfo.tokens)
+        debuglog.info('lex: literals = %r', linfo.literals)
+        debuglog.info('lex: states   = %r', linfo.stateinfo)
 
     # Build a dictionary of valid token names
-    lexobj.lextokens = { }
-    if not optimize:
-        for n in tokens:
-            if not _is_identifier.match(n):
-                print("lex: Bad token name '%s'" % n)
-                error = 1
-            if warn and n in lexobj.lextokens:
-                print("lex: Warning. Token '%s' multiply defined." % n)
-            lexobj.lextokens[n] = None
+    lexobj.lextokens = set()
+    for n in linfo.tokens:
+        lexobj.lextokens.add(n)
+
+    # Get literals specification
+    if isinstance(linfo.literals, (list, tuple)):
+        lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals)
     else:
-        for n in tokens: lexobj.lextokens[n] = None
+        lexobj.lexliterals = linfo.literals
 
-    if debug:
-        print("lex: tokens = '%s'" % list(lexobj.lextokens.keys()))
+    lexobj.lextokens_all = lexobj.lextokens | set(lexobj.lexliterals)
 
-    try:
-         for c in literals:
-               if not (isinstance(c,util.bytes_type) or isinstance(c, util.string_type)) or len(c) > 1:
-                    print("lex: Invalid literal %s. Must be a single character" % repr(c))
-                    error = 1
-                    continue
+    # Get the stateinfo dictionary
+    stateinfo = linfo.stateinfo
 
-    except TypeError:
-         print("lex: Invalid literals specification. literals must be a sequence of characters.")
-         error = 1
-
-    lexobj.lexliterals = literals
-
-    # Build statemap
-    if states:
-         if not (isinstance(states,tuple) or isinstance(states,list)):
-              print("lex: states must be defined as a tuple or list.")
-              error = 1
-         else:
-              for s in states:
-                    if not isinstance(s,tuple) or len(s) != 2:
-                           print("lex: invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')" % repr(s))
-                           error = 1
-                           continue
-                    name, statetype = s
-                    if isinstance(name, util.string_type):
-                           original_name = name
-                           name = util.encode_input(name)
-                    if not isinstance(name,util.bytes_type) or len(original_name) != len(name):
-                           print("lex: state name %s must be a byte string" % repr(original_name))
-                           error = 1
-                           continue
-                    if not (statetype == 'inclusive' or statetype == 'exclusive'):
-                           print("lex: state type for state %s must be 'inclusive' or 'exclusive'" % name)
-                           error = 1
-                           continue
-                    if name in stateinfo:
-                           print("lex: state '%s' already defined." % name)
-                           error = 1
-                           continue
-                    stateinfo[name] = statetype
-
-    # Get a list of symbols with the t_ or s_ prefix
-    tsymbols = [f for f in ldict.keys() if f[:2] == 't_' ]
-
-    # Now build up a list of functions and a list of strings
-
-    funcsym =  { }        # Symbols defined as functions
-    strsym =   { }        # Symbols defined as strings
-    toknames = { }        # Mapping of symbols to token names
-
-    for s in stateinfo.keys():
-         funcsym[s] = []
-         strsym[s] = []
-
-    ignore   = { }        # Ignore strings by state
-    errorf   = { }        # Error functions by state
-
-    if len(tsymbols) == 0:
-        raise SyntaxError("lex: no rules of the form t_rulename are defined.")
-
-    for f in tsymbols:
-        t = ldict[f]
-        states, tokname = _statetoken(f,stateinfo)
-        toknames[f] = tokname
-
-        if isinstance(t, collections.Callable):
-            for s in states: funcsym[s].append((f,t))
-        elif (isinstance(t, util.bytes_type) or isinstance(t,util.string_type)):
-            for s in states: strsym[s].append((f,t))
-        else:
-            print("lex: %s not defined as a function or string" % f)
-            error = 1
-
-    # Sort the functions by line number
-    for f in funcsym.values():
-        f.sort(key=lambda x: x[1].__code__.co_firstlineno)
-
-    # Sort the strings by regular expression length
-    for s in strsym.values():
-        s.sort(key=lambda x: len(x[1]))
-
-    regexs = { }
-
+    regexs = {}
     # Build the master regular expressions
-    for state in stateinfo.keys():
+    for state in stateinfo:
         regex_list = []
 
         # Add rules defined by functions first
-        for fname, f in funcsym[state]:
-            line = f.__code__.co_firstlineno
-            file = f.__code__.co_filename
-            files[file] = None
-            tokname = toknames[fname]
-
-            ismethod = isinstance(f, types.MethodType)
-
-            if not optimize:
-                nargs = f.__code__.co_argcount
-                if ismethod:
-                    reqargs = 2
-                else:
-                    reqargs = 1
-                if nargs > reqargs:
-                    print("%s:%d: Rule '%s' has too many arguments." % (file,line,f.__name__))
-                    error = 1
-                    continue
-
-                if nargs < reqargs:
-                    print("%s:%d: Rule '%s' requires an argument." % (file,line,f.__name__))
-                    error = 1
-                    continue
-
-                if tokname == 'ignore':
-                    print("%s:%d: Rule '%s' must be defined as a string." % (file,line,f.__name__))
-                    error = 1
-                    continue
-        
-            if tokname == 'error':
-                errorf[state] = f
-                continue
-
-            if f.__doc__:
-                if not optimize:
-                    try:
-                        c = re.compile("(?P<%s>%s)" % (f.__name__,f.__doc__), re.VERBOSE | reflags)
-                        if c.match(""):
-                             print("%s:%d: Regular expression for rule '%s' matches empty string." % (file,line,f.__name__))
-                             error = 1
-                             continue
-                    except re.error as e:
-                        print("%s:%d: Invalid regular expression for rule '%s'. %s" % (file,line,f.__name__,e))
-                        if '#' in f.__doc__:
-                             print("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'." % (file,line, f.__name__))
-                        error = 1
-                        continue
-
-                    if debug:
-                        print("lex: Adding rule %s -> '%s' (state '%s')" % (f.__name__,f.__doc__, state))
-
-                # Okay. The regular expression seemed okay.  Let's append it to the master regular
-                # expression we're building
-  
-                regex_list.append("(?P<%s>%s)" % (f.__name__,f.__doc__))
-            else:
-                print("%s:%d: No regular expression defined for rule '%s'" % (file,line,f.__name__))
+        for fname, f in linfo.funcsym[state]:
+            regex_list.append('(?P<%s>%s)' % (fname, _get_regex(f)))
+            if debug:
+                debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", fname, _get_regex(f), state)
 
         # Now add all of the simple rules
-        for name,r in strsym[state]:
-            tokname = toknames[name]       
-
-            if tokname == 'ignore':
-                 ignore[state] = r
-                 continue
-
-            if not optimize:
-                if tokname == 'error':
-                    raise SyntaxError("lex: Rule '%s' must be defined as a function" % name)
-                    error = 1
-                    continue
-        
-                if tokname not in lexobj.lextokens and tokname.find("ignore_") < 0:
-                    print("lex: Rule '%s' defined for an unspecified token %s." % (name,tokname))
-                    error = 1
-                    continue
-                try:
-                    c = re.compile("(?P<%s>%s)" % (name,r),re.VERBOSE | reflags)
-                    if (c.match("")):
-                         print("lex: Regular expression for rule '%s' matches empty string." % name)
-                         error = 1
-                         continue
-                except re.error as e:
-                    print("lex: Invalid regular expression for rule '%s'. %s" % (name,e))
-                    if '#' in r:
-                         print("lex: Make sure '#' in rule '%s' is escaped with '\\#'." % name)
-
-                    error = 1
-                    continue
-                if debug:
-                    print("lex: Adding rule %s -> '%s' (state '%s')" % (name,r,state))
-                
-            regex_list.append("(?P<%s>%s)" % (name,r))
-
-        if not regex_list:
-             print("lex: No rules defined for state '%s'" % state)
-             error = 1
+        for name, r in linfo.strsym[state]:
+            regex_list.append('(?P<%s>%s)' % (name, r))
+            if debug:
+                debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", name, r, state)
 
         regexs[state] = regex_list
 
-
-    if not optimize:
-        for f in files.keys(): 
-           if not _validate_file(f):
-                error = 1
-
-    if error:
-        raise SyntaxError("lex: Unable to build lexer.")
-
-    # From this point forward, we're reasonably confident that we can build the lexer.
-    # No more errors will be generated, but there might be some warning messages.
-
     # Build the master regular expressions
 
-    for state in regexs.keys():
-        lexre, re_text = _form_master_re(regexs[state],reflags,ldict)
+    if debug:
+        debuglog.info('lex: ==== MASTER REGEXS FOLLOW ====')
+
+    for state in regexs:
+        lexre, re_text, re_names = _form_master_re(regexs[state], reflags, ldict, linfo.toknames)
         lexobj.lexstatere[state] = lexre
         lexobj.lexstateretext[state] = re_text
+        lexobj.lexstaterenames[state] = re_names
         if debug:
-            for i in range(len(re_text)):
-                 print("lex: state '%s'. regex[%d] = '%s'" % (state, i, re_text[i]))
+            for i, text in enumerate(re_text):
+                debuglog.info("lex: state '%s' : regex[%d] = '%s'", state, i, text)
 
-    # For inclusive states, we need to add the INITIAL state
-    for state,type in stateinfo.items():
-        if state != "INITIAL" and type == 'inclusive':
-             lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL'])
-             lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL'])
+    # For inclusive states, we need to add the regular expressions from the INITIAL state
+    for state, stype in stateinfo.items():
+        if state != 'INITIAL' and stype == 'inclusive':
+            lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL'])
+            lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL'])
+            lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL'])
 
     lexobj.lexstateinfo = stateinfo
-    lexobj.lexre = lexobj.lexstatere["INITIAL"]
-    lexobj.lexretext = lexobj.lexstateretext["INITIAL"]
+    lexobj.lexre = lexobj.lexstatere['INITIAL']
+    lexobj.lexretext = lexobj.lexstateretext['INITIAL']
+    lexobj.lexreflags = reflags
 
     # Set up ignore variables
-    lexobj.lexstateignore = ignore
-    lexobj.lexignore = lexobj.lexstateignore.get("INITIAL","")
+    lexobj.lexstateignore = linfo.ignore
+    lexobj.lexignore = lexobj.lexstateignore.get('INITIAL', '')
 
     # Set up error functions
-    lexobj.lexstateerrorf = errorf
-    lexobj.lexerrorf = errorf.get("INITIAL",None)
-    if warn and not lexobj.lexerrorf:
-        print("lex: Warning. no t_error rule is defined.")
+    lexobj.lexstateerrorf = linfo.errorf
+    lexobj.lexerrorf = linfo.errorf.get('INITIAL', None)
+    if not lexobj.lexerrorf:
+        errorlog.warning('No t_error rule is defined')
+
+    # Set up eof functions
+    lexobj.lexstateeoff = linfo.eoff
+    lexobj.lexeoff = linfo.eoff.get('INITIAL', None)
 
     # Check state information for ignore and error rules
-    for s,stype in stateinfo.items():
+    for s, stype in stateinfo.items():
         if stype == 'exclusive':
-              if warn and s not in errorf:
-                   print("lex: Warning. no error rule is defined for exclusive state '%s'" % s)
-              if warn and s not in ignore and lexobj.lexignore:
-                   print("lex: Warning. no ignore rule is defined for exclusive state '%s'" % s)
+            if s not in linfo.errorf:
+                errorlog.warning("No error rule is defined for exclusive state '%s'", s)
+            if s not in linfo.ignore and lexobj.lexignore:
+                errorlog.warning("No ignore rule is defined for exclusive state '%s'", s)
         elif stype == 'inclusive':
-              if s not in errorf:
-                   errorf[s] = errorf.get("INITIAL",None)
-              if s not in ignore:
-                   ignore[s] = ignore.get("INITIAL","")
-   
+            if s not in linfo.errorf:
+                linfo.errorf[s] = linfo.errorf.get('INITIAL', None)
+            if s not in linfo.ignore:
+                linfo.ignore[s] = linfo.ignore.get('INITIAL', '')
 
     # Create global versions of the token() and input() functions
     token = lexobj.token
     input = lexobj.input
     lexer = lexobj
 
-    # If in optimize mode, we write the lextab   
+    # If in optimize mode, we write the lextab
     if lextab and optimize:
-        lexobj.writetab(lextab)
+        if outputdir is None:
+            # If no output directory is set, the location of the output files
+            # is determined according to the following rules:
+            #     - If lextab specifies a package, files go into that package directory
+            #     - Otherwise, files go in the same directory as the specifying module
+            if isinstance(lextab, types.ModuleType):
+                srcfile = lextab.__file__
+            else:
+                if '.' not in lextab:
+                    srcfile = ldict['__file__']
+                else:
+                    parts = lextab.split('.')
+                    pkgname = '.'.join(parts[:-1])
+                    exec('import %s' % pkgname)
+                    srcfile = getattr(sys.modules[pkgname], '__file__', '')
+            outputdir = os.path.dirname(srcfile)
+        try:
+            lexobj.writetab(lextab, outputdir)
+            if lextab in sys.modules:
+                del sys.modules[lextab]
+        except IOError as e:
+            errorlog.warning("Couldn't write lextab module %r. %s" % (lextab, e))
 
     return lexobj
 
@@ -827,7 +1051,7 @@
 # This runs the lexer as a main program
 # -----------------------------------------------------------------------------
 
-def runmain(lexer=None,data=None):
+def runmain(lexer=None, data=None):
     if not data:
         try:
             filename = sys.argv[1]
@@ -835,7 +1059,7 @@
             data = f.read()
             f.close()
         except IndexError:
-            print("Reading from standard input (type EOF to end):")
+            sys.stdout.write('Reading from standard input (type EOF to end):\n')
             data = sys.stdin.read()
 
     if lexer:
@@ -847,12 +1071,12 @@
         _token = lexer.token
     else:
         _token = token
-        
-    while 1:
+
+    while True:
         tok = _token()
-        if not tok: break
-        print("(%s,%r,%d,%d)" % (tok.type, tok.value, tok.lineno,tok.lexpos))
-        
+        if not tok:
+            break
+        sys.stdout.write('(%s,%r,%d,%d)\n' % (tok.type, tok.value, tok.lineno, tok.lexpos))
 
 # -----------------------------------------------------------------------------
 # @TOKEN(regex)
@@ -862,11 +1086,13 @@
 # -----------------------------------------------------------------------------
 
 def TOKEN(r):
-    def set_doc(f):
-        f.__doc__ = r
+    def set_regex(f):
+        if hasattr(r, '__call__'):
+            f.regex = _get_regex(r)
+        else:
+            f.regex = r
         return f
-    return set_doc
+    return set_regex
 
 # Alternative spelling of the TOKEN decorator
 Token = TOKEN
-
diff --git a/python/sepolgen/src/sepolgen/yacc.py b/python/sepolgen/src/sepolgen/yacc.py
index afef174..88188a1 100644
--- a/python/sepolgen/src/sepolgen/yacc.py
+++ b/python/sepolgen/src/sepolgen/yacc.py
@@ -1,29 +1,38 @@
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # ply: yacc.py
 #
-# Author(s): David M. Beazley (dave@dabeaz.com)
+# Copyright (C) 2001-2018
+# David M. Beazley (Dabeaz LLC)
+# All rights reserved.
 #
-# Copyright (C) 2001-2006, David M. Beazley
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
 #
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-# 
-# See the file COPYING for a complete copy of the LGPL.
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the David Beazley or Dabeaz LLC may be used to
+#   endorse or promote products derived from this software without
+#  specific prior written permission.
 #
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
 #
 # This implements an LR parser that is constructed from grammar rules defined
-# as Python functions. The grammer is specified by supplying the BNF inside
+# as Python functions. The grammar is specified by supplying the BNF inside
 # Python documentation strings.  The inspiration for this technique was borrowed
 # from John Aycock's Spark parsing system.  PLY might be viewed as cross between
 # Spark and the GNU bison utility.
@@ -50,7 +59,15 @@
 # own risk!
 # ----------------------------------------------------------------------------
 
-__version__ = "2.2"
+import re
+import types
+import sys
+import os.path
+import inspect
+import warnings
+
+__version__    = '3.11'
+__tabversion__ = '3.10'
 
 #-----------------------------------------------------------------------------
 #                     === User configurable parameters ===
@@ -58,7 +75,7 @@
 # Change these to modify the default behavior of yacc (if you wish)
 #-----------------------------------------------------------------------------
 
-yaccdebug   = 1                # Debugging mode.  If set, yacc generates a
+yaccdebug   = True             # Debugging mode.  If set, yacc generates a
                                # a 'parser.out' file in the current directory
 
 debug_file  = 'parser.out'     # Default name of the debugging file
@@ -67,16 +84,117 @@
 
 error_count = 3                # Number of symbols that must be shifted to leave recovery mode
 
-import re, types, sys, hashlib, os.path
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
+yaccdevel   = False            # Set to True if developing yacc.  This turns off optimized
+                               # implementations of certain functions.
 
-from . import util
+resultlimit = 40               # Size limit of results when running in debug mode.
+
+pickle_protocol = 0            # Protocol to use when writing pickle files
+
+# String type-checking compatibility
+if sys.version_info[0] < 3:
+    string_types = basestring
+else:
+    string_types = str
+
+MAXINT = sys.maxsize
+
+# This object is a stand-in for a logging object created by the
+# logging module.   PLY will use this by default to create things
+# such as the parser.out file.  If a user wants more detailed
+# information, they can create their own logging object and pass
+# it into PLY.
+
+class PlyLogger(object):
+    def __init__(self, f):
+        self.f = f
+
+    def debug(self, msg, *args, **kwargs):
+        self.f.write((msg % args) + '\n')
+
+    info = debug
+
+    def warning(self, msg, *args, **kwargs):
+        self.f.write('WARNING: ' + (msg % args) + '\n')
+
+    def error(self, msg, *args, **kwargs):
+        self.f.write('ERROR: ' + (msg % args) + '\n')
+
+    critical = debug
+
+# Null logger is used when no output is generated. Does nothing.
+class NullLogger(object):
+    def __getattribute__(self, name):
+        return self
+
+    def __call__(self, *args, **kwargs):
+        return self
 
 # Exception raised for yacc-related errors
-class YaccError(Exception):   pass
+class YaccError(Exception):
+    pass
+
+# Format the result message that the parser produces when running in debug mode.
+def format_result(r):
+    repr_str = repr(r)
+    if '\n' in repr_str:
+        repr_str = repr(repr_str)
+    if len(repr_str) > resultlimit:
+        repr_str = repr_str[:resultlimit] + ' ...'
+    result = '<%s @ 0x%x> (%s)' % (type(r).__name__, id(r), repr_str)
+    return result
+
+# Format stack entries when the parser is running in debug mode
+def format_stack_entry(r):
+    repr_str = repr(r)
+    if '\n' in repr_str:
+        repr_str = repr(repr_str)
+    if len(repr_str) < 16:
+        return repr_str
+    else:
+        return '<%s @ 0x%x>' % (type(r).__name__, id(r))
+
+# Panic mode error recovery support.   This feature is being reworked--much of the
+# code here is to offer a deprecation/backwards compatible transition
+
+_errok = None
+_token = None
+_restart = None
+_warnmsg = '''PLY: Don't use global functions errok(), token(), and restart() in p_error().
+Instead, invoke the methods on the associated parser instance:
+
+    def p_error(p):
+        ...
+        # Use parser.errok(), parser.token(), parser.restart()
+        ...
+
+    parser = yacc.yacc()
+'''
+
+def errok():
+    warnings.warn(_warnmsg)
+    return _errok()
+
+def restart():
+    warnings.warn(_warnmsg)
+    return _restart()
+
+def token():
+    warnings.warn(_warnmsg)
+    return _token()
+
+# Utility function to call the p_error() function with some deprecation hacks
+def call_errorfunc(errorfunc, token, parser):
+    global _errok, _token, _restart
+    _errok = parser.errok
+    _token = parser.token
+    _restart = parser.restart
+    r = errorfunc(token)
+    try:
+        del _errok, _token, _restart
+    except NameError:
+        pass
+    return r
 
 #-----------------------------------------------------------------------------
 #                        ===  LR Parsing Engine ===
@@ -96,8 +214,11 @@
 #        .endlexpos  = Ending lex position (optional, set automatically)
 
 class YaccSymbol:
-    def __str__(self):    return self.type
-    def __repr__(self):   return str(self)
+    def __str__(self):
+        return self.type
+
+    def __repr__(self):
+        return str(self)
 
 # This class is a wrapper around the objects actually passed to each
 # grammar rule.   Index lookup and assignment actually assign the
@@ -109,72 +230,71 @@
 # representing the range of positional information for a symbol.
 
 class YaccProduction:
-    def __init__(self,s,stack=None):
+    def __init__(self, s, stack=None):
         self.slice = s
-        self.pbstack = []
         self.stack = stack
+        self.lexer = None
+        self.parser = None
 
-    def __getitem__(self,n):
-        if type(n) == int:
-             if n >= 0: return self.slice[n].value
-             else: return self.stack[n].value
+    def __getitem__(self, n):
+        if isinstance(n, slice):
+            return [s.value for s in self.slice[n]]
+        elif n >= 0:
+            return self.slice[n].value
         else:
-             return [s.value for s in self.slice[n.start:n.stop:n.step]]
+            return self.stack[n].value
 
-    def __setitem__(self,n,v):
+    def __setitem__(self, n, v):
         self.slice[n].value = v
 
+    def __getslice__(self, i, j):
+        return [s.value for s in self.slice[i:j]]
+
     def __len__(self):
         return len(self.slice)
-    
-    def lineno(self,n):
-        return getattr(self.slice[n],"lineno",0)
 
-    def linespan(self,n):
-        startline = getattr(self.slice[n],"lineno",0)
-        endline = getattr(self.slice[n],"endlineno",startline)
-        return startline,endline
+    def lineno(self, n):
+        return getattr(self.slice[n], 'lineno', 0)
 
-    def lexpos(self,n):
-        return getattr(self.slice[n],"lexpos",0)
+    def set_lineno(self, n, lineno):
+        self.slice[n].lineno = lineno
 
-    def lexspan(self,n):
-        startpos = getattr(self.slice[n],"lexpos",0)
-        endpos = getattr(self.slice[n],"endlexpos",startpos)
-        return startpos,endpos
+    def linespan(self, n):
+        startline = getattr(self.slice[n], 'lineno', 0)
+        endline = getattr(self.slice[n], 'endlineno', startline)
+        return startline, endline
 
-    def pushback(self,n):
-        if n <= 0:
-            raise ValueError("Expected a positive value")
-        if n > (len(self.slice)-1):
-            raise ValueError("Can't push %d tokens. Only %d are available." % (n,len(self.slice)-1))
-        for i in range(0,n):
-            self.pbstack.append(self.slice[-i-1])
+    def lexpos(self, n):
+        return getattr(self.slice[n], 'lexpos', 0)
 
-# The LR Parsing engine.   This is defined as a class so that multiple parsers
-# can exist in the same process.  A user never instantiates this directly.
-# Instead, the global yacc() function should be used to create a suitable Parser
-# object. 
+    def set_lexpos(self, n, lexpos):
+        self.slice[n].lexpos = lexpos
 
-class Parser:
-    def __init__(self,magic=None):
+    def lexspan(self, n):
+        startpos = getattr(self.slice[n], 'lexpos', 0)
+        endpos = getattr(self.slice[n], 'endlexpos', startpos)
+        return startpos, endpos
 
-        # This is a hack to keep users from trying to instantiate a Parser
-        # object directly.
+    def error(self):
+        raise SyntaxError
 
-        if magic != "xyzzy":
-            raise YaccError("Can't instantiate Parser. Use yacc() instead.")
+# -----------------------------------------------------------------------------
+#                               == LRParser ==
+#
+# The LR Parsing engine.
+# -----------------------------------------------------------------------------
 
-        # Reset internal state
-        self.productions = None          # List of productions
-        self.errorfunc   = None          # Error handling function
-        self.action      = { }           # LR Action table
-        self.goto        = { }           # LR goto table
-        self.require     = { }           # Attribute require table
-        self.method      = "Unknown LR"  # Table construction method used
+class LRParser:
+    def __init__(self, lrtab, errorf):
+        self.productions = lrtab.lr_productions
+        self.action = lrtab.lr_action
+        self.goto = lrtab.lr_goto
+        self.errorfunc = errorf
+        self.set_defaulted_states()
+        self.errorok = True
 
     def errok(self):
-        self.errorcount = 0
+        self.errorok = True
 
     def restart(self):
         del self.statestack[:]
@@ -183,88 +303,156 @@
         sym.type = '$end'
         self.symstack.append(sym)
         self.statestack.append(0)
-        
-    def parse(self,input=None,lexer=None,debug=0):
-        lookahead = None                 # Current lookahead symbol
-        lookaheadstack = [ ]             # Stack of lookahead symbols
-        actions = self.action            # Local reference to action table
-        goto    = self.goto              # Local reference to goto table
-        prod    = self.productions       # Local reference to production list
-        pslice  = YaccProduction(None)   # Production object passed to grammar rules
-        pslice.parser = self             # Parser object
-        self.errorcount = 0              # Used during error recovery
+
+    # Defaulted state support.
+    # This method identifies parser states where there is only one possible reduction action.
+    # For such states, the parser can make a choose to make a rule reduction without consuming
+    # the next look-ahead token.  This delayed invocation of the tokenizer can be useful in
+    # certain kinds of advanced parsing situations where the lexer and parser interact with
+    # each other or change states (i.e., manipulation of scope, lexer states, etc.).
+    #
+    # See:  http://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html#Default-Reductions
+    def set_defaulted_states(self):
+        self.defaulted_states = {}
+        for state, actions in self.action.items():
+            rules = list(actions.values())
+            if len(rules) == 1 and rules[0] < 0:
+                self.defaulted_states[state] = rules[0]
+
+    def disable_defaulted_states(self):
+        self.defaulted_states = {}
+
+    def parse(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):
+        if debug or yaccdevel:
+            if isinstance(debug, int):
+                debug = PlyLogger(sys.stderr)
+            return self.parsedebug(input, lexer, debug, tracking, tokenfunc)
+        elif tracking:
+            return self.parseopt(input, lexer, debug, tracking, tokenfunc)
+        else:
+            return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc)
+
+
+    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    # parsedebug().
+    #
+    # This is the debugging enabled version of parse().  All changes made to the
+    # parsing engine should be made here.   Optimized versions of this function
+    # are automatically created by the ply/ygen.py script.  This script cuts out
+    # sections enclosed in markers such as this:
+    #
+    #      #--! DEBUG
+    #      statements
+    #      #--! DEBUG
+    #
+    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+    def parsedebug(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):
+        #--! parsedebug-start
+        lookahead = None                         # Current lookahead symbol
+        lookaheadstack = []                      # Stack of lookahead symbols
+        actions = self.action                    # Local reference to action table (to avoid lookup on self.)
+        goto    = self.goto                      # Local reference to goto table (to avoid lookup on self.)
+        prod    = self.productions               # Local reference to production list (to avoid lookup on self.)
+        defaulted_states = self.defaulted_states # Local reference to defaulted states
+        pslice  = YaccProduction(None)           # Production object passed to grammar rules
+        errorcount = 0                           # Used during error recovery
+
+        #--! DEBUG
+        debug.info('PLY: PARSE DEBUG START')
+        #--! DEBUG
 
         # If no lexer was given, we will try to use the lex module
         if not lexer:
             from . import lex
             lexer = lex.lexer
 
+        # Set up the lexer and parser objects on pslice
         pslice.lexer = lexer
-        
+        pslice.parser = self
+
         # If input was supplied, pass to lexer
-        if input:
+        if input is not None:
             lexer.input(input)
 
-        # Tokenize function
-        get_token = lexer.token
+        if tokenfunc is None:
+            # Tokenize function
+            get_token = lexer.token
+        else:
+            get_token = tokenfunc
 
-        statestack = [ ]                # Stack of parsing states
+        # Set the parser() token method (sometimes used in error recovery)
+        self.token = get_token
+
+        # Set up the state and symbol stacks
+
+        statestack = []                # Stack of parsing states
         self.statestack = statestack
-        symstack   = [ ]                # Stack of grammar symbols
+        symstack   = []                # Stack of grammar symbols
         self.symstack = symstack
 
         pslice.stack = symstack         # Put in the production
         errtoken   = None               # Err token
 
         # The start state is assumed to be (0,$end)
+
         statestack.append(0)
         sym = YaccSymbol()
         sym.type = '$end'
         symstack.append(sym)
-        
-        while 1:
+        state = 0
+        while True:
             # Get the next symbol on the input.  If a lookahead symbol
             # is already set, we just use that. Otherwise, we'll pull
             # the next token off of the lookaheadstack or from the lexer
-            if debug > 1:
-                print('state', statestack[-1])
-            if not lookahead:
-                if not lookaheadstack:
-                    lookahead = get_token()     # Get the next token
-                else:
-                    lookahead = lookaheadstack.pop()
+
+            #--! DEBUG
+            debug.debug('')
+            debug.debug('State  : %s', state)
+            #--! DEBUG
+
+            if state not in defaulted_states:
                 if not lookahead:
-                    lookahead = YaccSymbol()
-                    lookahead.type = '$end'
-            if debug:
-                errorlead = ("%s . %s" % (" ".join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip()
+                    if not lookaheadstack:
+                        lookahead = get_token()     # Get the next token
+                    else:
+                        lookahead = lookaheadstack.pop()
+                    if not lookahead:
+                        lookahead = YaccSymbol()
+                        lookahead.type = '$end'
 
-            # Check the action table
-            s = statestack[-1]
-            ltype = lookahead.type
-            t = actions.get((s,ltype),None)
+                # Check the action table
+                ltype = lookahead.type
+                t = actions[state].get(ltype)
+            else:
+                t = defaulted_states[state]
+                #--! DEBUG
+                debug.debug('Defaulted state %s: Reduce using %d', state, -t)
+                #--! DEBUG
 
-            if debug > 1:
-                print('action', t)
+            #--! DEBUG
+            debug.debug('Stack  : %s',
+                        ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip())
+            #--! DEBUG
+
             if t is not None:
                 if t > 0:
                     # shift a symbol on the stack
-                    if ltype == '$end':
-                        # Error, end of input
-                        sys.stderr.write("yacc: Parse error. EOF\n")
-                        return
                     statestack.append(t)
-                    if debug > 1:
-                        sys.stderr.write("%-60s shift state %s\n" % (errorlead, t))
+                    state = t
+
+                    #--! DEBUG
+                    debug.debug('Action : Shift and goto state %s', t)
+                    #--! DEBUG
+
                     symstack.append(lookahead)
                     lookahead = None
 
                     # Decrease error count on successful shift
-                    if self.errorcount > 0:
-                        self.errorcount -= 1
-                        
+                    if errorcount:
+                        errorcount -= 1
                     continue
-                
+
                 if t < 0:
                     # reduce a symbol on the stack, emit a production
                     p = prod[-t]
@@ -275,48 +463,123 @@
                     sym = YaccSymbol()
                     sym.type = pname       # Production name
                     sym.value = None
-                    if debug > 1:
-                        sys.stderr.write("%-60s reduce %d\n" % (errorlead, -t))
+
+                    #--! DEBUG
+                    if plen:
+                        debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str,
+                                   '['+','.join([format_stack_entry(_v.value) for _v in symstack[-plen:]])+']',
+                                   goto[statestack[-1-plen]][pname])
+                    else:
+                        debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str, [],
+                                   goto[statestack[-1]][pname])
+
+                    #--! DEBUG
 
                     if plen:
                         targ = symstack[-plen-1:]
                         targ[0] = sym
+
+                        #--! TRACKING
+                        if tracking:
+                            t1 = targ[1]
+                            sym.lineno = t1.lineno
+                            sym.lexpos = t1.lexpos
+                            t1 = targ[-1]
+                            sym.endlineno = getattr(t1, 'endlineno', t1.lineno)
+                            sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos)
+                        #--! TRACKING
+
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                        # The code enclosed in this section is duplicated
+                        # below as a performance optimization.  Make sure
+                        # changes get made in both locations.
+
+                        pslice.slice = targ
+
                         try:
-                            sym.lineno = targ[1].lineno
-                            sym.endlineno = getattr(targ[-1],"endlineno",targ[-1].lineno)
-                            sym.lexpos = targ[1].lexpos
-                            sym.endlexpos = getattr(targ[-1],"endlexpos",targ[-1].lexpos)
-                        except AttributeError:
-                            sym.lineno = 0
-                        del symstack[-plen:]
-                        del statestack[-plen:]
+                            # Call the grammar rule with our special slice object
+                            del symstack[-plen:]
+                            self.state = state
+                            p.callable(pslice)
+                            del statestack[-plen:]
+                            #--! DEBUG
+                            debug.info('Result : %s', format_result(pslice[0]))
+                            #--! DEBUG
+                            symstack.append(sym)
+                            state = goto[statestack[-1]][pname]
+                            statestack.append(state)
+                        except SyntaxError:
+                            # If an error was set. Enter error recovery state
+                            lookaheadstack.append(lookahead)    # Save the current lookahead token
+                            symstack.extend(targ[1:-1])         # Put the production slice back on the stack
+                            statestack.pop()                    # Pop back one state (before the reduce)
+                            state = statestack[-1]
+                            sym.type = 'error'
+                            sym.value = 'error'
+                            lookahead = sym
+                            errorcount = error_count
+                            self.errorok = False
+
+                        continue
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
                     else:
-                        sym.lineno = 0
-                        targ = [ sym ]
-                    pslice.slice = targ
-                    pslice.pbstack = []
-                    # Call the grammar rule with our special slice object
-                    p.func(pslice)
 
-                    # If there was a pushback, put that on the stack
-                    if pslice.pbstack:
-                        lookaheadstack.append(lookahead)
-                        for _t in pslice.pbstack:
-                            lookaheadstack.append(_t)
-                        lookahead = None
+                        #--! TRACKING
+                        if tracking:
+                            sym.lineno = lexer.lineno
+                            sym.lexpos = lexer.lexpos
+                        #--! TRACKING
 
-                    symstack.append(sym)
-                    statestack.append(goto[statestack[-1],pname])
-                    continue
+                        targ = [sym]
+
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                        # The code enclosed in this section is duplicated
+                        # above as a performance optimization.  Make sure
+                        # changes get made in both locations.
+
+                        pslice.slice = targ
+
+                        try:
+                            # Call the grammar rule with our special slice object
+                            self.state = state
+                            p.callable(pslice)
+                            #--! DEBUG
+                            debug.info('Result : %s', format_result(pslice[0]))
+                            #--! DEBUG
+                            symstack.append(sym)
+                            state = goto[statestack[-1]][pname]
+                            statestack.append(state)
+                        except SyntaxError:
+                            # If an error was set. Enter error recovery state
+                            lookaheadstack.append(lookahead)    # Save the current lookahead token
+                            statestack.pop()                    # Pop back one state (before the reduce)
+                            state = statestack[-1]
+                            sym.type = 'error'
+                            sym.value = 'error'
+                            lookahead = sym
+                            errorcount = error_count
+                            self.errorok = False
+
+                        continue
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
                 if t == 0:
                     n = symstack[-1]
-                    return getattr(n,"value",None)
-                    sys.stderr.write(errorlead, "\n")
+                    result = getattr(n, 'value', None)
+                    #--! DEBUG
+                    debug.info('Done   : Returning %s', format_result(result))
+                    debug.info('PLY: PARSE DEBUG END')
+                    #--! DEBUG
+                    return result
 
-            if t == None:
-                if debug:
-                    sys.stderr.write(errorlead + "\n")
+            if t is None:
+
+                #--! DEBUG
+                debug.error('Error  : %s',
+                            ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip())
+                #--! DEBUG
+
                 # We have some kind of parsing error here.  To handle
                 # this, we are going to push the current token onto
                 # the tokenstack and replace it with an 'error' token.
@@ -327,20 +590,18 @@
                 # the user defined p_error() function if this is the
                 # first syntax error.  This function is only called if
                 # errorcount == 0.
-                if not self.errorcount:
-                    self.errorcount = error_count
+                if errorcount == 0 or self.errorok:
+                    errorcount = error_count
+                    self.errorok = False
                     errtoken = lookahead
                     if errtoken.type == '$end':
                         errtoken = None               # End of file!
                     if self.errorfunc:
-                        global errok,token,restart
-                        errok = self.errok        # Set some special functions available in error recovery
-                        token = get_token
-                        restart = self.restart
-                        tok = self.errorfunc(errtoken)
-                        del errok, token, restart   # Delete special functions
-                        
-                        if not self.errorcount:
+                        if errtoken and not hasattr(errtoken, 'lexer'):
+                            errtoken.lexer = lexer
+                        self.state = state
+                        tok = call_errorfunc(self.errorfunc, errtoken, self)
+                        if self.errorok:
                             # User must have done some kind of panic
                             # mode recovery on their own.  The
                             # returned token is the next lookahead
@@ -349,19 +610,21 @@
                             continue
                     else:
                         if errtoken:
-                            if hasattr(errtoken,"lineno"): lineno = lookahead.lineno
-                            else: lineno = 0
-                            if lineno:
-                                sys.stderr.write("yacc: Syntax error at line %d, token=%s\n" % (lineno, errtoken.type))
+                            if hasattr(errtoken, 'lineno'):
+                                lineno = lookahead.lineno
                             else:
-                                sys.stderr.write("yacc: Syntax error, token=%s" % errtoken.type)
+                                lineno = 0
+                            if lineno:
+                                sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type))
+                            else:
+                                sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type)
                         else:
-                            sys.stderr.write("yacc: Parse error in input. EOF\n")
+                            sys.stderr.write('yacc: Parse error in input. EOF\n')
                             return
 
                 else:
-                    self.errorcount = error_count
-                
+                    errorcount = error_count
+
                 # case 1:  the statestack only has 1 entry on it.  If we're in this state, the
                 # entire parse has been rolled back and we're completely hosed.   The token is
                 # discarded and we just keep going.
@@ -369,6 +632,7 @@
                 if len(statestack) <= 1 and lookahead.type != '$end':
                     lookahead = None
                     errtoken = None
+                    state = 0
                     # Nuke the pushback stack
                     del lookaheadstack[:]
                     continue
@@ -379,7 +643,605 @@
                 # Start nuking entries on the stack
                 if lookahead.type == '$end':
                     # Whoa. We're really hosed here. Bail out
-                    return 
+                    return
+
+                if lookahead.type != 'error':
+                    sym = symstack[-1]
+                    if sym.type == 'error':
+                        # Hmmm. Error is on top of stack, we'll just nuke input
+                        # symbol and continue
+                        #--! TRACKING
+                        if tracking:
+                            sym.endlineno = getattr(lookahead, 'lineno', sym.lineno)
+                            sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos)
+                        #--! TRACKING
+                        lookahead = None
+                        continue
+
+                    # Create the error symbol for the first time and make it the new lookahead symbol
+                    t = YaccSymbol()
+                    t.type = 'error'
+
+                    if hasattr(lookahead, 'lineno'):
+                        t.lineno = t.endlineno = lookahead.lineno
+                    if hasattr(lookahead, 'lexpos'):
+                        t.lexpos = t.endlexpos = lookahead.lexpos
+                    t.value = lookahead
+                    lookaheadstack.append(lookahead)
+                    lookahead = t
+                else:
+                    sym = symstack.pop()
+                    #--! TRACKING
+                    if tracking:
+                        lookahead.lineno = sym.lineno
+                        lookahead.lexpos = sym.lexpos
+                    #--! TRACKING
+                    statestack.pop()
+                    state = statestack[-1]
+
+                continue
+
+            # Call an error function here
+            raise RuntimeError('yacc: internal parser error!!!\n')
+
+        #--! parsedebug-end
+
+    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    # parseopt().
+    #
+    # Optimized version of parse() method.  DO NOT EDIT THIS CODE DIRECTLY!
+    # This code is automatically generated by the ply/ygen.py script. Make
+    # changes to the parsedebug() method instead.
+    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+    def parseopt(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):
+        #--! parseopt-start
+        lookahead = None                         # Current lookahead symbol
+        lookaheadstack = []                      # Stack of lookahead symbols
+        actions = self.action                    # Local reference to action table (to avoid lookup on self.)
+        goto    = self.goto                      # Local reference to goto table (to avoid lookup on self.)
+        prod    = self.productions               # Local reference to production list (to avoid lookup on self.)
+        defaulted_states = self.defaulted_states # Local reference to defaulted states
+        pslice  = YaccProduction(None)           # Production object passed to grammar rules
+        errorcount = 0                           # Used during error recovery
+
+
+        # If no lexer was given, we will try to use the lex module
+        if not lexer:
+            from . import lex
+            lexer = lex.lexer
+
+        # Set up the lexer and parser objects on pslice
+        pslice.lexer = lexer
+        pslice.parser = self
+
+        # If input was supplied, pass to lexer
+        if input is not None:
+            lexer.input(input)
+
+        if tokenfunc is None:
+            # Tokenize function
+            get_token = lexer.token
+        else:
+            get_token = tokenfunc
+
+        # Set the parser() token method (sometimes used in error recovery)
+        self.token = get_token
+
+        # Set up the state and symbol stacks
+
+        statestack = []                # Stack of parsing states
+        self.statestack = statestack
+        symstack   = []                # Stack of grammar symbols
+        self.symstack = symstack
+
+        pslice.stack = symstack         # Put in the production
+        errtoken   = None               # Err token
+
+        # The start state is assumed to be (0,$end)
+
+        statestack.append(0)
+        sym = YaccSymbol()
+        sym.type = '$end'
+        symstack.append(sym)
+        state = 0
+        while True:
+            # Get the next symbol on the input.  If a lookahead symbol
+            # is already set, we just use that. Otherwise, we'll pull
+            # the next token off of the lookaheadstack or from the lexer
+
+
+            if state not in defaulted_states:
+                if not lookahead:
+                    if not lookaheadstack:
+                        lookahead = get_token()     # Get the next token
+                    else:
+                        lookahead = lookaheadstack.pop()
+                    if not lookahead:
+                        lookahead = YaccSymbol()
+                        lookahead.type = '$end'
+
+                # Check the action table
+                ltype = lookahead.type
+                t = actions[state].get(ltype)
+            else:
+                t = defaulted_states[state]
+
+
+            if t is not None:
+                if t > 0:
+                    # shift a symbol on the stack
+                    statestack.append(t)
+                    state = t
+
+
+                    symstack.append(lookahead)
+                    lookahead = None
+
+                    # Decrease error count on successful shift
+                    if errorcount:
+                        errorcount -= 1
+                    continue
+
+                if t < 0:
+                    # reduce a symbol on the stack, emit a production
+                    p = prod[-t]
+                    pname = p.name
+                    plen  = p.len
+
+                    # Get production function
+                    sym = YaccSymbol()
+                    sym.type = pname       # Production name
+                    sym.value = None
+
+
+                    if plen:
+                        targ = symstack[-plen-1:]
+                        targ[0] = sym
+
+                        #--! TRACKING
+                        if tracking:
+                            t1 = targ[1]
+                            sym.lineno = t1.lineno
+                            sym.lexpos = t1.lexpos
+                            t1 = targ[-1]
+                            sym.endlineno = getattr(t1, 'endlineno', t1.lineno)
+                            sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos)
+                        #--! TRACKING
+
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                        # The code enclosed in this section is duplicated
+                        # below as a performance optimization.  Make sure
+                        # changes get made in both locations.
+
+                        pslice.slice = targ
+
+                        try:
+                            # Call the grammar rule with our special slice object
+                            del symstack[-plen:]
+                            self.state = state
+                            p.callable(pslice)
+                            del statestack[-plen:]
+                            symstack.append(sym)
+                            state = goto[statestack[-1]][pname]
+                            statestack.append(state)
+                        except SyntaxError:
+                            # If an error was set. Enter error recovery state
+                            lookaheadstack.append(lookahead)    # Save the current lookahead token
+                            symstack.extend(targ[1:-1])         # Put the production slice back on the stack
+                            statestack.pop()                    # Pop back one state (before the reduce)
+                            state = statestack[-1]
+                            sym.type = 'error'
+                            sym.value = 'error'
+                            lookahead = sym
+                            errorcount = error_count
+                            self.errorok = False
+
+                        continue
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                    else:
+
+                        #--! TRACKING
+                        if tracking:
+                            sym.lineno = lexer.lineno
+                            sym.lexpos = lexer.lexpos
+                        #--! TRACKING
+
+                        targ = [sym]
+
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                        # The code enclosed in this section is duplicated
+                        # above as a performance optimization.  Make sure
+                        # changes get made in both locations.
+
+                        pslice.slice = targ
+
+                        try:
+                            # Call the grammar rule with our special slice object
+                            self.state = state
+                            p.callable(pslice)
+                            symstack.append(sym)
+                            state = goto[statestack[-1]][pname]
+                            statestack.append(state)
+                        except SyntaxError:
+                            # If an error was set. Enter error recovery state
+                            lookaheadstack.append(lookahead)    # Save the current lookahead token
+                            statestack.pop()                    # Pop back one state (before the reduce)
+                            state = statestack[-1]
+                            sym.type = 'error'
+                            sym.value = 'error'
+                            lookahead = sym
+                            errorcount = error_count
+                            self.errorok = False
+
+                        continue
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                if t == 0:
+                    n = symstack[-1]
+                    result = getattr(n, 'value', None)
+                    return result
+
+            if t is None:
+
+
+                # We have some kind of parsing error here.  To handle
+                # this, we are going to push the current token onto
+                # the tokenstack and replace it with an 'error' token.
+                # If there are any synchronization rules, they may
+                # catch it.
+                #
+                # In addition to pushing the error token, we call call
+                # the user defined p_error() function if this is the
+                # first syntax error.  This function is only called if
+                # errorcount == 0.
+                if errorcount == 0 or self.errorok:
+                    errorcount = error_count
+                    self.errorok = False
+                    errtoken = lookahead
+                    if errtoken.type == '$end':
+                        errtoken = None               # End of file!
+                    if self.errorfunc:
+                        if errtoken and not hasattr(errtoken, 'lexer'):
+                            errtoken.lexer = lexer
+                        self.state = state
+                        tok = call_errorfunc(self.errorfunc, errtoken, self)
+                        if self.errorok:
+                            # User must have done some kind of panic
+                            # mode recovery on their own.  The
+                            # returned token is the next lookahead
+                            lookahead = tok
+                            errtoken = None
+                            continue
+                    else:
+                        if errtoken:
+                            if hasattr(errtoken, 'lineno'):
+                                lineno = lookahead.lineno
+                            else:
+                                lineno = 0
+                            if lineno:
+                                sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type))
+                            else:
+                                sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type)
+                        else:
+                            sys.stderr.write('yacc: Parse error in input. EOF\n')
+                            return
+
+                else:
+                    errorcount = error_count
+
+                # case 1:  the statestack only has 1 entry on it.  If we're in this state, the
+                # entire parse has been rolled back and we're completely hosed.   The token is
+                # discarded and we just keep going.
+
+                if len(statestack) <= 1 and lookahead.type != '$end':
+                    lookahead = None
+                    errtoken = None
+                    state = 0
+                    # Nuke the pushback stack
+                    del lookaheadstack[:]
+                    continue
+
+                # case 2: the statestack has a couple of entries on it, but we're
+                # at the end of the file. nuke the top entry and generate an error token
+
+                # Start nuking entries on the stack
+                if lookahead.type == '$end':
+                    # Whoa. We're really hosed here. Bail out
+                    return
+
+                if lookahead.type != 'error':
+                    sym = symstack[-1]
+                    if sym.type == 'error':
+                        # Hmmm. Error is on top of stack, we'll just nuke input
+                        # symbol and continue
+                        #--! TRACKING
+                        if tracking:
+                            sym.endlineno = getattr(lookahead, 'lineno', sym.lineno)
+                            sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos)
+                        #--! TRACKING
+                        lookahead = None
+                        continue
+
+                    # Create the error symbol for the first time and make it the new lookahead symbol
+                    t = YaccSymbol()
+                    t.type = 'error'
+
+                    if hasattr(lookahead, 'lineno'):
+                        t.lineno = t.endlineno = lookahead.lineno
+                    if hasattr(lookahead, 'lexpos'):
+                        t.lexpos = t.endlexpos = lookahead.lexpos
+                    t.value = lookahead
+                    lookaheadstack.append(lookahead)
+                    lookahead = t
+                else:
+                    sym = symstack.pop()
+                    #--! TRACKING
+                    if tracking:
+                        lookahead.lineno = sym.lineno
+                        lookahead.lexpos = sym.lexpos
+                    #--! TRACKING
+                    statestack.pop()
+                    state = statestack[-1]
+
+                continue
+
+            # Call an error function here
+            raise RuntimeError('yacc: internal parser error!!!\n')
+
+        #--! parseopt-end
+
+    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    # parseopt_notrack().
+    #
+    # Optimized version of parseopt() with line number tracking removed.
+    # DO NOT EDIT THIS CODE DIRECTLY. This code is automatically generated
+    # by the ply/ygen.py script. Make changes to the parsedebug() method instead.
+    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+    def parseopt_notrack(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None):
+        #--! parseopt-notrack-start
+        lookahead = None                         # Current lookahead symbol
+        lookaheadstack = []                      # Stack of lookahead symbols
+        actions = self.action                    # Local reference to action table (to avoid lookup on self.)
+        goto    = self.goto                      # Local reference to goto table (to avoid lookup on self.)
+        prod    = self.productions               # Local reference to production list (to avoid lookup on self.)
+        defaulted_states = self.defaulted_states # Local reference to defaulted states
+        pslice  = YaccProduction(None)           # Production object passed to grammar rules
+        errorcount = 0                           # Used during error recovery
+
+
+        # If no lexer was given, we will try to use the lex module
+        if not lexer:
+            from . import lex
+            lexer = lex.lexer
+
+        # Set up the lexer and parser objects on pslice
+        pslice.lexer = lexer
+        pslice.parser = self
+
+        # If input was supplied, pass to lexer
+        if input is not None:
+            lexer.input(input)
+
+        if tokenfunc is None:
+            # Tokenize function
+            get_token = lexer.token
+        else:
+            get_token = tokenfunc
+
+        # Set the parser() token method (sometimes used in error recovery)
+        self.token = get_token
+
+        # Set up the state and symbol stacks
+
+        statestack = []                # Stack of parsing states
+        self.statestack = statestack
+        symstack   = []                # Stack of grammar symbols
+        self.symstack = symstack
+
+        pslice.stack = symstack         # Put in the production
+        errtoken   = None               # Err token
+
+        # The start state is assumed to be (0,$end)
+
+        statestack.append(0)
+        sym = YaccSymbol()
+        sym.type = '$end'
+        symstack.append(sym)
+        state = 0
+        while True:
+            # Get the next symbol on the input.  If a lookahead symbol
+            # is already set, we just use that. Otherwise, we'll pull
+            # the next token off of the lookaheadstack or from the lexer
+
+
+            if state not in defaulted_states:
+                if not lookahead:
+                    if not lookaheadstack:
+                        lookahead = get_token()     # Get the next token
+                    else:
+                        lookahead = lookaheadstack.pop()
+                    if not lookahead:
+                        lookahead = YaccSymbol()
+                        lookahead.type = '$end'
+
+                # Check the action table
+                ltype = lookahead.type
+                t = actions[state].get(ltype)
+            else:
+                t = defaulted_states[state]
+
+
+            if t is not None:
+                if t > 0:
+                    # shift a symbol on the stack
+                    statestack.append(t)
+                    state = t
+
+
+                    symstack.append(lookahead)
+                    lookahead = None
+
+                    # Decrease error count on successful shift
+                    if errorcount:
+                        errorcount -= 1
+                    continue
+
+                if t < 0:
+                    # reduce a symbol on the stack, emit a production
+                    p = prod[-t]
+                    pname = p.name
+                    plen  = p.len
+
+                    # Get production function
+                    sym = YaccSymbol()
+                    sym.type = pname       # Production name
+                    sym.value = None
+
+
+                    if plen:
+                        targ = symstack[-plen-1:]
+                        targ[0] = sym
+
+
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                        # The code enclosed in this section is duplicated
+                        # below as a performance optimization.  Make sure
+                        # changes get made in both locations.
+
+                        pslice.slice = targ
+
+                        try:
+                            # Call the grammar rule with our special slice object
+                            del symstack[-plen:]
+                            self.state = state
+                            p.callable(pslice)
+                            del statestack[-plen:]
+                            symstack.append(sym)
+                            state = goto[statestack[-1]][pname]
+                            statestack.append(state)
+                        except SyntaxError:
+                            # If an error was set. Enter error recovery state
+                            lookaheadstack.append(lookahead)    # Save the current lookahead token
+                            symstack.extend(targ[1:-1])         # Put the production slice back on the stack
+                            statestack.pop()                    # Pop back one state (before the reduce)
+                            state = statestack[-1]
+                            sym.type = 'error'
+                            sym.value = 'error'
+                            lookahead = sym
+                            errorcount = error_count
+                            self.errorok = False
+
+                        continue
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                    else:
+
+
+                        targ = [sym]
+
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+                        # The code enclosed in this section is duplicated
+                        # above as a performance optimization.  Make sure
+                        # changes get made in both locations.
+
+                        pslice.slice = targ
+
+                        try:
+                            # Call the grammar rule with our special slice object
+                            self.state = state
+                            p.callable(pslice)
+                            symstack.append(sym)
+                            state = goto[statestack[-1]][pname]
+                            statestack.append(state)
+                        except SyntaxError:
+                            # If an error was set. Enter error recovery state
+                            lookaheadstack.append(lookahead)    # Save the current lookahead token
+                            statestack.pop()                    # Pop back one state (before the reduce)
+                            state = statestack[-1]
+                            sym.type = 'error'
+                            sym.value = 'error'
+                            lookahead = sym
+                            errorcount = error_count
+                            self.errorok = False
+
+                        continue
+                        # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+                if t == 0:
+                    n = symstack[-1]
+                    result = getattr(n, 'value', None)
+                    return result
+
+            if t is None:
+
+
+                # We have some kind of parsing error here.  To handle
+                # this, we are going to push the current token onto
+                # the tokenstack and replace it with an 'error' token.
+                # If there are any synchronization rules, they may
+                # catch it.
+                #
+                # In addition to pushing the error token, we call call
+                # the user defined p_error() function if this is the
+                # first syntax error.  This function is only called if
+                # errorcount == 0.
+                if errorcount == 0 or self.errorok:
+                    errorcount = error_count
+                    self.errorok = False
+                    errtoken = lookahead
+                    if errtoken.type == '$end':
+                        errtoken = None               # End of file!
+                    if self.errorfunc:
+                        if errtoken and not hasattr(errtoken, 'lexer'):
+                            errtoken.lexer = lexer
+                        self.state = state
+                        tok = call_errorfunc(self.errorfunc, errtoken, self)
+                        if self.errorok:
+                            # User must have done some kind of panic
+                            # mode recovery on their own.  The
+                            # returned token is the next lookahead
+                            lookahead = tok
+                            errtoken = None
+                            continue
+                    else:
+                        if errtoken:
+                            if hasattr(errtoken, 'lineno'):
+                                lineno = lookahead.lineno
+                            else:
+                                lineno = 0
+                            if lineno:
+                                sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type))
+                            else:
+                                sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type)
+                        else:
+                            sys.stderr.write('yacc: Parse error in input. EOF\n')
+                            return
+
+                else:
+                    errorcount = error_count
+
+                # case 1:  the statestack only has 1 entry on it.  If we're in this state, the
+                # entire parse has been rolled back and we're completely hosed.   The token is
+                # discarded and we just keep going.
+
+                if len(statestack) <= 1 and lookahead.type != '$end':
+                    lookahead = None
+                    errtoken = None
+                    state = 0
+                    # Nuke the pushback stack
+                    del lookaheadstack[:]
+                    continue
+
+                # case 2: the statestack has a couple of entries on it, but we're
+                # at the end of the file. nuke the top entry and generate an error token
+
+                # Start nuking entries on the stack
+                if lookahead.type == '$end':
+                    # Whoa. We're really hosed here. Bail out
+                    return
 
                 if lookahead.type != 'error':
                     sym = symstack[-1]
@@ -388,1096 +1250,793 @@
                         # symbol and continue
                         lookahead = None
                         continue
+
+                    # Create the error symbol for the first time and make it the new lookahead symbol
                     t = YaccSymbol()
                     t.type = 'error'
-                    if hasattr(lookahead,"lineno"):
-                        t.lineno = lookahead.lineno
+
+                    if hasattr(lookahead, 'lineno'):
+                        t.lineno = t.endlineno = lookahead.lineno
+                    if hasattr(lookahead, 'lexpos'):
+                        t.lexpos = t.endlexpos = lookahead.lexpos
                     t.value = lookahead
                     lookaheadstack.append(lookahead)
                     lookahead = t
                 else:
-                    symstack.pop()
+                    sym = symstack.pop()
                     statestack.pop()
+                    state = statestack[-1]
 
                 continue
 
             # Call an error function here
-            raise RuntimeError("yacc: internal parser error!!!\n")
+            raise RuntimeError('yacc: internal parser error!!!\n')
+
+        #--! parseopt-notrack-end
 
 # -----------------------------------------------------------------------------
-#                          === Parser Construction ===
+#                          === Grammar Representation ===
 #
-# The following functions and variables are used to implement the yacc() function
-# itself.   This is pretty hairy stuff involving lots of error checking,
-# construction of LR items, kernels, and so forth.   Although a lot of
-# this work is done using global variables, the resulting Parser object
-# is completely self contained--meaning that it is safe to repeatedly
-# call yacc() with different grammars in the same application.
-# -----------------------------------------------------------------------------
-        
-# -----------------------------------------------------------------------------
-# validate_file()
-#
-# This function checks to see if there are duplicated p_rulename() functions
-# in the parser module file.  Without this function, it is really easy for
-# users to make mistakes by cutting and pasting code fragments (and it's a real
-# bugger to try and figure out why the resulting parser doesn't work).  Therefore,
-# we just do a little regular expression pattern matching of def statements
-# to try and detect duplicates.
+# The following functions, classes, and variables are used to represent and
+# manipulate the rules that make up a grammar.
 # -----------------------------------------------------------------------------
 
-def validate_file(filename):
-    base,ext = os.path.splitext(filename)
-    if ext != '.py': return 1          # No idea. Assume it's okay.
-
-    try:
-        f = open(filename)
-        lines = f.readlines()
-        f.close()
-    except IOError:
-        return 1                       # Oh well
-
-    # Match def p_funcname(
-    fre = re.compile(r'\s*def\s+(p_[a-zA-Z_0-9]*)\(')
-    counthash = { }
-    linen = 1
-    noerror = 1
-    for l in lines:
-        m = fre.match(l)
-        if m:
-            name = m.group(1)
-            prev = counthash.get(name)
-            if not prev:
-                counthash[name] = linen
-            else:
-                sys.stderr.write("%s:%d: Function %s redefined. Previously defined on line %d\n" % (filename,linen,name,prev))
-                noerror = 0
-        linen += 1
-    return noerror
-
-# This function looks for functions that might be grammar rules, but which don't have the proper p_suffix.
-def validate_dict(d):
-    for n,v in d.items(): 
-        if n[0:2] == 'p_' and type(v) in (types.FunctionType, types.MethodType): continue
-        if n[0:2] == 't_': continue
-
-        if n[0:2] == 'p_':
-            sys.stderr.write("yacc: Warning. '%s' not defined as a function\n" % n)
-        if 1 and isinstance(v,types.FunctionType) and v.__code__.co_argcount == 1:
-            try:
-                doc = v.__doc__.split(" ")
-                if doc[1] == ':':
-                    sys.stderr.write("%s:%d: Warning. Possible grammar rule '%s' defined without p_ prefix.\n" % (v.__code__.co_filename, v.__code__.co_firstlineno,n))
-            except Exception:
-                pass
-
-# -----------------------------------------------------------------------------
-#                           === GRAMMAR FUNCTIONS ===
-#
-# The following global variables and functions are used to store, manipulate,
-# and verify the grammar rules specified by the user.
-# -----------------------------------------------------------------------------
-
-# Initialize all of the global variables used during grammar construction
-def initialize_vars():
-    global Productions, Prodnames, Prodmap, Terminals 
-    global Nonterminals, First, Follow, Precedence, LRitems
-    global Errorfunc, Signature, Requires
-
-    Productions  = [None]  # A list of all of the productions.  The first
-                           # entry is always reserved for the purpose of
-                           # building an augmented grammar
-                        
-    Prodnames    = { }     # A dictionary mapping the names of nonterminals to a list of all
-                           # productions of that nonterminal.
-                        
-    Prodmap      = { }     # A dictionary that is only used to detect duplicate
-                           # productions.
-
-    Terminals    = { }     # A dictionary mapping the names of terminal symbols to a
-                           # list of the rules where they are used.
-
-    Nonterminals = { }     # A dictionary mapping names of nonterminals to a list
-                           # of rule numbers where they are used.
-
-    First        = { }     # A dictionary of precomputed FIRST(x) symbols
-    
-    Follow       = { }     # A dictionary of precomputed FOLLOW(x) symbols
-
-    Precedence   = { }     # Precedence rules for each terminal. Contains tuples of the
-                           # form ('right',level) or ('nonassoc', level) or ('left',level)
-
-    LRitems      = [ ]     # A list of all LR items for the grammar.  These are the
-                           # productions with the "dot" like E -> E . PLUS E
-
-    Errorfunc    = None    # User defined error handler
-
-    Signature    = hashlib.sha256()   # Digital signature of the grammar rules, precedence
-                               # and other information.  Used to determined when a
-                               # parsing table needs to be regenerated.
-
-    Requires     = { }     # Requires list
-
-    # File objects used when creating the parser.out debugging file
-    global _vf, _vfc
-    _vf           = StringIO()
-    _vfc          = StringIO()
+# regex matching identifiers
+_is_identifier = re.compile(r'^[a-zA-Z0-9_-]+$')
 
 # -----------------------------------------------------------------------------
 # class Production:
 #
 # This class stores the raw information about a single production or grammar rule.
-# It has a few required attributes:
+# A grammar rule refers to a specification such as this:
 #
-#       name     - Name of the production (nonterminal)
-#       prod     - A list of symbols making up its production
+#       expr : expr PLUS term
+#
+# Here are the basic attributes defined on all productions
+#
+#       name     - Name of the production.  For example 'expr'
+#       prod     - A list of symbols on the right side ['expr','PLUS','term']
+#       prec     - Production precedence level
 #       number   - Production number.
+#       func     - Function that executes on reduce
+#       file     - File where production function is defined
+#       lineno   - Line number where production function is defined
 #
-# In addition, a few additional attributes are used to help with debugging or
-# optimization of table generation.
+# The following attributes are defined or optional.
 #
-#       file     - File where production action is defined.
-#       lineno   - Line number where action is defined
-#       func     - Action function
-#       prec     - Precedence level
-#       lr_next  - Next LR item. Example, if we are ' E -> E . PLUS E'
-#                  then lr_next refers to 'E -> E PLUS . E'   
-#       lr_index - LR item index (location of the ".") in the prod list.
-#       lookaheads - LALR lookahead symbols for this item
-#       len      - Length of the production (number of symbols on right hand side)
+#       len       - Length of the production (number of symbols on right hand side)
+#       usyms     - Set of unique symbols found in the production
 # -----------------------------------------------------------------------------
 
-class Production:
-    def __init__(self,**kw):
-        for k,v in kw.items():
-            setattr(self,k,v)
-        self.lr_index = -1
-        self.lr0_added = 0    # Flag indicating whether or not added to LR0 closure
-        self.lr1_added = 0    # Flag indicating whether or not added to LR1
-        self.usyms = [ ]
-        self.lookaheads = { }
-        self.lk_added = { }
-        self.setnumbers = [ ]
-        
+class Production(object):
+    reduced = 0
+    def __init__(self, number, name, prod, precedence=('right', 0), func=None, file='', line=0):
+        self.name     = name
+        self.prod     = tuple(prod)
+        self.number   = number
+        self.func     = func
+        self.callable = None
+        self.file     = file
+        self.line     = line
+        self.prec     = precedence
+
+        # Internal settings used during table construction
+
+        self.len  = len(self.prod)   # Length of the production
+
+        # Create a list of unique production symbols used in the production
+        self.usyms = []
+        for s in self.prod:
+            if s not in self.usyms:
+                self.usyms.append(s)
+
+        # List of all LR items for the production
+        self.lr_items = []
+        self.lr_next = None
+
+        # Create a string representation
+        if self.prod:
+            self.str = '%s -> %s' % (self.name, ' '.join(self.prod))
+        else:
+            self.str = '%s -> <empty>' % self.name
+
+    def __str__(self):
+        return self.str
+
+    def __repr__(self):
+        return 'Production(' + str(self) + ')'
+
+    def __len__(self):
+        return len(self.prod)
+
+    def __nonzero__(self):
+        return 1
+
+    def __getitem__(self, index):
+        return self.prod[index]
+
+    # Return the nth lr_item from the production (or None if at the end)
+    def lr_item(self, n):
+        if n > len(self.prod):
+            return None
+        p = LRItem(self, n)
+        # Precompute the list of productions immediately following.
+        try:
+            p.lr_after = self.Prodnames[p.prod[n+1]]
+        except (IndexError, KeyError):
+            p.lr_after = []
+        try:
+            p.lr_before = p.prod[n-1]
+        except IndexError:
+            p.lr_before = None
+        return p
+
+    # Bind the production function name to a callable
+    def bind(self, pdict):
+        if self.func:
+            self.callable = pdict[self.func]
+
+# This class serves as a minimal standin for Production objects when
+# reading table data from files.   It only contains information
+# actually used by the LR parsing engine, plus some additional
+# debugging information.
+class MiniProduction(object):
+    def __init__(self, str, name, len, func, file, line):
+        self.name     = name
+        self.len      = len
+        self.func     = func
+        self.callable = None
+        self.file     = file
+        self.line     = line
+        self.str      = str
+
+    def __str__(self):
+        return self.str
+
+    def __repr__(self):
+        return 'MiniProduction(%s)' % self.str
+
+    # Bind the production function name to a callable
+    def bind(self, pdict):
+        if self.func:
+            self.callable = pdict[self.func]
+
+
+# -----------------------------------------------------------------------------
+# class LRItem
+#
+# This class represents a specific stage of parsing a production rule.  For
+# example:
+#
+#       expr : expr . PLUS term
+#
+# In the above, the "." represents the current location of the parse.  Here
+# basic attributes:
+#
+#       name       - Name of the production.  For example 'expr'
+#       prod       - A list of symbols on the right side ['expr','.', 'PLUS','term']
+#       number     - Production number.
+#
+#       lr_next      Next LR item. Example, if we are ' expr -> expr . PLUS term'
+#                    then lr_next refers to 'expr -> expr PLUS . term'
+#       lr_index   - LR item index (location of the ".") in the prod list.
+#       lookaheads - LALR lookahead symbols for this item
+#       len        - Length of the production (number of symbols on right hand side)
+#       lr_after    - List of all productions that immediately follow
+#       lr_before   - Grammar symbol immediately before
+# -----------------------------------------------------------------------------
+
+class LRItem(object):
+    def __init__(self, p, n):
+        self.name       = p.name
+        self.prod       = list(p.prod)
+        self.number     = p.number
+        self.lr_index   = n
+        self.lookaheads = {}
+        self.prod.insert(n, '.')
+        self.prod       = tuple(self.prod)
+        self.len        = len(self.prod)
+        self.usyms      = p.usyms
+
     def __str__(self):
         if self.prod:
-            s = "%s -> %s" % (self.name," ".join(self.prod))
+            s = '%s -> %s' % (self.name, ' '.join(self.prod))
         else:
-            s = "%s -> <empty>" % self.name
+            s = '%s -> <empty>' % self.name
         return s
 
     def __repr__(self):
-        return str(self)
+        return 'LRItem(' + str(self) + ')'
 
-    # Compute lr_items from the production
-    def lr_item(self,n):
-        if n > len(self.prod): return None
-        p = Production()
-        p.name = self.name
-        p.prod = list(self.prod)
-        p.number = self.number
-        p.lr_index = n
-        p.lookaheads = { }
-        p.setnumbers = self.setnumbers
-        p.prod.insert(n,".")
-        p.prod = tuple(p.prod)
-        p.len = len(p.prod)
-        p.usyms = self.usyms
+# -----------------------------------------------------------------------------
+# rightmost_terminal()
+#
+# Return the rightmost terminal from a list of symbols.  Used in add_production()
+# -----------------------------------------------------------------------------
+def rightmost_terminal(symbols, terminals):
+    i = len(symbols) - 1
+    while i >= 0:
+        if symbols[i] in terminals:
+            return symbols[i]
+        i -= 1
+    return None
 
-        # Precompute list of productions immediately following
-        try:
-            p.lrafter = Prodnames[p.prod[n+1]]
-        except (IndexError,KeyError) as e:
-            p.lrafter = []
-        try:
-            p.lrbefore = p.prod[n-1]
-        except IndexError:
-            p.lrbefore = None
+# -----------------------------------------------------------------------------
+#                           === GRAMMAR CLASS ===
+#
+# The following class represents the contents of the specified grammar along
+# with various computed properties such as first sets, follow sets, LR items, etc.
+# This data is used for critical parts of the table generation process later.
+# -----------------------------------------------------------------------------
 
-        return p
-
-class MiniProduction:
+class GrammarError(YaccError):
     pass
 
-# regex matching identifiers
-_is_identifier = re.compile(r'^[a-zA-Z0-9_-~]+$')
+class Grammar(object):
+    def __init__(self, terminals):
+        self.Productions  = [None]  # A list of all of the productions.  The first
+                                    # entry is always reserved for the purpose of
+                                    # building an augmented grammar
 
-# -----------------------------------------------------------------------------
-# add_production()
-#
-# Given an action function, this function assembles a production rule.
-# The production rule is assumed to be found in the function's docstring.
-# This rule has the general syntax:
-#
-#              name1 ::= production1
-#                     |  production2
-#                     |  production3
-#                    ...
-#                     |  productionn
-#              name2 ::= production1
-#                     |  production2
-#                    ... 
-# -----------------------------------------------------------------------------
+        self.Prodnames    = {}      # A dictionary mapping the names of nonterminals to a list of all
+                                    # productions of that nonterminal.
 
-def add_production(f,file,line,prodname,syms):
-    
-    if prodname in Terminals:
-        sys.stderr.write("%s:%d: Illegal rule name '%s'. Already defined as a token.\n" % (file,line,prodname))
-        return -1
-    if prodname == 'error':
-        sys.stderr.write("%s:%d: Illegal rule name '%s'. error is a reserved word.\n" % (file,line,prodname))
-        return -1
-                
-    if not _is_identifier.match(prodname):
-        sys.stderr.write("%s:%d: Illegal rule name '%s'\n" % (file,line,prodname))
-        return -1
+        self.Prodmap      = {}      # A dictionary that is only used to detect duplicate
+                                    # productions.
 
-    for x in range(len(syms)):
-        s = syms[x]
-        if s[0] in "'\"":
-             try:
-                 c = eval(s)
-                 if (len(c) > 1):
-                      sys.stderr.write("%s:%d: Literal token %s in rule '%s' may only be a single character\n" % (file,line,s, prodname)) 
-                      return -1
-                 if c not in Terminals:
-                      Terminals[c] = []
-                 syms[x] = c
-                 continue
-             except SyntaxError:
-                 pass
-        if not _is_identifier.match(s) and s != '%prec':
-            sys.stderr.write("%s:%d: Illegal name '%s' in rule '%s'\n" % (file,line,s, prodname))
-            return -1
+        self.Terminals    = {}      # A dictionary mapping the names of terminal symbols to a
+                                    # list of the rules where they are used.
 
-    # See if the rule is already in the rulemap
-    map = "%s -> %s" % (prodname,syms)
-    if map in Prodmap:
-        m = Prodmap[map]
-        sys.stderr.write("%s:%d: Duplicate rule %s.\n" % (file,line, m))
-        sys.stderr.write("%s:%d: Previous definition at %s:%d\n" % (file,line, m.file, m.line))
-        return -1
+        for term in terminals:
+            self.Terminals[term] = []
 
-    p = Production()
-    p.name = prodname
-    p.prod = syms
-    p.file = file
-    p.line = line
-    p.func = f
-    p.number = len(Productions)
+        self.Terminals['error'] = []
 
-            
-    Productions.append(p)
-    Prodmap[map] = p
-    if prodname not in Nonterminals:
-        Nonterminals[prodname] = [ ]
-    
-    # Add all terminals to Terminals
-    i = 0
-    while i < len(p.prod):
-        t = p.prod[i]
-        if t == '%prec':
-            try:
-                precname = p.prod[i+1]
-            except IndexError:
-                sys.stderr.write("%s:%d: Syntax error. Nothing follows %%prec.\n" % (p.file,p.line))
-                return -1
+        self.Nonterminals = {}      # A dictionary mapping names of nonterminals to a list
+                                    # of rule numbers where they are used.
 
-            prec = Precedence.get(precname,None)
-            if not prec:
-                sys.stderr.write("%s:%d: Nothing known about the precedence of '%s'\n" % (p.file,p.line,precname))
-                return -1
+        self.First        = {}      # A dictionary of precomputed FIRST(x) symbols
+
+        self.Follow       = {}      # A dictionary of precomputed FOLLOW(x) symbols
+
+        self.Precedence   = {}      # Precedence rules for each terminal. Contains tuples of the
+                                    # form ('right',level) or ('nonassoc', level) or ('left',level)
+
+        self.UsedPrecedence = set() # Precedence rules that were actually used by the grammer.
+                                    # This is only used to provide error checking and to generate
+                                    # a warning about unused precedence rules.
+
+        self.Start = None           # Starting symbol for the grammar
+
+
+    def __len__(self):
+        return len(self.Productions)
+
+    def __getitem__(self, index):
+        return self.Productions[index]
+
+    # -----------------------------------------------------------------------------
+    # set_precedence()
+    #
+    # Sets the precedence for a given terminal. assoc is the associativity such as
+    # 'left','right', or 'nonassoc'.  level is a numeric level.
+    #
+    # -----------------------------------------------------------------------------
+
+    def set_precedence(self, term, assoc, level):
+        assert self.Productions == [None], 'Must call set_precedence() before add_production()'
+        if term in self.Precedence:
+            raise GrammarError('Precedence already specified for terminal %r' % term)
+        if assoc not in ['left', 'right', 'nonassoc']:
+            raise GrammarError("Associativity must be one of 'left','right', or 'nonassoc'")
+        self.Precedence[term] = (assoc, level)
+
+    # -----------------------------------------------------------------------------
+    # add_production()
+    #
+    # Given an action function, this function assembles a production rule and
+    # computes its precedence level.
+    #
+    # The production rule is supplied as a list of symbols.   For example,
+    # a rule such as 'expr : expr PLUS term' has a production name of 'expr' and
+    # symbols ['expr','PLUS','term'].
+    #
+    # Precedence is determined by the precedence of the right-most non-terminal
+    # or the precedence of a terminal specified by %prec.
+    #
+    # A variety of error checks are performed to make sure production symbols
+    # are valid and that %prec is used correctly.
+    # -----------------------------------------------------------------------------
+
+    def add_production(self, prodname, syms, func=None, file='', line=0):
+
+        if prodname in self.Terminals:
+            raise GrammarError('%s:%d: Illegal rule name %r. Already defined as a token' % (file, line, prodname))
+        if prodname == 'error':
+            raise GrammarError('%s:%d: Illegal rule name %r. error is a reserved word' % (file, line, prodname))
+        if not _is_identifier.match(prodname):
+            raise GrammarError('%s:%d: Illegal rule name %r' % (file, line, prodname))
+
+        # Look for literal tokens
+        for n, s in enumerate(syms):
+            if s[0] in "'\"":
+                try:
+                    c = eval(s)
+                    if (len(c) > 1):
+                        raise GrammarError('%s:%d: Literal token %s in rule %r may only be a single character' %
+                                           (file, line, s, prodname))
+                    if c not in self.Terminals:
+                        self.Terminals[c] = []
+                    syms[n] = c
+                    continue
+                except SyntaxError:
+                    pass
+            if not _is_identifier.match(s) and s != '%prec':
+                raise GrammarError('%s:%d: Illegal name %r in rule %r' % (file, line, s, prodname))
+
+        # Determine the precedence level
+        if '%prec' in syms:
+            if syms[-1] == '%prec':
+                raise GrammarError('%s:%d: Syntax error. Nothing follows %%prec' % (file, line))
+            if syms[-2] != '%prec':
+                raise GrammarError('%s:%d: Syntax error. %%prec can only appear at the end of a grammar rule' %
+                                   (file, line))
+            precname = syms[-1]
+            prodprec = self.Precedence.get(precname)
+            if not prodprec:
+                raise GrammarError('%s:%d: Nothing known about the precedence of %r' % (file, line, precname))
             else:
-                p.prec = prec
-            del p.prod[i]
-            del p.prod[i]
-            continue
-
-        if t in Terminals:
-            Terminals[t].append(p.number)
-            # Is a terminal.  We'll assign a precedence to p based on this
-            if not hasattr(p,"prec"):
-                p.prec = Precedence.get(t,('right',0))
+                self.UsedPrecedence.add(precname)
+            del syms[-2:]     # Drop %prec from the rule
         else:
-            if t not in Nonterminals:
-                Nonterminals[t] = [ ]
-            Nonterminals[t].append(p.number)
-        i += 1
+            # If no %prec, precedence is determined by the rightmost terminal symbol
+            precname = rightmost_terminal(syms, self.Terminals)
+            prodprec = self.Precedence.get(precname, ('right', 0))
 
-    if not hasattr(p,"prec"):
-        p.prec = ('right',0)
-        
-    # Set final length of productions
-    p.len  = len(p.prod)
-    p.prod = tuple(p.prod)
+        # See if the rule is already in the rulemap
+        map = '%s -> %s' % (prodname, syms)
+        if map in self.Prodmap:
+            m = self.Prodmap[map]
+            raise GrammarError('%s:%d: Duplicate rule %s. ' % (file, line, m) +
+                               'Previous definition at %s:%d' % (m.file, m.line))
 
-    # Calculate unique syms in the production
-    p.usyms = [ ]
-    for s in p.prod:
-        if s not in p.usyms:
-            p.usyms.append(s)
-    
-    # Add to the global productions list
-    try:
-        Prodnames[p.name].append(p)
-    except KeyError:
-        Prodnames[p.name] = [ p ]
-    return 0
+        # From this point on, everything is valid.  Create a new Production instance
+        pnumber  = len(self.Productions)
+        if prodname not in self.Nonterminals:
+            self.Nonterminals[prodname] = []
 
-# Given a raw rule function, this function rips out its doc string
-# and adds rules to the grammar
+        # Add the production number to Terminals and Nonterminals
+        for t in syms:
+            if t in self.Terminals:
+                self.Terminals[t].append(pnumber)
+            else:
+                if t not in self.Nonterminals:
+                    self.Nonterminals[t] = []
+                self.Nonterminals[t].append(pnumber)
 
-def add_function(f):
-    line = f.__code__.co_firstlineno
-    file = f.__code__.co_filename
-    error = 0
+        # Create a production and add it to the list of productions
+        p = Production(pnumber, prodname, syms, prodprec, func, file, line)
+        self.Productions.append(p)
+        self.Prodmap[map] = p
 
-    if isinstance(f,types.MethodType):
-        reqdargs = 2
-    else:
-        reqdargs = 1
-        
-    if f.__code__.co_argcount > reqdargs:
-        sys.stderr.write("%s:%d: Rule '%s' has too many arguments.\n" % (file,line,f.__name__))
-        return -1
+        # Add to the global productions list
+        try:
+            self.Prodnames[prodname].append(p)
+        except KeyError:
+            self.Prodnames[prodname] = [p]
 
-    if f.__code__.co_argcount < reqdargs:
-        sys.stderr.write("%s:%d: Rule '%s' requires an argument.\n" % (file,line,f.__name__))
-        return -1
-          
-    if f.__doc__:
-        # Split the doc string into lines
-        pstrings = f.__doc__.splitlines()
-        lastp = None
-        dline = line
-        for ps in pstrings:
-            dline += 1
-            p = ps.split()
-            if not p: continue
-            try:
-                if p[0] == '|':
-                    # This is a continuation of a previous rule
-                    if not lastp:
-                        sys.stderr.write("%s:%d: Misplaced '|'.\n" % (file,dline))
-                        return -1
-                    prodname = lastp
-                    if len(p) > 1:
-                        syms = p[1:]
+    # -----------------------------------------------------------------------------
+    # set_start()
+    #
+    # Sets the starting symbol and creates the augmented grammar.  Production
+    # rule 0 is S' -> start where start is the start symbol.
+    # -----------------------------------------------------------------------------
+
+    def set_start(self, start=None):
+        if not start:
+            start = self.Productions[1].name
+        if start not in self.Nonterminals:
+            raise GrammarError('start symbol %s undefined' % start)
+        self.Productions[0] = Production(0, "S'", [start])
+        self.Nonterminals[start].append(0)
+        self.Start = start
+
+    # -----------------------------------------------------------------------------
+    # find_unreachable()
+    #
+    # Find all of the nonterminal symbols that can't be reached from the starting
+    # symbol.  Returns a list of nonterminals that can't be reached.
+    # -----------------------------------------------------------------------------
+
+    def find_unreachable(self):
+
+        # Mark all symbols that are reachable from a symbol s
+        def mark_reachable_from(s):
+            if s in reachable:
+                return
+            reachable.add(s)
+            for p in self.Prodnames.get(s, []):
+                for r in p.prod:
+                    mark_reachable_from(r)
+
+        reachable = set()
+        mark_reachable_from(self.Productions[0].prod[0])
+        return [s for s in self.Nonterminals if s not in reachable]
+
+    # -----------------------------------------------------------------------------
+    # infinite_cycles()
+    #
+    # This function looks at the various parsing rules and tries to detect
+    # infinite recursion cycles (grammar rules where there is no possible way
+    # to derive a string of only terminals).
+    # -----------------------------------------------------------------------------
+
+    def infinite_cycles(self):
+        terminates = {}
+
+        # Terminals:
+        for t in self.Terminals:
+            terminates[t] = True
+
+        terminates['$end'] = True
+
+        # Nonterminals:
+
+        # Initialize to false:
+        for n in self.Nonterminals:
+            terminates[n] = False
+
+        # Then propagate termination until no change:
+        while True:
+            some_change = False
+            for (n, pl) in self.Prodnames.items():
+                # Nonterminal n terminates iff any of its productions terminates.
+                for p in pl:
+                    # Production p terminates iff all of its rhs symbols terminate.
+                    for s in p.prod:
+                        if not terminates[s]:
+                            # The symbol s does not terminate,
+                            # so production p does not terminate.
+                            p_terminates = False
+                            break
                     else:
-                        syms = [ ]
-                else:
-                    prodname = p[0]
-                    lastp = prodname
-                    assign = p[1]
-                    if len(p) > 2:
-                        syms = p[2:]
-                    else:
-                        syms = [ ]
-                    if assign != ':' and assign != '::=':
-                        sys.stderr.write("%s:%d: Syntax error. Expected ':'\n" % (file,dline))
-                        return -1
-                         
- 
-                e = add_production(f,file,dline,prodname,syms)
-                error += e
+                        # didn't break from the loop,
+                        # so every symbol s terminates
+                        # so production p terminates.
+                        p_terminates = True
 
-                
-            except Exception:
-                sys.stderr.write("%s:%d: Syntax error in rule '%s'\n" % (file,dline,ps))
-                error -= 1
-    else:
-        sys.stderr.write("%s:%d: No documentation string specified in function '%s'\n" % (file,line,f.__name__))
-    return error
-
-
-# Cycle checking code (Michael Dyck)
-
-def compute_reachable():
-    '''
-    Find each symbol that can be reached from the start symbol.
-    Print a warning for any nonterminals that can't be reached.
-    (Unused terminals have already had their warning.)
-    '''
-    Reachable = { }
-    for s in list(Terminals.keys()) + list(Nonterminals.keys()):
-        Reachable[s] = 0
-
-    mark_reachable_from( Productions[0].prod[0], Reachable )
-
-    for s in Nonterminals.keys():
-        if not Reachable[s]:
-            sys.stderr.write("yacc: Symbol '%s' is unreachable.\n" % s)
-
-def mark_reachable_from(s, Reachable):
-    '''
-    Mark all symbols that are reachable from symbol s.
-    '''
-    if Reachable[s]:
-        # We've already reached symbol s.
-        return
-    Reachable[s] = 1
-    for p in Prodnames.get(s,[]):
-        for r in p.prod:
-            mark_reachable_from(r, Reachable)
-
-# -----------------------------------------------------------------------------
-# compute_terminates()
-#
-# This function looks at the various parsing rules and tries to detect
-# infinite recursion cycles (grammar rules where there is no possible way
-# to derive a string of only terminals).
-# -----------------------------------------------------------------------------
-def compute_terminates():
-    '''
-    Raise an error for any symbols that don't terminate.
-    '''
-    Terminates = {}
-
-    # Terminals:
-    for t in Terminals.keys():
-        Terminates[t] = 1
-
-    Terminates['$end'] = 1
-
-    # Nonterminals:
-
-    # Initialize to false:
-    for n in Nonterminals.keys():
-        Terminates[n] = 0
-
-    # Then propagate termination until no change:
-    while 1:
-        some_change = 0
-        for (n,pl) in Prodnames.items():
-            # Nonterminal n terminates iff any of its productions terminates.
-            for p in pl:
-                # Production p terminates iff all of its rhs symbols terminate.
-                for s in p.prod:
-                    if not Terminates[s]:
-                        # The symbol s does not terminate,
-                        # so production p does not terminate.
-                        p_terminates = 0
+                    if p_terminates:
+                        # symbol n terminates!
+                        if not terminates[n]:
+                            terminates[n] = True
+                            some_change = True
+                        # Don't need to consider any more productions for this n.
                         break
+
+            if not some_change:
+                break
+
+        infinite = []
+        for (s, term) in terminates.items():
+            if not term:
+                if s not in self.Prodnames and s not in self.Terminals and s != 'error':
+                    # s is used-but-not-defined, and we've already warned of that,
+                    # so it would be overkill to say that it's also non-terminating.
+                    pass
                 else:
-                    # didn't break from the loop,
-                    # so every symbol s terminates
-                    # so production p terminates.
-                    p_terminates = 1
+                    infinite.append(s)
 
-                if p_terminates:
-                    # symbol n terminates!
-                    if not Terminates[n]:
-                        Terminates[n] = 1
-                        some_change = 1
-                    # Don't need to consider any more productions for this n.
-                    break
+        return infinite
 
-        if not some_change:
-            break
+    # -----------------------------------------------------------------------------
+    # undefined_symbols()
+    #
+    # Find all symbols that were used the grammar, but not defined as tokens or
+    # grammar rules.  Returns a list of tuples (sym, prod) where sym in the symbol
+    # and prod is the production where the symbol was used.
+    # -----------------------------------------------------------------------------
+    def undefined_symbols(self):
+        result = []
+        for p in self.Productions:
+            if not p:
+                continue
 
-    some_error = 0
-    for (s,terminates) in Terminates.items():
-        if not terminates:
-            if s not in Prodnames and s not in Terminals and s != 'error':
-                # s is used-but-not-defined, and we've already warned of that,
-                # so it would be overkill to say that it's also non-terminating.
+            for s in p.prod:
+                if s not in self.Prodnames and s not in self.Terminals and s != 'error':
+                    result.append((s, p))
+        return result
+
+    # -----------------------------------------------------------------------------
+    # unused_terminals()
+    #
+    # Find all terminals that were defined, but not used by the grammar.  Returns
+    # a list of all symbols.
+    # -----------------------------------------------------------------------------
+    def unused_terminals(self):
+        unused_tok = []
+        for s, v in self.Terminals.items():
+            if s != 'error' and not v:
+                unused_tok.append(s)
+
+        return unused_tok
+
+    # ------------------------------------------------------------------------------
+    # unused_rules()
+    #
+    # Find all grammar rules that were defined,  but not used (maybe not reachable)
+    # Returns a list of productions.
+    # ------------------------------------------------------------------------------
+
+    def unused_rules(self):
+        unused_prod = []
+        for s, v in self.Nonterminals.items():
+            if not v:
+                p = self.Prodnames[s][0]
+                unused_prod.append(p)
+        return unused_prod
+
+    # -----------------------------------------------------------------------------
+    # unused_precedence()
+    #
+    # Returns a list of tuples (term,precedence) corresponding to precedence
+    # rules that were never used by the grammar.  term is the name of the terminal
+    # on which precedence was applied and precedence is a string such as 'left' or
+    # 'right' corresponding to the type of precedence.
+    # -----------------------------------------------------------------------------
+
+    def unused_precedence(self):
+        unused = []
+        for termname in self.Precedence:
+            if not (termname in self.Terminals or termname in self.UsedPrecedence):
+                unused.append((termname, self.Precedence[termname][0]))
+
+        return unused
+
+    # -------------------------------------------------------------------------
+    # _first()
+    #
+    # Compute the value of FIRST1(beta) where beta is a tuple of symbols.
+    #
+    # During execution of compute_first1, the result may be incomplete.
+    # Afterward (e.g., when called from compute_follow()), it will be complete.
+    # -------------------------------------------------------------------------
+    def _first(self, beta):
+
+        # We are computing First(x1,x2,x3,...,xn)
+        result = []
+        for x in beta:
+            x_produces_empty = False
+
+            # Add all the non-<empty> symbols of First[x] to the result.
+            for f in self.First[x]:
+                if f == '<empty>':
+                    x_produces_empty = True
+                else:
+                    if f not in result:
+                        result.append(f)
+
+            if x_produces_empty:
+                # We have to consider the next x in beta,
+                # i.e. stay in the loop.
                 pass
             else:
-                sys.stderr.write("yacc: Infinite recursion detected for symbol '%s'.\n" % s)
-                some_error = 1
+                # We don't have to consider any further symbols in beta.
+                break
+        else:
+            # There was no 'break' from the loop,
+            # so x_produces_empty was true for all x in beta,
+            # so beta produces empty as well.
+            result.append('<empty>')
 
-    return some_error
+        return result
+
+    # -------------------------------------------------------------------------
+    # compute_first()
+    #
+    # Compute the value of FIRST1(X) for all symbols
+    # -------------------------------------------------------------------------
+    def compute_first(self):
+        if self.First:
+            return self.First
+
+        # Terminals:
+        for t in self.Terminals:
+            self.First[t] = [t]
+
+        self.First['$end'] = ['$end']
+
+        # Nonterminals:
+
+        # Initialize to the empty set:
+        for n in self.Nonterminals:
+            self.First[n] = []
+
+        # Then propagate symbols until no change:
+        while True:
+            some_change = False
+            for n in self.Nonterminals:
+                for p in self.Prodnames[n]:
+                    for f in self._first(p.prod):
+                        if f not in self.First[n]:
+                            self.First[n].append(f)
+                            some_change = True
+            if not some_change:
+                break
+
+        return self.First
+
+    # ---------------------------------------------------------------------
+    # compute_follow()
+    #
+    # Computes all of the follow sets for every non-terminal symbol.  The
+    # follow set is the set of all symbols that might follow a given
+    # non-terminal.  See the Dragon book, 2nd Ed. p. 189.
+    # ---------------------------------------------------------------------
+    def compute_follow(self, start=None):
+        # If already computed, return the result
+        if self.Follow:
+            return self.Follow
+
+        # If first sets not computed yet, do that first.
+        if not self.First:
+            self.compute_first()
+
+        # Add '$end' to the follow list of the start symbol
+        for k in self.Nonterminals:
+            self.Follow[k] = []
+
+        if not start:
+            start = self.Productions[1].name
+
+        self.Follow[start] = ['$end']
+
+        while True:
+            didadd = False
+            for p in self.Productions[1:]:
+                # Here is the production set
+                for i, B in enumerate(p.prod):
+                    if B in self.Nonterminals:
+                        # Okay. We got a non-terminal in a production
+                        fst = self._first(p.prod[i+1:])
+                        hasempty = False
+                        for f in fst:
+                            if f != '<empty>' and f not in self.Follow[B]:
+                                self.Follow[B].append(f)
+                                didadd = True
+                            if f == '<empty>':
+                                hasempty = True
+                        if hasempty or i == (len(p.prod)-1):
+                            # Add elements of follow(a) to follow(b)
+                            for f in self.Follow[p.name]:
+                                if f not in self.Follow[B]:
+                                    self.Follow[B].append(f)
+                                    didadd = True
+            if not didadd:
+                break
+        return self.Follow
+
+
+    # -----------------------------------------------------------------------------
+    # build_lritems()
+    #
+    # This function walks the list of productions and builds a complete set of the
+    # LR items.  The LR items are stored in two ways:  First, they are uniquely
+    # numbered and placed in the list _lritems.  Second, a linked list of LR items
+    # is built for each production.  For example:
+    #
+    #   E -> E PLUS E
+    #
+    # Creates the list
+    #
+    #  [E -> . E PLUS E, E -> E . PLUS E, E -> E PLUS . E, E -> E PLUS E . ]
+    # -----------------------------------------------------------------------------
+
+    def build_lritems(self):
+        for p in self.Productions:
+            lastlri = p
+            i = 0
+            lr_items = []
+            while True:
+                if i > len(p):
+                    lri = None
+                else:
+                    lri = LRItem(p, i)
+                    # Precompute the list of productions immediately following
+                    try:
+                        lri.lr_after = self.Prodnames[lri.prod[i+1]]
+                    except (IndexError, KeyError):
+                        lri.lr_after = []
+                    try:
+                        lri.lr_before = lri.prod[i-1]
+                    except IndexError:
+                        lri.lr_before = None
+
+                lastlri.lr_next = lri
+                if not lri:
+                    break
+                lr_items.append(lri)
+                lastlri = lri
+                i += 1
+            p.lr_items = lr_items
 
 # -----------------------------------------------------------------------------
-# verify_productions()
+#                            == Class LRTable ==
 #
-# This function examines all of the supplied rules to see if they seem valid.
-# -----------------------------------------------------------------------------
-def verify_productions(cycle_check=1):
-    error = 0
-    for p in Productions:
-        if not p: continue
-
-        for s in p.prod:
-            if s not in Prodnames and s not in Terminals and s != 'error':
-                sys.stderr.write("%s:%d: Symbol '%s' used, but not defined as a token or a rule.\n" % (p.file,p.line,s))
-                error = 1
-                continue
-
-    unused_tok = 0 
-    # Now verify all of the tokens
-    if yaccdebug:
-        _vf.write("Unused terminals:\n\n")
-    for s,v in Terminals.items():
-        if s != 'error' and not v:
-            sys.stderr.write("yacc: Warning. Token '%s' defined, but not used.\n" % s)
-            if yaccdebug: _vf.write("   %s\n"% s)
-            unused_tok += 1
-
-    # Print out all of the productions
-    if yaccdebug:
-        _vf.write("\nGrammar\n\n")
-        for i in range(1,len(Productions)):
-            _vf.write("Rule %-5d %s\n" % (i, Productions[i]))
-        
-    unused_prod = 0
-    # Verify the use of all productions
-    for s,v in Nonterminals.items():
-        if not v:
-            p = Prodnames[s][0]
-            sys.stderr.write("%s:%d: Warning. Rule '%s' defined, but not used.\n" % (p.file,p.line, s))
-            unused_prod += 1
-
-    
-    if unused_tok == 1:
-        sys.stderr.write("yacc: Warning. There is 1 unused token.\n")
-    if unused_tok > 1:
-        sys.stderr.write("yacc: Warning. There are %d unused tokens.\n" % unused_tok)
-
-    if unused_prod == 1:
-        sys.stderr.write("yacc: Warning. There is 1 unused rule.\n")
-    if unused_prod > 1:
-        sys.stderr.write("yacc: Warning. There are %d unused rules.\n" % unused_prod)
-
-    if yaccdebug:
-        _vf.write("\nTerminals, with rules where they appear\n\n")
-        ks = list(Terminals.keys())
-        ks.sort()
-        for k in ks:
-            _vf.write("%-20s : %s\n" % (k, " ".join([str(s) for s in Terminals[k]])))
-        _vf.write("\nNonterminals, with rules where they appear\n\n")
-        ks = list(Nonterminals.keys())
-        ks.sort()
-        for k in ks:
-            _vf.write("%-20s : %s\n" % (k, " ".join([str(s) for s in Nonterminals[k]])))
-
-    if (cycle_check):
-        compute_reachable()
-        error += compute_terminates()
-#        error += check_cycles()
-    return error
-
-# -----------------------------------------------------------------------------
-# build_lritems()
-#
-# This function walks the list of productions and builds a complete set of the
-# LR items.  The LR items are stored in two ways:  First, they are uniquely
-# numbered and placed in the list _lritems.  Second, a linked list of LR items
-# is built for each production.  For example:
-#
-#   E -> E PLUS E
-#
-# Creates the list
-#
-#  [E -> . E PLUS E, E -> E . PLUS E, E -> E PLUS . E, E -> E PLUS E . ] 
+# This basic class represents a basic table of LR parsing information.
+# Methods for generating the tables are not defined here.  They are defined
+# in the derived class LRGeneratedTable.
 # -----------------------------------------------------------------------------
 
-def build_lritems():
-    for p in Productions:
-        lastlri = p
-        lri = p.lr_item(0)
-        i = 0
-        while 1:
-            lri = p.lr_item(i)
-            lastlri.lr_next = lri
-            if not lri: break
-            lri.lr_num = len(LRitems)
-            LRitems.append(lri)
-            lastlri = lri
-            i += 1
+class VersionError(YaccError):
+    pass
 
-    # In order for the rest of the parser generator to work, we need to
-    # guarantee that no more lritems are generated.  Therefore, we nuke
-    # the p.lr_item method.  (Only used in debugging)
-    # Production.lr_item = None
+class LRTable(object):
+    def __init__(self):
+        self.lr_action = None
+        self.lr_goto = None
+        self.lr_productions = None
+        self.lr_method = None
 
-# -----------------------------------------------------------------------------
-# add_precedence()
-#
-# Given a list of precedence rules, add to the precedence table.
-# -----------------------------------------------------------------------------
+    def read_table(self, module):
+        if isinstance(module, types.ModuleType):
+            parsetab = module
+        else:
+            exec('import %s' % module)
+            parsetab = sys.modules[module]
 
-def add_precedence(plist):
-    plevel = 0
-    error = 0
-    for p in plist:
-        plevel += 1
+        if parsetab._tabversion != __tabversion__:
+            raise VersionError('yacc table file version is out of date')
+
+        self.lr_action = parsetab._lr_action
+        self.lr_goto = parsetab._lr_goto
+
+        self.lr_productions = []
+        for p in parsetab._lr_productions:
+            self.lr_productions.append(MiniProduction(*p))
+
+        self.lr_method = parsetab._lr_method
+        return parsetab._lr_signature
+
+    def read_pickle(self, filename):
         try:
-            prec = p[0]
-            terms = p[1:]
-            if prec != 'left' and prec != 'right' and prec != 'nonassoc':
-                sys.stderr.write("yacc: Invalid precedence '%s'\n" % prec)
-                return -1
-            for t in terms:
-                if t in Precedence:
-                    sys.stderr.write("yacc: Precedence already specified for terminal '%s'\n" % t)
-                    error += 1
-                    continue
-                Precedence[t] = (prec,plevel)
-        except:
-            sys.stderr.write("yacc: Invalid precedence table.\n")
-            error += 1
+            import cPickle as pickle
+        except ImportError:
+            import pickle
 
-    return error
+        if not os.path.exists(filename):
+          raise ImportError
+
+        in_f = open(filename, 'rb')
+
+        tabversion = pickle.load(in_f)
+        if tabversion != __tabversion__:
+            raise VersionError('yacc table file version is out of date')
+        self.lr_method = pickle.load(in_f)
+        signature      = pickle.load(in_f)
+        self.lr_action = pickle.load(in_f)
+        self.lr_goto   = pickle.load(in_f)
+        productions    = pickle.load(in_f)
+
+        self.lr_productions = []
+        for p in productions:
+            self.lr_productions.append(MiniProduction(*p))
+
+        in_f.close()
+        return signature
+
+    # Bind all production function names to callable objects in pdict
+    def bind_callables(self, pdict):
+        for p in self.lr_productions:
+            p.bind(pdict)
+
 
 # -----------------------------------------------------------------------------
-# augment_grammar()
+#                           === LR Generator ===
 #
-# Compute the augmented grammar.  This is just a rule S' -> start where start
-# is the starting symbol.
+# The following classes and functions are used to generate LR parsing tables on
+# a grammar.
 # -----------------------------------------------------------------------------
 
-def augment_grammar(start=None):
-    if not start:
-        start = Productions[1].name
-    Productions[0] = Production(name="S'",prod=[start],number=0,len=1,prec=('right',0),func=None)
-    Productions[0].usyms = [ start ]
-    Nonterminals[start].append(0)
-
-
-# -------------------------------------------------------------------------
-# first()
-#
-# Compute the value of FIRST1(beta) where beta is a tuple of symbols.
-#
-# During execution of compute_first1, the result may be incomplete.
-# Afterward (e.g., when called from compute_follow()), it will be complete.
-# -------------------------------------------------------------------------
-def first(beta):
-
-    # We are computing First(x1,x2,x3,...,xn)
-    result = [ ]
-    for x in beta:
-        x_produces_empty = 0
-
-        # Add all the non-<empty> symbols of First[x] to the result.
-        for f in First[x]:
-            if f == '<empty>':
-                x_produces_empty = 1
-            else:
-                if f not in result: result.append(f)
-
-        if x_produces_empty:
-            # We have to consider the next x in beta,
-            # i.e. stay in the loop.
-            pass
-        else:
-            # We don't have to consider any further symbols in beta.
-            break
-    else:
-        # There was no 'break' from the loop,
-        # so x_produces_empty was true for all x in beta,
-        # so beta produces empty as well.
-        result.append('<empty>')
-
-    return result
-
-
-# FOLLOW(x)
-# Given a non-terminal.  This function computes the set of all symbols
-# that might follow it.  Dragon book, p. 189.
-
-def compute_follow(start=None):
-    # Add '$end' to the follow list of the start symbol
-    for k in Nonterminals.keys():
-        Follow[k] = [ ]
-
-    if not start:
-        start = Productions[1].name
-        
-    Follow[start] = [ '$end' ]
-        
-    while 1:
-        didadd = 0
-        for p in Productions[1:]:
-            # Here is the production set
-            for i in range(len(p.prod)):
-                B = p.prod[i]
-                if B in Nonterminals:
-                    # Okay. We got a non-terminal in a production
-                    fst = first(p.prod[i+1:])
-                    hasempty = 0
-                    for f in fst:
-                        if f != '<empty>' and f not in Follow[B]:
-                            Follow[B].append(f)
-                            didadd = 1
-                        if f == '<empty>':
-                            hasempty = 1
-                    if hasempty or i == (len(p.prod)-1):
-                        # Add elements of follow(a) to follow(b)
-                        for f in Follow[p.name]:
-                            if f not in Follow[B]:
-                                Follow[B].append(f)
-                                didadd = 1
-        if not didadd: break
-
-    if 0 and yaccdebug:
-        _vf.write('\nFollow:\n')
-        for k in Nonterminals.keys():
-            _vf.write("%-20s : %s\n" % (k, " ".join([str(s) for s in Follow[k]])))
-
-# -------------------------------------------------------------------------
-# compute_first1()
-#
-# Compute the value of FIRST1(X) for all symbols
-# -------------------------------------------------------------------------
-def compute_first1():
-
-    # Terminals:
-    for t in Terminals.keys():
-        First[t] = [t]
-
-    First['$end'] = ['$end']
-    First['#'] = ['#'] # what's this for?
-
-    # Nonterminals:
-
-    # Initialize to the empty set:
-    for n in Nonterminals.keys():
-        First[n] = []
-
-    # Then propagate symbols until no change:
-    while 1:
-        some_change = 0
-        for n in Nonterminals.keys():
-            for p in Prodnames[n]:
-                for f in first(p.prod):
-                    if f not in First[n]:
-                        First[n].append( f )
-                        some_change = 1
-        if not some_change:
-            break
-
-    if 0 and yaccdebug:
-        _vf.write('\nFirst:\n')
-        for k in Nonterminals.keys():
-            _vf.write("%-20s : %s\n" %
-                (k, " ".join([str(s) for s in First[k]])))
-
-# -----------------------------------------------------------------------------
-#                           === SLR Generation ===
-#
-# The following functions are used to construct SLR (Simple LR) parsing tables
-# as described on p.221-229 of the dragon book.
-# -----------------------------------------------------------------------------
-
-# Global variables for the LR parsing engine
-def lr_init_vars():
-    global _lr_action, _lr_goto, _lr_method
-    global _lr_goto_cache, _lr0_cidhash
-    
-    _lr_action       = { }        # Action table
-    _lr_goto         = { }        # Goto table
-    _lr_method       = "Unknown"  # LR method used
-    _lr_goto_cache   = { }
-    _lr0_cidhash     = { }
-
-
-# Compute the LR(0) closure operation on I, where I is a set of LR(0) items.
-# prodlist is a list of productions.
-
-_add_count = 0       # Counter used to detect cycles
-
-def lr0_closure(I):
-    global _add_count
-    
-    _add_count += 1
-    prodlist = Productions
-    
-    # Add everything in I to J        
-    J = I[:]
-    didadd = 1
-    while didadd:
-        didadd = 0
-        for j in J:
-            for x in j.lrafter:
-                if x.lr0_added == _add_count: continue
-                # Add B --> .G to J
-                J.append(x.lr_next)
-                x.lr0_added = _add_count
-                didadd = 1
-               
-    return J
-
-# Compute the LR(0) goto function goto(I,X) where I is a set
-# of LR(0) items and X is a grammar symbol.   This function is written
-# in a way that guarantees uniqueness of the generated goto sets
-# (i.e. the same goto set will never be returned as two different Python
-# objects).  With uniqueness, we can later do fast set comparisons using
-# id(obj) instead of element-wise comparison.
-
-def lr0_goto(I,x):
-    # First we look for a previously cached entry
-    g = _lr_goto_cache.get((id(I),x),None)
-    if g: return g
-
-    # Now we generate the goto set in a way that guarantees uniqueness
-    # of the result
-    
-    s = _lr_goto_cache.get(x,None)
-    if not s:
-        s = { }
-        _lr_goto_cache[x] = s
-
-    gs = [ ]
-    for p in I:
-        n = p.lr_next
-        if n and n.lrbefore == x:
-            s1 = s.get(id(n),None)
-            if not s1:
-                s1 = { }
-                s[id(n)] = s1
-            gs.append(n)
-            s = s1
-    g = s.get('$end',None)
-    if not g:
-        if gs:
-            g = lr0_closure(gs)
-            s['$end'] = g
-        else:
-            s['$end'] = gs
-    _lr_goto_cache[(id(I),x)] = g
-    return g
-
-_lr0_cidhash = { }
-
-# Compute the LR(0) sets of item function
-def lr0_items():
-    
-    C = [ lr0_closure([Productions[0].lr_next]) ]
-    i = 0
-    for I in C:
-        _lr0_cidhash[id(I)] = i
-        i += 1
-
-    # Loop over the items in C and each grammar symbols
-    i = 0
-    while i < len(C):
-        I = C[i]
-        i += 1
-
-        # Collect all of the symbols that could possibly be in the goto(I,X) sets
-        asyms = { }
-        for ii in I:
-            for s in ii.usyms:
-                asyms[s] = None
-
-        for x in asyms.keys():
-            g = lr0_goto(I,x)
-            if not g:  continue
-            if id(g) in _lr0_cidhash: continue
-            _lr0_cidhash[id(g)] = len(C)            
-            C.append(g)
-            
-    return C
-
-# -----------------------------------------------------------------------------
-#                       ==== LALR(1) Parsing ====
-#
-# LALR(1) parsing is almost exactly the same as SLR except that instead of
-# relying upon Follow() sets when performing reductions, a more selective
-# lookahead set that incorporates the state of the LR(0) machine is utilized.
-# Thus, we mainly just have to focus on calculating the lookahead sets.
-#
-# The method used here is due to DeRemer and Pennelo (1982).
-#
-# DeRemer, F. L., and T. J. Pennelo: "Efficient Computation of LALR(1)
-#     Lookahead Sets", ACM Transactions on Programming Languages and Systems,
-#     Vol. 4, No. 4, Oct. 1982, pp. 615-649
-#
-# Further details can also be found in:
-#
-#  J. Tremblay and P. Sorenson, "The Theory and Practice of Compiler Writing",
-#      McGraw-Hill Book Company, (1985).
-#
-# Note:  This implementation is a complete replacement of the LALR(1) 
-#        implementation in PLY-1.x releases.   That version was based on
-#        a less efficient algorithm and it had bugs in its implementation.
-# -----------------------------------------------------------------------------
-
-# -----------------------------------------------------------------------------
-# compute_nullable_nonterminals()
-#
-# Creates a dictionary containing all of the non-terminals that might produce
-# an empty production.   
-# -----------------------------------------------------------------------------
-
-def compute_nullable_nonterminals():
-    nullable = {}
-    num_nullable = 0
-    while 1:
-       for p in Productions[1:]:
-           if p.len == 0:
-                nullable[p.name] = 1
-                continue
-           for t in p.prod:
-                if t not in nullable: break
-           else:
-                nullable[p.name] = 1
-       if len(nullable) == num_nullable: break
-       num_nullable = len(nullable)
-    return nullable
-
-# -----------------------------------------------------------------------------
-# find_nonterminal_trans(C)
-#
-# Given a set of LR(0) items, this functions finds all of the non-terminal
-# transitions.    These are transitions in which a dot appears immediately before
-# a non-terminal.   Returns a list of tuples of the form (state,N) where state
-# is the state number and N is the nonterminal symbol.
-#
-# The input C is the set of LR(0) items.
-# -----------------------------------------------------------------------------
-
-def find_nonterminal_transitions(C):
-     trans = []
-     for state in range(len(C)):
-         for p in C[state]:
-             if p.lr_index < p.len - 1:
-                  t = (state,p.prod[p.lr_index+1])
-                  if t[1] in Nonterminals:
-                        if t not in trans: trans.append(t)
-         state = state + 1
-     return trans
-
-# -----------------------------------------------------------------------------
-# dr_relation()
-#
-# Computes the DR(p,A) relationships for non-terminal transitions.  The input
-# is a tuple (state,N) where state is a number and N is a nonterminal symbol.
-#
-# Returns a list of terminals.
-# -----------------------------------------------------------------------------
-
-def dr_relation(C,trans,nullable):
-    dr_set = { }
-    state,N = trans
-    terms = []
-
-    g = lr0_goto(C[state],N)
-    for p in g:
-       if p.lr_index < p.len - 1:
-           a = p.prod[p.lr_index+1]
-           if a in Terminals:
-               if a not in terms: terms.append(a)
-
-    # This extra bit is to handle the start state
-    if state == 0 and N == Productions[0].prod[0]:
-       terms.append('$end')
- 
-    return terms
-
-# -----------------------------------------------------------------------------
-# reads_relation()
-#
-# Computes the READS() relation (p,A) READS (t,C).
-# -----------------------------------------------------------------------------
-
-def reads_relation(C, trans, empty):
-    # Look for empty transitions
-    rel = []
-    state, N = trans
-
-    g = lr0_goto(C[state],N)
-    j = _lr0_cidhash.get(id(g),-1)
-    for p in g:
-        if p.lr_index < p.len - 1:
-             a = p.prod[p.lr_index + 1]
-             if a in empty:
-                  rel.append((j,a))
-
-    return rel
-
-# -----------------------------------------------------------------------------
-# compute_lookback_includes()
-#
-# Determines the lookback and includes relations
-#
-# LOOKBACK:
-# 
-# This relation is determined by running the LR(0) state machine forward.
-# For example, starting with a production "N : . A B C", we run it forward
-# to obtain "N : A B C ."   We then build a relationship between this final
-# state and the starting state.   These relationships are stored in a dictionary
-# lookdict.   
-#
-# INCLUDES:
-#
-# Computes the INCLUDE() relation (p,A) INCLUDES (p',B).   
-#
-# This relation is used to determine non-terminal transitions that occur
-# inside of other non-terminal transition states.   (p,A) INCLUDES (p', B)
-# if the following holds:
-#
-#       B -> LAT, where T -> epsilon and p' -L-> p 
-#
-# L is essentially a prefix (which may be empty), T is a suffix that must be
-# able to derive an empty string.  State p' must lead to state p with the string L.
-# 
-# -----------------------------------------------------------------------------
-
-def compute_lookback_includes(C,trans,nullable):
-    
-    lookdict = {}          # Dictionary of lookback relations
-    includedict = {}       # Dictionary of include relations
-
-    # Make a dictionary of non-terminal transitions
-    dtrans = {}
-    for t in trans:
-        dtrans[t] = 1
-    
-    # Loop over all transitions and compute lookbacks and includes
-    for state,N in trans:
-        lookb = []
-        includes = []
-        for p in C[state]:
-            if p.name != N: continue
-        
-            # Okay, we have a name match.  We now follow the production all the way
-            # through the state machine until we get the . on the right hand side
-
-            lr_index = p.lr_index
-            j = state
-            while lr_index < p.len - 1:
-                 lr_index = lr_index + 1
-                 t = p.prod[lr_index]
-
-                 # Check to see if this symbol and state are a non-terminal transition
-                 if (j,t) in dtrans:
-                       # Yes.  Okay, there is some chance that this is an includes relation
-                       # the only way to know for certain is whether the rest of the 
-                       # production derives empty
-
-                       li = lr_index + 1
-                       while li < p.len:
-                            if p.prod[li] in Terminals: break      # No forget it
-                            if p.prod[li] not in nullable: break
-                            li = li + 1
-                       else:
-                            # Appears to be a relation between (j,t) and (state,N)
-                            includes.append((j,t))
-
-                 g = lr0_goto(C[j],t)               # Go to next set             
-                 j = _lr0_cidhash.get(id(g),-1)     # Go to next state
-             
-            # When we get here, j is the final state, now we have to locate the production
-            for r in C[j]:
-                 if r.name != p.name: continue
-                 if r.len != p.len:   continue
-                 i = 0
-                 # This look is comparing a production ". A B C" with "A B C ."
-                 while i < r.lr_index:
-                      if r.prod[i] != p.prod[i+1]: break
-                      i = i + 1
-                 else:
-                      lookb.append((j,r))
-        for i in includes:
-             if i not in includedict: includedict[i] = []
-             includedict[i].append((state,N))
-        lookdict[(state,N)] = lookb
-
-    return lookdict,includedict
-
 # -----------------------------------------------------------------------------
 # digraph()
 # traverse()
@@ -1495,720 +2054,1449 @@
 #          FP   - Set-valued function
 # ------------------------------------------------------------------------------
 
-def digraph(X,R,FP):
-    N = { }
+def digraph(X, R, FP):
+    N = {}
     for x in X:
-       N[x] = 0
+        N[x] = 0
     stack = []
-    F = { }
+    F = {}
     for x in X:
-        if N[x] == 0: traverse(x,N,stack,F,X,R,FP)
+        if N[x] == 0:
+            traverse(x, N, stack, F, X, R, FP)
     return F
 
-def traverse(x,N,stack,F,X,R,FP):
+def traverse(x, N, stack, F, X, R, FP):
     stack.append(x)
     d = len(stack)
     N[x] = d
     F[x] = FP(x)             # F(X) <- F'(x)
-    
+
     rel = R(x)               # Get y's related to x
     for y in rel:
         if N[y] == 0:
-             traverse(y,N,stack,F,X,R,FP)
-        N[x] = min(N[x],N[y])
-        for a in F.get(y,[]):
-            if a not in F[x]: F[x].append(a)
+            traverse(y, N, stack, F, X, R, FP)
+        N[x] = min(N[x], N[y])
+        for a in F.get(y, []):
+            if a not in F[x]:
+                F[x].append(a)
     if N[x] == d:
-       N[stack[-1]] = sys.maxsize
-       F[stack[-1]] = F[x]
-       element = stack.pop()
-       while element != x:
-           N[stack[-1]] = sys.maxsize
-           F[stack[-1]] = F[x]
-           element = stack.pop()
+        N[stack[-1]] = MAXINT
+        F[stack[-1]] = F[x]
+        element = stack.pop()
+        while element != x:
+            N[stack[-1]] = MAXINT
+            F[stack[-1]] = F[x]
+            element = stack.pop()
+
+class LALRError(YaccError):
+    pass
 
 # -----------------------------------------------------------------------------
-# compute_read_sets()
+#                             == LRGeneratedTable ==
 #
-# Given a set of LR(0) items, this function computes the read sets.
-#
-# Inputs:  C        =  Set of LR(0) items
-#          ntrans   = Set of nonterminal transitions
-#          nullable = Set of empty transitions
-#
-# Returns a set containing the read sets
+# This class implements the LR table generation algorithm.  There are no
+# public methods except for write()
 # -----------------------------------------------------------------------------
 
-def compute_read_sets(C, ntrans, nullable):
-    FP = lambda x: dr_relation(C,x,nullable)
-    R =  lambda x: reads_relation(C,x,nullable)
-    F = digraph(ntrans,R,FP)
-    return F
+class LRGeneratedTable(LRTable):
+    def __init__(self, grammar, method='LALR', log=None):
+        if method not in ['SLR', 'LALR']:
+            raise LALRError('Unsupported method %s' % method)
 
-# -----------------------------------------------------------------------------
-# compute_follow_sets()
-#
-# Given a set of LR(0) items, a set of non-terminal transitions, a readset, 
-# and an include set, this function computes the follow sets
-#
-# Follow(p,A) = Read(p,A) U U {Follow(p',B) | (p,A) INCLUDES (p',B)}
-#
-# Inputs:    
-#            ntrans     = Set of nonterminal transitions
-#            readsets   = Readset (previously computed)
-#            inclsets   = Include sets (previously computed)
-#
-# Returns a set containing the follow sets      
-# -----------------------------------------------------------------------------
+        self.grammar = grammar
+        self.lr_method = method
 
-def compute_follow_sets(ntrans,readsets,inclsets):
-     FP = lambda x: readsets[x]
-     R  = lambda x: inclsets.get(x,[])
-     F = digraph(ntrans,R,FP)
-     return F
+        # Set up the logger
+        if not log:
+            log = NullLogger()
+        self.log = log
 
-# -----------------------------------------------------------------------------
-# add_lookaheads()
-#
-# Attaches the lookahead symbols to grammar rules. 
-#
-# Inputs:    lookbacks         -  Set of lookback relations
-#            followset         -  Computed follow set
-#
-# This function directly attaches the lookaheads to productions contained
-# in the lookbacks set
-# -----------------------------------------------------------------------------
+        # Internal attributes
+        self.lr_action     = {}        # Action table
+        self.lr_goto       = {}        # Goto table
+        self.lr_productions  = grammar.Productions    # Copy of grammar Production array
+        self.lr_goto_cache = {}        # Cache of computed gotos
+        self.lr0_cidhash   = {}        # Cache of closures
 
-def add_lookaheads(lookbacks,followset):
-    for trans,lb in lookbacks.items():
-        # Loop over productions in lookback
-        for state,p in lb:
-             if state not in p.lookaheads:
-                  p.lookaheads[state] = []
-             f = followset.get(trans,[])
-             for a in f:
-                  if a not in p.lookaheads[state]: p.lookaheads[state].append(a)
+        self._add_count    = 0         # Internal counter used to detect cycles
 
-# -----------------------------------------------------------------------------
-# add_lalr_lookaheads()
-#
-# This function does all of the work of adding lookahead information for use
-# with LALR parsing
-# -----------------------------------------------------------------------------
+        # Diagonistic information filled in by the table generator
+        self.sr_conflict   = 0
+        self.rr_conflict   = 0
+        self.conflicts     = []        # List of conflicts
 
-def add_lalr_lookaheads(C):
-    # Determine all of the nullable nonterminals
-    nullable = compute_nullable_nonterminals()
+        self.sr_conflicts  = []
+        self.rr_conflicts  = []
 
-    # Find all non-terminal transitions
-    trans = find_nonterminal_transitions(C)
+        # Build the tables
+        self.grammar.build_lritems()
+        self.grammar.compute_first()
+        self.grammar.compute_follow()
+        self.lr_parse_table()
 
-    # Compute read sets
-    readsets = compute_read_sets(C,trans,nullable)
+    # Compute the LR(0) closure operation on I, where I is a set of LR(0) items.
 
-    # Compute lookback/includes relations
-    lookd, included = compute_lookback_includes(C,trans,nullable)
+    def lr0_closure(self, I):
+        self._add_count += 1
 
-    # Compute LALR FOLLOW sets
-    followsets = compute_follow_sets(trans,readsets,included)
-    
-    # Add all of the lookaheads
-    add_lookaheads(lookd,followsets)
+        # Add everything in I to J
+        J = I[:]
+        didadd = True
+        while didadd:
+            didadd = False
+            for j in J:
+                for x in j.lr_after:
+                    if getattr(x, 'lr0_added', 0) == self._add_count:
+                        continue
+                    # Add B --> .G to J
+                    J.append(x.lr_next)
+                    x.lr0_added = self._add_count
+                    didadd = True
 
-# -----------------------------------------------------------------------------
-# lr_parse_table()
-#
-# This function constructs the parse tables for SLR or LALR
-# -----------------------------------------------------------------------------
-def lr_parse_table(method):
-    global _lr_method
-    goto = _lr_goto           # Goto array
-    action = _lr_action       # Action array
-    actionp = { }             # Action production array (temporary)
+        return J
 
-    _lr_method = method
-    
-    n_srconflict = 0
-    n_rrconflict = 0
+    # Compute the LR(0) goto function goto(I,X) where I is a set
+    # of LR(0) items and X is a grammar symbol.   This function is written
+    # in a way that guarantees uniqueness of the generated goto sets
+    # (i.e. the same goto set will never be returned as two different Python
+    # objects).  With uniqueness, we can later do fast set comparisons using
+    # id(obj) instead of element-wise comparison.
 
-    if yaccdebug:
-        sys.stderr.write("yacc: Generating %s parsing table...\n" % method)        
-        _vf.write("\n\nParsing method: %s\n\n" % method)
-        
-    # Step 1: Construct C = { I0, I1, ... IN}, collection of LR(0) items
-    # This determines the number of states
-    
-    C = lr0_items()
+    def lr0_goto(self, I, x):
+        # First we look for a previously cached entry
+        g = self.lr_goto_cache.get((id(I), x))
+        if g:
+            return g
 
-    if method == 'LALR':
-        add_lalr_lookaheads(C)
+        # Now we generate the goto set in a way that guarantees uniqueness
+        # of the result
 
-    # Build the parser table, state by state
-    st = 0
-    for I in C:
-        # Loop over each production in I
-        actlist = [ ]              # List of actions
-        
-        if yaccdebug:
-            _vf.write("\nstate %d\n\n" % st)
-            for p in I:
-                _vf.write("    (%d) %s\n" % (p.number, str(p)))
-            _vf.write("\n")
+        s = self.lr_goto_cache.get(x)
+        if not s:
+            s = {}
+            self.lr_goto_cache[x] = s
 
+        gs = []
         for p in I:
-            try:
-                if p.prod[-1] == ".":
-                    if p.name == "S'":
-                        # Start symbol. Accept!
-                        action[st,"$end"] = 0
-                        actionp[st,"$end"] = p
-                    else:
-                        # We are at the end of a production.  Reduce!
-                        if method == 'LALR':
-                            laheads = p.lookaheads[st]
-                        else:
-                            laheads = Follow[p.name]
-                        for a in laheads:
-                            actlist.append((a,p,"reduce using rule %d (%s)" % (p.number,p)))
-                            r = action.get((st,a),None)
-                            if r is not None:
-                                # Whoa. Have a shift/reduce or reduce/reduce conflict
-                                if r > 0:
-                                    # Need to decide on shift or reduce here
-                                    # By default we favor shifting. Need to add
-                                    # some precedence rules here.
-                                    sprec,slevel = Productions[actionp[st,a].number].prec                                    
-                                    rprec,rlevel = Precedence.get(a,('right',0))
-                                    if (slevel < rlevel) or ((slevel == rlevel) and (rprec == 'left')):
-                                        # We really need to reduce here.  
-                                        action[st,a] = -p.number
-                                        actionp[st,a] = p
-                                        if not slevel and not rlevel:
-                                            _vfc.write("shift/reduce conflict in state %d resolved as reduce.\n" % st)
-                                            _vf.write("  ! shift/reduce conflict for %s resolved as reduce.\n" % a)
-                                            n_srconflict += 1
-                                    elif (slevel == rlevel) and (rprec == 'nonassoc'):
-                                        action[st,a] = None
-                                    else:
-                                        # Hmmm. Guess we'll keep the shift
-                                        if not rlevel:
-                                            _vfc.write("shift/reduce conflict in state %d resolved as shift.\n" % st)
-                                            _vf.write("  ! shift/reduce conflict for %s resolved as shift.\n" % a)
-                                            n_srconflict +=1                                    
-                                elif r < 0:
-                                    # Reduce/reduce conflict.   In this case, we favor the rule
-                                    # that was defined first in the grammar file
-                                    oldp = Productions[-r]
-                                    pp = Productions[p.number]
-                                    if oldp.line > pp.line:
-                                        action[st,a] = -p.number
-                                        actionp[st,a] = p
-                                    # sys.stderr.write("Reduce/reduce conflict in state %d\n" % st)
-                                    n_rrconflict += 1
-                                    _vfc.write("reduce/reduce conflict in state %d resolved using rule %d (%s).\n" % (st, actionp[st,a].number, actionp[st,a]))
-                                    _vf.write("  ! reduce/reduce conflict for %s resolved using rule %d (%s).\n" % (a,actionp[st,a].number, actionp[st,a]))
-                                else:
-                                    sys.stderr.write("Unknown conflict in state %d\n" % st)
-                            else:
-                                action[st,a] = -p.number
-                                actionp[st,a] = p
+            n = p.lr_next
+            if n and n.lr_before == x:
+                s1 = s.get(id(n))
+                if not s1:
+                    s1 = {}
+                    s[id(n)] = s1
+                gs.append(n)
+                s = s1
+        g = s.get('$end')
+        if not g:
+            if gs:
+                g = self.lr0_closure(gs)
+                s['$end'] = g
+            else:
+                s['$end'] = gs
+        self.lr_goto_cache[(id(I), x)] = g
+        return g
+
+    # Compute the LR(0) sets of item function
+    def lr0_items(self):
+        C = [self.lr0_closure([self.grammar.Productions[0].lr_next])]
+        i = 0
+        for I in C:
+            self.lr0_cidhash[id(I)] = i
+            i += 1
+
+        # Loop over the items in C and each grammar symbols
+        i = 0
+        while i < len(C):
+            I = C[i]
+            i += 1
+
+            # Collect all of the symbols that could possibly be in the goto(I,X) sets
+            asyms = {}
+            for ii in I:
+                for s in ii.usyms:
+                    asyms[s] = None
+
+            for x in asyms:
+                g = self.lr0_goto(I, x)
+                if not g or id(g) in self.lr0_cidhash:
+                    continue
+                self.lr0_cidhash[id(g)] = len(C)
+                C.append(g)
+
+        return C
+
+    # -----------------------------------------------------------------------------
+    #                       ==== LALR(1) Parsing ====
+    #
+    # LALR(1) parsing is almost exactly the same as SLR except that instead of
+    # relying upon Follow() sets when performing reductions, a more selective
+    # lookahead set that incorporates the state of the LR(0) machine is utilized.
+    # Thus, we mainly just have to focus on calculating the lookahead sets.
+    #
+    # The method used here is due to DeRemer and Pennelo (1982).
+    #
+    # DeRemer, F. L., and T. J. Pennelo: "Efficient Computation of LALR(1)
+    #     Lookahead Sets", ACM Transactions on Programming Languages and Systems,
+    #     Vol. 4, No. 4, Oct. 1982, pp. 615-649
+    #
+    # Further details can also be found in:
+    #
+    #  J. Tremblay and P. Sorenson, "The Theory and Practice of Compiler Writing",
+    #      McGraw-Hill Book Company, (1985).
+    #
+    # -----------------------------------------------------------------------------
+
+    # -----------------------------------------------------------------------------
+    # compute_nullable_nonterminals()
+    #
+    # Creates a dictionary containing all of the non-terminals that might produce
+    # an empty production.
+    # -----------------------------------------------------------------------------
+
+    def compute_nullable_nonterminals(self):
+        nullable = set()
+        num_nullable = 0
+        while True:
+            for p in self.grammar.Productions[1:]:
+                if p.len == 0:
+                    nullable.add(p.name)
+                    continue
+                for t in p.prod:
+                    if t not in nullable:
+                        break
                 else:
-                    i = p.lr_index
-                    a = p.prod[i+1]       # Get symbol right after the "."
-                    if a in Terminals:
-                        g = lr0_goto(I,a)
-                        j = _lr0_cidhash.get(id(g),-1)
-                        if j >= 0:
-                            # We are in a shift state
-                            actlist.append((a,p,"shift and go to state %d" % j))
-                            r = action.get((st,a),None)
-                            if r is not None:
-                                # Whoa have a shift/reduce or shift/shift conflict
-                                if r > 0:
-                                    if r != j:
-                                        sys.stderr.write("Shift/shift conflict in state %d\n" % st)
-                                elif r < 0:
-                                    # Do a precedence check.
-                                    #   -  if precedence of reduce rule is higher, we reduce.
-                                    #   -  if precedence of reduce is same and left assoc, we reduce.
-                                    #   -  otherwise we shift
-                                    rprec,rlevel = Productions[actionp[st,a].number].prec
-                                    sprec,slevel = Precedence.get(a,('right',0))
-                                    if (slevel > rlevel) or ((slevel == rlevel) and (rprec != 'left')):
-                                        # We decide to shift here... highest precedence to shift
-                                        action[st,a] = j
-                                        actionp[st,a] = p
-                                        if not rlevel:
-                                            n_srconflict += 1
-                                            _vfc.write("shift/reduce conflict in state %d resolved as shift.\n" % st)
-                                            _vf.write("  ! shift/reduce conflict for %s resolved as shift.\n" % a)
-                                    elif (slevel == rlevel) and (rprec == 'nonassoc'):
-                                        action[st,a] = None
-                                    else:                                            
-                                        # Hmmm. Guess we'll keep the reduce
-                                        if not slevel and not rlevel:
-                                            n_srconflict +=1
-                                            _vfc.write("shift/reduce conflict in state %d resolved as reduce.\n" % st)
-                                            _vf.write("  ! shift/reduce conflict for %s resolved as reduce.\n" % a)
-                                            
-                                else:
-                                    sys.stderr.write("Unknown conflict in state %d\n" % st)
+                    nullable.add(p.name)
+            if len(nullable) == num_nullable:
+                break
+            num_nullable = len(nullable)
+        return nullable
+
+    # -----------------------------------------------------------------------------
+    # find_nonterminal_trans(C)
+    #
+    # Given a set of LR(0) items, this functions finds all of the non-terminal
+    # transitions.    These are transitions in which a dot appears immediately before
+    # a non-terminal.   Returns a list of tuples of the form (state,N) where state
+    # is the state number and N is the nonterminal symbol.
+    #
+    # The input C is the set of LR(0) items.
+    # -----------------------------------------------------------------------------
+
+    def find_nonterminal_transitions(self, C):
+        trans = []
+        for stateno, state in enumerate(C):
+            for p in state:
+                if p.lr_index < p.len - 1:
+                    t = (stateno, p.prod[p.lr_index+1])
+                    if t[1] in self.grammar.Nonterminals:
+                        if t not in trans:
+                            trans.append(t)
+        return trans
+
+    # -----------------------------------------------------------------------------
+    # dr_relation()
+    #
+    # Computes the DR(p,A) relationships for non-terminal transitions.  The input
+    # is a tuple (state,N) where state is a number and N is a nonterminal symbol.
+    #
+    # Returns a list of terminals.
+    # -----------------------------------------------------------------------------
+
+    def dr_relation(self, C, trans, nullable):
+        state, N = trans
+        terms = []
+
+        g = self.lr0_goto(C[state], N)
+        for p in g:
+            if p.lr_index < p.len - 1:
+                a = p.prod[p.lr_index+1]
+                if a in self.grammar.Terminals:
+                    if a not in terms:
+                        terms.append(a)
+
+        # This extra bit is to handle the start state
+        if state == 0 and N == self.grammar.Productions[0].prod[0]:
+            terms.append('$end')
+
+        return terms
+
+    # -----------------------------------------------------------------------------
+    # reads_relation()
+    #
+    # Computes the READS() relation (p,A) READS (t,C).
+    # -----------------------------------------------------------------------------
+
+    def reads_relation(self, C, trans, empty):
+        # Look for empty transitions
+        rel = []
+        state, N = trans
+
+        g = self.lr0_goto(C[state], N)
+        j = self.lr0_cidhash.get(id(g), -1)
+        for p in g:
+            if p.lr_index < p.len - 1:
+                a = p.prod[p.lr_index + 1]
+                if a in empty:
+                    rel.append((j, a))
+
+        return rel
+
+    # -----------------------------------------------------------------------------
+    # compute_lookback_includes()
+    #
+    # Determines the lookback and includes relations
+    #
+    # LOOKBACK:
+    #
+    # This relation is determined by running the LR(0) state machine forward.
+    # For example, starting with a production "N : . A B C", we run it forward
+    # to obtain "N : A B C ."   We then build a relationship between this final
+    # state and the starting state.   These relationships are stored in a dictionary
+    # lookdict.
+    #
+    # INCLUDES:
+    #
+    # Computes the INCLUDE() relation (p,A) INCLUDES (p',B).
+    #
+    # This relation is used to determine non-terminal transitions that occur
+    # inside of other non-terminal transition states.   (p,A) INCLUDES (p', B)
+    # if the following holds:
+    #
+    #       B -> LAT, where T -> epsilon and p' -L-> p
+    #
+    # L is essentially a prefix (which may be empty), T is a suffix that must be
+    # able to derive an empty string.  State p' must lead to state p with the string L.
+    #
+    # -----------------------------------------------------------------------------
+
+    def compute_lookback_includes(self, C, trans, nullable):
+        lookdict = {}          # Dictionary of lookback relations
+        includedict = {}       # Dictionary of include relations
+
+        # Make a dictionary of non-terminal transitions
+        dtrans = {}
+        for t in trans:
+            dtrans[t] = 1
+
+        # Loop over all transitions and compute lookbacks and includes
+        for state, N in trans:
+            lookb = []
+            includes = []
+            for p in C[state]:
+                if p.name != N:
+                    continue
+
+                # Okay, we have a name match.  We now follow the production all the way
+                # through the state machine until we get the . on the right hand side
+
+                lr_index = p.lr_index
+                j = state
+                while lr_index < p.len - 1:
+                    lr_index = lr_index + 1
+                    t = p.prod[lr_index]
+
+                    # Check to see if this symbol and state are a non-terminal transition
+                    if (j, t) in dtrans:
+                        # Yes.  Okay, there is some chance that this is an includes relation
+                        # the only way to know for certain is whether the rest of the
+                        # production derives empty
+
+                        li = lr_index + 1
+                        while li < p.len:
+                            if p.prod[li] in self.grammar.Terminals:
+                                break      # No forget it
+                            if p.prod[li] not in nullable:
+                                break
+                            li = li + 1
+                        else:
+                            # Appears to be a relation between (j,t) and (state,N)
+                            includes.append((j, t))
+
+                    g = self.lr0_goto(C[j], t)               # Go to next set
+                    j = self.lr0_cidhash.get(id(g), -1)      # Go to next state
+
+                # When we get here, j is the final state, now we have to locate the production
+                for r in C[j]:
+                    if r.name != p.name:
+                        continue
+                    if r.len != p.len:
+                        continue
+                    i = 0
+                    # This look is comparing a production ". A B C" with "A B C ."
+                    while i < r.lr_index:
+                        if r.prod[i] != p.prod[i+1]:
+                            break
+                        i = i + 1
+                    else:
+                        lookb.append((j, r))
+            for i in includes:
+                if i not in includedict:
+                    includedict[i] = []
+                includedict[i].append((state, N))
+            lookdict[(state, N)] = lookb
+
+        return lookdict, includedict
+
+    # -----------------------------------------------------------------------------
+    # compute_read_sets()
+    #
+    # Given a set of LR(0) items, this function computes the read sets.
+    #
+    # Inputs:  C        =  Set of LR(0) items
+    #          ntrans   = Set of nonterminal transitions
+    #          nullable = Set of empty transitions
+    #
+    # Returns a set containing the read sets
+    # -----------------------------------------------------------------------------
+
+    def compute_read_sets(self, C, ntrans, nullable):
+        FP = lambda x: self.dr_relation(C, x, nullable)
+        R =  lambda x: self.reads_relation(C, x, nullable)
+        F = digraph(ntrans, R, FP)
+        return F
+
+    # -----------------------------------------------------------------------------
+    # compute_follow_sets()
+    #
+    # Given a set of LR(0) items, a set of non-terminal transitions, a readset,
+    # and an include set, this function computes the follow sets
+    #
+    # Follow(p,A) = Read(p,A) U U {Follow(p',B) | (p,A) INCLUDES (p',B)}
+    #
+    # Inputs:
+    #            ntrans     = Set of nonterminal transitions
+    #            readsets   = Readset (previously computed)
+    #            inclsets   = Include sets (previously computed)
+    #
+    # Returns a set containing the follow sets
+    # -----------------------------------------------------------------------------
+
+    def compute_follow_sets(self, ntrans, readsets, inclsets):
+        FP = lambda x: readsets[x]
+        R  = lambda x: inclsets.get(x, [])
+        F = digraph(ntrans, R, FP)
+        return F
+
+    # -----------------------------------------------------------------------------
+    # add_lookaheads()
+    #
+    # Attaches the lookahead symbols to grammar rules.
+    #
+    # Inputs:    lookbacks         -  Set of lookback relations
+    #            followset         -  Computed follow set
+    #
+    # This function directly attaches the lookaheads to productions contained
+    # in the lookbacks set
+    # -----------------------------------------------------------------------------
+
+    def add_lookaheads(self, lookbacks, followset):
+        for trans, lb in lookbacks.items():
+            # Loop over productions in lookback
+            for state, p in lb:
+                if state not in p.lookaheads:
+                    p.lookaheads[state] = []
+                f = followset.get(trans, [])
+                for a in f:
+                    if a not in p.lookaheads[state]:
+                        p.lookaheads[state].append(a)
+
+    # -----------------------------------------------------------------------------
+    # add_lalr_lookaheads()
+    #
+    # This function does all of the work of adding lookahead information for use
+    # with LALR parsing
+    # -----------------------------------------------------------------------------
+
+    def add_lalr_lookaheads(self, C):
+        # Determine all of the nullable nonterminals
+        nullable = self.compute_nullable_nonterminals()
+
+        # Find all non-terminal transitions
+        trans = self.find_nonterminal_transitions(C)
+
+        # Compute read sets
+        readsets = self.compute_read_sets(C, trans, nullable)
+
+        # Compute lookback/includes relations
+        lookd, included = self.compute_lookback_includes(C, trans, nullable)
+
+        # Compute LALR FOLLOW sets
+        followsets = self.compute_follow_sets(trans, readsets, included)
+
+        # Add all of the lookaheads
+        self.add_lookaheads(lookd, followsets)
+
+    # -----------------------------------------------------------------------------
+    # lr_parse_table()
+    #
+    # This function constructs the parse tables for SLR or LALR
+    # -----------------------------------------------------------------------------
+    def lr_parse_table(self):
+        Productions = self.grammar.Productions
+        Precedence  = self.grammar.Precedence
+        goto   = self.lr_goto         # Goto array
+        action = self.lr_action       # Action array
+        log    = self.log             # Logger for output
+
+        actionp = {}                  # Action production array (temporary)
+
+        log.info('Parsing method: %s', self.lr_method)
+
+        # Step 1: Construct C = { I0, I1, ... IN}, collection of LR(0) items
+        # This determines the number of states
+
+        C = self.lr0_items()
+
+        if self.lr_method == 'LALR':
+            self.add_lalr_lookaheads(C)
+
+        # Build the parser table, state by state
+        st = 0
+        for I in C:
+            # Loop over each production in I
+            actlist = []              # List of actions
+            st_action  = {}
+            st_actionp = {}
+            st_goto    = {}
+            log.info('')
+            log.info('state %d', st)
+            log.info('')
+            for p in I:
+                log.info('    (%d) %s', p.number, p)
+            log.info('')
+
+            for p in I:
+                    if p.len == p.lr_index + 1:
+                        if p.name == "S'":
+                            # Start symbol. Accept!
+                            st_action['$end'] = 0
+                            st_actionp['$end'] = p
+                        else:
+                            # We are at the end of a production.  Reduce!
+                            if self.lr_method == 'LALR':
+                                laheads = p.lookaheads[st]
                             else:
-                                action[st,a] = j
-                                actionp[st,a] = p
-                                
-            except Exception as e:
-                raise YaccError("Hosed in lr_parse_table").with_traceback(e)
+                                laheads = self.grammar.Follow[p.name]
+                            for a in laheads:
+                                actlist.append((a, p, 'reduce using rule %d (%s)' % (p.number, p)))
+                                r = st_action.get(a)
+                                if r is not None:
+                                    # Whoa. Have a shift/reduce or reduce/reduce conflict
+                                    if r > 0:
+                                        # Need to decide on shift or reduce here
+                                        # By default we favor shifting. Need to add
+                                        # some precedence rules here.
 
-        # Print the actions associated with each terminal
-        if yaccdebug:
-          _actprint = { }
-          for a,p,m in actlist:
-            if (st,a) in action:
-                if p is actionp[st,a]:
-                    _vf.write("    %-15s %s\n" % (a,m))
-                    _actprint[(a,m)] = 1
-          _vf.write("\n")
-          for a,p,m in actlist:
-            if (st,a) in action:
-                if p is not actionp[st,a]:
-                    if (a,m) not in _actprint:
-                        _vf.write("  ! %-15s [ %s ]\n" % (a,m))
-                        _actprint[(a,m)] = 1
-            
-        # Construct the goto table for this state
-        if yaccdebug:
-            _vf.write("\n")
-        nkeys = { }
-        for ii in I:
-            for s in ii.usyms:
-                if s in Nonterminals:
-                    nkeys[s] = None
-        for n in nkeys.keys():
-            g = lr0_goto(I,n)
-            j = _lr0_cidhash.get(id(g),-1)            
-            if j >= 0:
-                goto[st,n] = j
-                if yaccdebug:
-                    _vf.write("    %-30s shift and go to state %d\n" % (n,j))
+                                        # Shift precedence comes from the token
+                                        sprec, slevel = Precedence.get(a, ('right', 0))
 
-        st += 1
+                                        # Reduce precedence comes from rule being reduced (p)
+                                        rprec, rlevel = Productions[p.number].prec
 
-    if yaccdebug:
-        if n_srconflict == 1:
-            sys.stderr.write("yacc: %d shift/reduce conflict\n" % n_srconflict)
-        if n_srconflict > 1:
-            sys.stderr.write("yacc: %d shift/reduce conflicts\n" % n_srconflict)
-        if n_rrconflict == 1:
-            sys.stderr.write("yacc: %d reduce/reduce conflict\n" % n_rrconflict)
-        if n_rrconflict > 1:
-            sys.stderr.write("yacc: %d reduce/reduce conflicts\n" % n_rrconflict)
+                                        if (slevel < rlevel) or ((slevel == rlevel) and (rprec == 'left')):
+                                            # We really need to reduce here.
+                                            st_action[a] = -p.number
+                                            st_actionp[a] = p
+                                            if not slevel and not rlevel:
+                                                log.info('  ! shift/reduce conflict for %s resolved as reduce', a)
+                                                self.sr_conflicts.append((st, a, 'reduce'))
+                                            Productions[p.number].reduced += 1
+                                        elif (slevel == rlevel) and (rprec == 'nonassoc'):
+                                            st_action[a] = None
+                                        else:
+                                            # Hmmm. Guess we'll keep the shift
+                                            if not rlevel:
+                                                log.info('  ! shift/reduce conflict for %s resolved as shift', a)
+                                                self.sr_conflicts.append((st, a, 'shift'))
+                                    elif r < 0:
+                                        # Reduce/reduce conflict.   In this case, we favor the rule
+                                        # that was defined first in the grammar file
+                                        oldp = Productions[-r]
+                                        pp = Productions[p.number]
+                                        if oldp.line > pp.line:
+                                            st_action[a] = -p.number
+                                            st_actionp[a] = p
+                                            chosenp, rejectp = pp, oldp
+                                            Productions[p.number].reduced += 1
+                                            Productions[oldp.number].reduced -= 1
+                                        else:
+                                            chosenp, rejectp = oldp, pp
+                                        self.rr_conflicts.append((st, chosenp, rejectp))
+                                        log.info('  ! reduce/reduce conflict for %s resolved using rule %d (%s)',
+                                                 a, st_actionp[a].number, st_actionp[a])
+                                    else:
+                                        raise LALRError('Unknown conflict in state %d' % st)
+                                else:
+                                    st_action[a] = -p.number
+                                    st_actionp[a] = p
+                                    Productions[p.number].reduced += 1
+                    else:
+                        i = p.lr_index
+                        a = p.prod[i+1]       # Get symbol right after the "."
+                        if a in self.grammar.Terminals:
+                            g = self.lr0_goto(I, a)
+                            j = self.lr0_cidhash.get(id(g), -1)
+                            if j >= 0:
+                                # We are in a shift state
+                                actlist.append((a, p, 'shift and go to state %d' % j))
+                                r = st_action.get(a)
+                                if r is not None:
+                                    # Whoa have a shift/reduce or shift/shift conflict
+                                    if r > 0:
+                                        if r != j:
+                                            raise LALRError('Shift/shift conflict in state %d' % st)
+                                    elif r < 0:
+                                        # Do a precedence check.
+                                        #   -  if precedence of reduce rule is higher, we reduce.
+                                        #   -  if precedence of reduce is same and left assoc, we reduce.
+                                        #   -  otherwise we shift
 
-# -----------------------------------------------------------------------------
-#                          ==== LR Utility functions ====
-# -----------------------------------------------------------------------------
+                                        # Shift precedence comes from the token
+                                        sprec, slevel = Precedence.get(a, ('right', 0))
 
-# -----------------------------------------------------------------------------
-# _lr_write_tables()
-#
-# This function writes the LR parsing tables to a file
-# -----------------------------------------------------------------------------
+                                        # Reduce precedence comes from the rule that could have been reduced
+                                        rprec, rlevel = Productions[st_actionp[a].number].prec
 
-def lr_write_tables(modulename=tab_module,outputdir=''):
-    filename = os.path.join(outputdir,modulename) + ".py"
-    try:
-        f = open(filename,"w")
+                                        if (slevel > rlevel) or ((slevel == rlevel) and (rprec == 'right')):
+                                            # We decide to shift here... highest precedence to shift
+                                            Productions[st_actionp[a].number].reduced -= 1
+                                            st_action[a] = j
+                                            st_actionp[a] = p
+                                            if not rlevel:
+                                                log.info('  ! shift/reduce conflict for %s resolved as shift', a)
+                                                self.sr_conflicts.append((st, a, 'shift'))
+                                        elif (slevel == rlevel) and (rprec == 'nonassoc'):
+                                            st_action[a] = None
+                                        else:
+                                            # Hmmm. Guess we'll keep the reduce
+                                            if not slevel and not rlevel:
+                                                log.info('  ! shift/reduce conflict for %s resolved as reduce', a)
+                                                self.sr_conflicts.append((st, a, 'reduce'))
 
-        f.write("""
+                                    else:
+                                        raise LALRError('Unknown conflict in state %d' % st)
+                                else:
+                                    st_action[a] = j
+                                    st_actionp[a] = p
+
+            # Print the actions associated with each terminal
+            _actprint = {}
+            for a, p, m in actlist:
+                if a in st_action:
+                    if p is st_actionp[a]:
+                        log.info('    %-15s %s', a, m)
+                        _actprint[(a, m)] = 1
+            log.info('')
+            # Print the actions that were not used. (debugging)
+            not_used = 0
+            for a, p, m in actlist:
+                if a in st_action:
+                    if p is not st_actionp[a]:
+                        if not (a, m) in _actprint:
+                            log.debug('  ! %-15s [ %s ]', a, m)
+                            not_used = 1
+                            _actprint[(a, m)] = 1
+            if not_used:
+                log.debug('')
+
+            # Construct the goto table for this state
+
+            nkeys = {}
+            for ii in I:
+                for s in ii.usyms:
+                    if s in self.grammar.Nonterminals:
+                        nkeys[s] = None
+            for n in nkeys:
+                g = self.lr0_goto(I, n)
+                j = self.lr0_cidhash.get(id(g), -1)
+                if j >= 0:
+                    st_goto[n] = j
+                    log.info('    %-30s shift and go to state %d', n, j)
+
+            action[st] = st_action
+            actionp[st] = st_actionp
+            goto[st] = st_goto
+            st += 1
+
+    # -----------------------------------------------------------------------------
+    # write()
+    #
+    # This function writes the LR parsing tables to a file
+    # -----------------------------------------------------------------------------
+
+    def write_table(self, tabmodule, outputdir='', signature=''):
+        if isinstance(tabmodule, types.ModuleType):
+            raise IOError("Won't overwrite existing tabmodule")
+
+        basemodulename = tabmodule.split('.')[-1]
+        filename = os.path.join(outputdir, basemodulename) + '.py'
+        try:
+            f = open(filename, 'w')
+
+            f.write('''
 # %s
 # This file is automatically generated. Do not edit.
+# pylint: disable=W,C,R
+_tabversion = %r
 
-_lr_method = %s
+_lr_method = %r
 
-_lr_signature = %s
-""" % (filename, repr(_lr_method), repr(Signature.digest())))
+_lr_signature = %r
+    ''' % (os.path.basename(filename), __tabversion__, self.lr_method, signature))
 
-        # Change smaller to 0 to go back to original tables
-        smaller = 1
-                
-        # Factor out names to try and make smaller
-        if smaller:
-            items = { }
-        
-            for k,v in _lr_action.items():
-                i = items.get(k[1])
-                if not i:
-                    i = ([],[])
-                    items[k[1]] = i
-                i[0].append(k[0])
-                i[1].append(v)
+            # Change smaller to 0 to go back to original tables
+            smaller = 1
 
-            f.write("\n_lr_action_items = {")
-            for k,v in items.items():
-                f.write("%r:([" % k)
-                for i in v[0]:
-                    f.write("%r," % i)
-                f.write("],[")
-                for i in v[1]:
-                    f.write("%r," % i)
-                           
-                f.write("]),")
-            f.write("}\n")
+            # Factor out names to try and make smaller
+            if smaller:
+                items = {}
 
-            f.write("""
-_lr_action = { }
+                for s, nd in self.lr_action.items():
+                    for name, v in nd.items():
+                        i = items.get(name)
+                        if not i:
+                            i = ([], [])
+                            items[name] = i
+                        i[0].append(s)
+                        i[1].append(v)
+
+                f.write('\n_lr_action_items = {')
+                for k, v in items.items():
+                    f.write('%r:([' % k)
+                    for i in v[0]:
+                        f.write('%r,' % i)
+                    f.write('],[')
+                    for i in v[1]:
+                        f.write('%r,' % i)
+
+                    f.write(']),')
+                f.write('}\n')
+
+                f.write('''
+_lr_action = {}
 for _k, _v in _lr_action_items.items():
    for _x,_y in zip(_v[0],_v[1]):
-       _lr_action[(_x,_k)] = _y
+      if not _x in _lr_action:  _lr_action[_x] = {}
+      _lr_action[_x][_k] = _y
 del _lr_action_items
-""")
-            
-        else:
-            f.write("\n_lr_action = { ")
-            for k,v in _lr_action.items():
-                f.write("(%r,%r):%r," % (k[0],k[1],v))
-            f.write("}\n")
+''')
 
-        if smaller:
-            # Factor out names to try and make smaller
-            items = { }
-        
-            for k,v in _lr_goto.items():
-                i = items.get(k[1])
-                if not i:
-                    i = ([],[])
-                    items[k[1]] = i
-                i[0].append(k[0])
-                i[1].append(v)
-
-            f.write("\n_lr_goto_items = {")
-            for k,v in items.items():
-                f.write("%r:([" % k)
-                for i in v[0]:
-                    f.write("%r," % i)
-                f.write("],[")
-                for i in v[1]:
-                    f.write("%r," % i)
-                           
-                f.write("]),")
-            f.write("}\n")
-
-            f.write("""
-_lr_goto = { }
-for _k, _v in _lr_goto_items.items():
-   for _x,_y in zip(_v[0],_v[1]):
-       _lr_goto[(_x,_k)] = _y
-del _lr_goto_items
-""")
-        else:
-            f.write("\n_lr_goto = { ")
-            for k,v in _lr_goto.items():
-                f.write("(%r,%r):%r," % (k[0],k[1],v))                    
-            f.write("}\n")
-
-        # Write production table
-        f.write("_lr_productions = [\n")
-        for p in Productions:
-            if p:
-                if (p.func):
-                    f.write("  (%r,%d,%r,%r,%d),\n" % (p.name, p.len, p.func.__name__,p.file,p.line))
-                else:
-                    f.write("  (%r,%d,None,None,None),\n" % (p.name, p.len))
             else:
-                f.write("  None,\n")
-        f.write("]\n")
-        
-        f.close()
+                f.write('\n_lr_action = { ')
+                for k, v in self.lr_action.items():
+                    f.write('(%r,%r):%r,' % (k[0], k[1], v))
+                f.write('}\n')
 
-    except IOError as e:
-        print("Unable to create '%s'" % filename)
-        print(e)
+            if smaller:
+                # Factor out names to try and make smaller
+                items = {}
 
-def lr_read_tables(module=tab_module,optimize=0):
-    global _lr_action, _lr_goto, _lr_productions, _lr_method
-    try:
-        exec("import %s as parsetab" % module)
-        
-        if (optimize) or (Signature.digest() == parsetab._lr_signature):
-            _lr_action = parsetab._lr_action
-            _lr_goto   = parsetab._lr_goto
-            _lr_productions = parsetab._lr_productions
-            _lr_method = parsetab._lr_method
-            return 1
+                for s, nd in self.lr_goto.items():
+                    for name, v in nd.items():
+                        i = items.get(name)
+                        if not i:
+                            i = ([], [])
+                            items[name] = i
+                        i[0].append(s)
+                        i[1].append(v)
+
+                f.write('\n_lr_goto_items = {')
+                for k, v in items.items():
+                    f.write('%r:([' % k)
+                    for i in v[0]:
+                        f.write('%r,' % i)
+                    f.write('],[')
+                    for i in v[1]:
+                        f.write('%r,' % i)
+
+                    f.write(']),')
+                f.write('}\n')
+
+                f.write('''
+_lr_goto = {}
+for _k, _v in _lr_goto_items.items():
+   for _x, _y in zip(_v[0], _v[1]):
+       if not _x in _lr_goto: _lr_goto[_x] = {}
+       _lr_goto[_x][_k] = _y
+del _lr_goto_items
+''')
+            else:
+                f.write('\n_lr_goto = { ')
+                for k, v in self.lr_goto.items():
+                    f.write('(%r,%r):%r,' % (k[0], k[1], v))
+                f.write('}\n')
+
+            # Write production table
+            f.write('_lr_productions = [\n')
+            for p in self.lr_productions:
+                if p.func:
+                    f.write('  (%r,%r,%d,%r,%r,%d),\n' % (p.str, p.name, p.len,
+                                                          p.func, os.path.basename(p.file), p.line))
+                else:
+                    f.write('  (%r,%r,%d,None,None,None),\n' % (str(p), p.name, p.len))
+            f.write(']\n')
+            f.close()
+
+        except IOError as e:
+            raise
+
+
+    # -----------------------------------------------------------------------------
+    # pickle_table()
+    #
+    # This function pickles the LR parsing tables to a supplied file object
+    # -----------------------------------------------------------------------------
+
+    def pickle_table(self, filename, signature=''):
+        try:
+            import cPickle as pickle
+        except ImportError:
+            import pickle
+        with open(filename, 'wb') as outf:
+            pickle.dump(__tabversion__, outf, pickle_protocol)
+            pickle.dump(self.lr_method, outf, pickle_protocol)
+            pickle.dump(signature, outf, pickle_protocol)
+            pickle.dump(self.lr_action, outf, pickle_protocol)
+            pickle.dump(self.lr_goto, outf, pickle_protocol)
+
+            outp = []
+            for p in self.lr_productions:
+                if p.func:
+                    outp.append((p.str, p.name, p.len, p.func, os.path.basename(p.file), p.line))
+                else:
+                    outp.append((str(p), p.name, p.len, None, None, None))
+            pickle.dump(outp, outf, pickle_protocol)
+
+# -----------------------------------------------------------------------------
+#                            === INTROSPECTION ===
+#
+# The following functions and classes are used to implement the PLY
+# introspection features followed by the yacc() function itself.
+# -----------------------------------------------------------------------------
+
+# -----------------------------------------------------------------------------
+# get_caller_module_dict()
+#
+# This function returns a dictionary containing all of the symbols defined within
+# a caller further down the call stack.  This is used to get the environment
+# associated with the yacc() call if none was provided.
+# -----------------------------------------------------------------------------
+
+def get_caller_module_dict(levels):
+    f = sys._getframe(levels)
+    ldict = f.f_globals.copy()
+    if f.f_globals != f.f_locals:
+        ldict.update(f.f_locals)
+    return ldict
+
+# -----------------------------------------------------------------------------
+# parse_grammar()
+#
+# This takes a raw grammar rule string and parses it into production data
+# -----------------------------------------------------------------------------
+def parse_grammar(doc, file, line):
+    grammar = []
+    # Split the doc string into lines
+    pstrings = doc.splitlines()
+    lastp = None
+    dline = line
+    for ps in pstrings:
+        dline += 1
+        p = ps.split()
+        if not p:
+            continue
+        try:
+            if p[0] == '|':
+                # This is a continuation of a previous rule
+                if not lastp:
+                    raise SyntaxError("%s:%d: Misplaced '|'" % (file, dline))
+                prodname = lastp
+                syms = p[1:]
+            else:
+                prodname = p[0]
+                lastp = prodname
+                syms   = p[2:]
+                assign = p[1]
+                if assign != ':' and assign != '::=':
+                    raise SyntaxError("%s:%d: Syntax error. Expected ':'" % (file, dline))
+
+            grammar.append((file, dline, prodname, syms))
+        except SyntaxError:
+            raise
+        except Exception:
+            raise SyntaxError('%s:%d: Syntax error in rule %r' % (file, dline, ps.strip()))
+
+    return grammar
+
+# -----------------------------------------------------------------------------
+# ParserReflect()
+#
+# This class represents information extracted for building a parser including
+# start symbol, error function, tokens, precedence list, action functions,
+# etc.
+# -----------------------------------------------------------------------------
+class ParserReflect(object):
+    def __init__(self, pdict, log=None):
+        self.pdict      = pdict
+        self.start      = None
+        self.error_func = None
+        self.tokens     = None
+        self.modules    = set()
+        self.grammar    = []
+        self.error      = False
+
+        if log is None:
+            self.log = PlyLogger(sys.stderr)
         else:
-            return 0
-        
-    except (ImportError,AttributeError):
-        return 0
+            self.log = log
 
+    # Get all of the basic information
+    def get_all(self):
+        self.get_start()
+        self.get_error_func()
+        self.get_tokens()
+        self.get_precedence()
+        self.get_pfunctions()
 
-# Available instance types.  This is used when parsers are defined by a class.
-# In Python3 the InstanceType and ObjectType are no more, they've passed, ceased
-# to be, they are ex-classes along with old-style classes
+    # Validate all of the information
+    def validate_all(self):
+        self.validate_start()
+        self.validate_error_func()
+        self.validate_tokens()
+        self.validate_precedence()
+        self.validate_pfunctions()
+        self.validate_modules()
+        return self.error
 
-try:
-   _INSTANCETYPE = (types.InstanceType, types.ObjectType)
-except AttributeError:
-   _INSTANCETYPE = object
+    # Compute a signature over the grammar
+    def signature(self):
+        parts = []
+        try:
+            if self.start:
+                parts.append(self.start)
+            if self.prec:
+                parts.append(''.join([''.join(p) for p in self.prec]))
+            if self.tokens:
+                parts.append(' '.join(self.tokens))
+            for f in self.pfuncs:
+                if f[3]:
+                    parts.append(f[3])
+        except (TypeError, ValueError):
+            pass
+        return ''.join(parts)
+
+    # -----------------------------------------------------------------------------
+    # validate_modules()
+    #
+    # This method checks to see if there are duplicated p_rulename() functions
+    # in the parser module file.  Without this function, it is really easy for
+    # users to make mistakes by cutting and pasting code fragments (and it's a real
+    # bugger to try and figure out why the resulting parser doesn't work).  Therefore,
+    # we just do a little regular expression pattern matching of def statements
+    # to try and detect duplicates.
+    # -----------------------------------------------------------------------------
+
+    def validate_modules(self):
+        # Match def p_funcname(
+        fre = re.compile(r'\s*def\s+(p_[a-zA-Z_0-9]*)\(')
+
+        for module in self.modules:
+            try:
+                lines, linen = inspect.getsourcelines(module)
+            except IOError:
+                continue
+
+            counthash = {}
+            for linen, line in enumerate(lines):
+                linen += 1
+                m = fre.match(line)
+                if m:
+                    name = m.group(1)
+                    prev = counthash.get(name)
+                    if not prev:
+                        counthash[name] = linen
+                    else:
+                        filename = inspect.getsourcefile(module)
+                        self.log.warning('%s:%d: Function %s redefined. Previously defined on line %d',
+                                         filename, linen, name, prev)
+
+    # Get the start symbol
+    def get_start(self):
+        self.start = self.pdict.get('start')
+
+    # Validate the start symbol
+    def validate_start(self):
+        if self.start is not None:
+            if not isinstance(self.start, string_types):
+                self.log.error("'start' must be a string")
+
+    # Look for error handler
+    def get_error_func(self):
+        self.error_func = self.pdict.get('p_error')
+
+    # Validate the error function
+    def validate_error_func(self):
+        if self.error_func:
+            if isinstance(self.error_func, types.FunctionType):
+                ismethod = 0
+            elif isinstance(self.error_func, types.MethodType):
+                ismethod = 1
+            else:
+                self.log.error("'p_error' defined, but is not a function or method")
+                self.error = True
+                return
+
+            eline = self.error_func.__code__.co_firstlineno
+            efile = self.error_func.__code__.co_filename
+            module = inspect.getmodule(self.error_func)
+            self.modules.add(module)
+
+            argcount = self.error_func.__code__.co_argcount - ismethod
+            if argcount != 1:
+                self.log.error('%s:%d: p_error() requires 1 argument', efile, eline)
+                self.error = True
+
+    # Get the tokens map
+    def get_tokens(self):
+        tokens = self.pdict.get('tokens')
+        if not tokens:
+            self.log.error('No token list is defined')
+            self.error = True
+            return
+
+        if not isinstance(tokens, (list, tuple)):
+            self.log.error('tokens must be a list or tuple')
+            self.error = True
+            return
+
+        if not tokens:
+            self.log.error('tokens is empty')
+            self.error = True
+            return
+
+        self.tokens = sorted(tokens)
+
+    # Validate the tokens
+    def validate_tokens(self):
+        # Validate the tokens.
+        if 'error' in self.tokens:
+            self.log.error("Illegal token name 'error'. Is a reserved word")
+            self.error = True
+            return
+
+        terminals = set()
+        for n in self.tokens:
+            if n in terminals:
+                self.log.warning('Token %r multiply defined', n)
+            terminals.add(n)
+
+    # Get the precedence map (if any)
+    def get_precedence(self):
+        self.prec = self.pdict.get('precedence')
+
+    # Validate and parse the precedence map
+    def validate_precedence(self):
+        preclist = []
+        if self.prec:
+            if not isinstance(self.prec, (list, tuple)):
+                self.log.error('precedence must be a list or tuple')
+                self.error = True
+                return
+            for level, p in enumerate(self.prec):
+                if not isinstance(p, (list, tuple)):
+                    self.log.error('Bad precedence table')
+                    self.error = True
+                    return
+
+                if len(p) < 2:
+                    self.log.error('Malformed precedence entry %s. Must be (assoc, term, ..., term)', p)
+                    self.error = True
+                    return
+                assoc = p[0]
+                if not isinstance(assoc, string_types):
+                    self.log.error('precedence associativity must be a string')
+                    self.error = True
+                    return
+                for term in p[1:]:
+                    if not isinstance(term, string_types):
+                        self.log.error('precedence items must be strings')
+                        self.error = True
+                        return
+                    preclist.append((term, assoc, level+1))
+        self.preclist = preclist
+
+    # Get all p_functions from the grammar
+    def get_pfunctions(self):
+        p_functions = []
+        for name, item in self.pdict.items():
+            if not name.startswith('p_') or name == 'p_error':
+                continue
+            if isinstance(item, (types.FunctionType, types.MethodType)):
+                line = getattr(item, 'co_firstlineno', item.__code__.co_firstlineno)
+                module = inspect.getmodule(item)
+                p_functions.append((line, module, name, item.__doc__))
+
+        # Sort all of the actions by line number; make sure to stringify
+        # modules to make them sortable, since `line` may not uniquely sort all
+        # p functions
+        p_functions.sort(key=lambda p_function: (
+            p_function[0],
+            str(p_function[1]),
+            p_function[2],
+            p_function[3]))
+        self.pfuncs = p_functions
+
+    # Validate all of the p_functions
+    def validate_pfunctions(self):
+        grammar = []
+        # Check for non-empty symbols
+        if len(self.pfuncs) == 0:
+            self.log.error('no rules of the form p_rulename are defined')
+            self.error = True
+            return
+
+        for line, module, name, doc in self.pfuncs:
+            file = inspect.getsourcefile(module)
+            func = self.pdict[name]
+            if isinstance(func, types.MethodType):
+                reqargs = 2
+            else:
+                reqargs = 1
+            if func.__code__.co_argcount > reqargs:
+                self.log.error('%s:%d: Rule %r has too many arguments', file, line, func.__name__)
+                self.error = True
+            elif func.__code__.co_argcount < reqargs:
+                self.log.error('%s:%d: Rule %r requires an argument', file, line, func.__name__)
+                self.error = True
+            elif not func.__doc__:
+                self.log.warning('%s:%d: No documentation string specified in function %r (ignored)',
+                                 file, line, func.__name__)
+            else:
+                try:
+                    parsed_g = parse_grammar(doc, file, line)
+                    for g in parsed_g:
+                        grammar.append((name, g))
+                except SyntaxError as e:
+                    self.log.error(str(e))
+                    self.error = True
+
+                # Looks like a valid grammar rule
+                # Mark the file in which defined.
+                self.modules.add(module)
+
+        # Secondary validation step that looks for p_ definitions that are not functions
+        # or functions that look like they might be grammar rules.
+
+        for n, v in self.pdict.items():
+            if n.startswith('p_') and isinstance(v, (types.FunctionType, types.MethodType)):
+                continue
+            if n.startswith('t_'):
+                continue
+            if n.startswith('p_') and n != 'p_error':
+                self.log.warning('%r not defined as a function', n)
+            if ((isinstance(v, types.FunctionType) and v.__code__.co_argcount == 1) or
+                   (isinstance(v, types.MethodType) and v.__func__.__code__.co_argcount == 2)):
+                if v.__doc__:
+                    try:
+                        doc = v.__doc__.split(' ')
+                        if doc[1] == ':':
+                            self.log.warning('%s:%d: Possible grammar rule %r defined without p_ prefix',
+                                             v.__code__.co_filename, v.__code__.co_firstlineno, n)
+                    except IndexError:
+                        pass
+
+        self.grammar = grammar
 
 # -----------------------------------------------------------------------------
 # yacc(module)
 #
-# Build the parser module
+# Build a parser
 # -----------------------------------------------------------------------------
 
-def yacc(method=default_lr, debug=yaccdebug, module=None, tabmodule=tab_module, start=None, check_recursion=1, optimize=0,write_tables=1,debugfile=debug_file,outputdir=''):
-    global yaccdebug
-    yaccdebug = debug
-    
-    initialize_vars()
-    files = { }
-    error = 0
+def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, start=None,
+         check_recursion=True, optimize=False, write_tables=True, debugfile=debug_file,
+         outputdir=None, debuglog=None, errorlog=None, picklefile=None):
 
+    if tabmodule is None:
+        tabmodule = tab_module
 
-    # Add parsing method to signature
-    Signature.update(util.encode_input(method))
-    
-    # If a "module" parameter was supplied, extract its dictionary.
-    # Note: a module may in fact be an instance as well.
-    
-    if module:
-        # User supplied a module object.
-        if isinstance(module, types.ModuleType):
-            ldict = module.__dict__
-        elif isinstance(module, _INSTANCETYPE):
-            _items = [(k,getattr(module,k)) for k in dir(module)]
-            ldict = { }
-            for i in _items:
-                ldict[i[0]] = i[1]
-        else:
-            raise ValueError("Expected a module")
-        
-    else:
-        # No module given.  We might be able to get information from the caller.
-        # Throw an exception and unwind the traceback to get the globals
-        
-        try:
-            raise RuntimeError
-        except RuntimeError:
-            e,b,t = sys.exc_info()
-            f = t.tb_frame
-            f = f.f_back           # Walk out to our calling function
-            ldict = f.f_globals    # Grab its globals dictionary
-
-    # Add starting symbol to signature
-    if not start:
-        start = ldict.get("start",None)
-    if start:
-        Signature.update(util.encode_input(start))
-
-    # If running in optimized mode.  We're going to
-
-    if (optimize and lr_read_tables(tabmodule,1)):
-        # Read parse table
-        del Productions[:]
-        for p in _lr_productions:
-            if not p:
-                Productions.append(None)
-            else:
-                m = MiniProduction()
-                m.name = p[0]
-                m.len  = p[1]
-                m.file = p[3]
-                m.line = p[4]
-                if p[2]:
-                    m.func = ldict[p[2]]
-                Productions.append(m)
-        
-    else:
-        # Get the tokens map
-        if (module and isinstance(module,_INSTANCETYPE)):
-            tokens = getattr(module,"tokens",None)
-        else:
-            tokens = ldict.get("tokens",None)
-    
-        if not tokens:
-            raise YaccError("module does not define a list 'tokens'")
-        if not (isinstance(tokens,list) or isinstance(tokens,tuple)):
-            raise YaccError("tokens must be a list or tuple.")
-
-        # Check to see if a requires dictionary is defined.
-        requires = ldict.get("require",None)
-        if requires:
-            if not (isinstance(requires,dict)):
-                raise YaccError("require must be a dictionary.")
-
-            for r,v in requires.items():
-                try:
-                    if not (isinstance(v,list)):
-                        raise TypeError
-                    v1 = [x.split(".") for x in v]
-                    Requires[r] = v1
-                except Exception:
-                    print("Invalid specification for rule '%s' in require. Expected a list of strings" % r)
-
-        
-        # Build the dictionary of terminals.  We a record a 0 in the
-        # dictionary to track whether or not a terminal is actually
-        # used in the grammar
-
-        if 'error' in tokens:
-            print("yacc: Illegal token 'error'.  Is a reserved word.")
-            raise YaccError("Illegal token name")
-
-        for n in tokens:
-            if n in Terminals:
-                print("yacc: Warning. Token '%s' multiply defined." % n)
-            Terminals[n] = [ ]
-
-        Terminals['error'] = [ ]
-
-        # Get the precedence map (if any)
-        prec = ldict.get("precedence",None)
-        if prec:
-            if not (isinstance(prec,list) or isinstance(prec,tuple)):
-                raise YaccError("precedence must be a list or tuple.")
-            add_precedence(prec)
-            Signature.update(util.encode_input(repr(prec)))
-
-        for n in tokens:
-            if n not in Precedence:
-                Precedence[n] = ('right',0)         # Default, right associative, 0 precedence
-
-        # Look for error handler
-        ef = ldict.get('p_error',None)
-        if ef:
-            if isinstance(ef,types.FunctionType):
-                ismethod = 0
-            elif isinstance(ef, types.MethodType):
-                ismethod = 1
-            else:
-                raise YaccError("'p_error' defined, but is not a function or method.")
-            eline = ef.__code__.co_firstlineno
-            efile = ef.__code__.co_filename
-            files[efile] = None
-
-            if (ef.__code__.co_argcount != 1+ismethod):
-                raise YaccError("%s:%d: p_error() requires 1 argument." % (efile,eline))
-            global Errorfunc
-            Errorfunc = ef
-        else:
-            print("yacc: Warning. no p_error() function is defined.")
-            
-        # Get the list of built-in functions with p_ prefix
-        symbols = [ldict[f] for f in ldict.keys()
-               if (type(ldict[f]) in (types.FunctionType, types.MethodType) and ldict[f].__name__[:2] == 'p_'
-                   and ldict[f].__name__ != 'p_error')]
-
-        # Check for non-empty symbols
-        if len(symbols) == 0:
-            raise YaccError("no rules of the form p_rulename are defined.")
-    
-        # Sort the symbols by line number
-        symbols.sort(key=lambda x: x.__code__.co_firstlineno)
-
-        # Add all of the symbols to the grammar
-        for f in symbols:
-            if (add_function(f)) < 0:
-                error += 1
-            else:
-                files[f.__code__.co_filename] = None
-
-        # Make a signature of the docstrings
-        for f in symbols:
-            if f.__doc__:
-                Signature.update(util.encode_input(f.__doc__))
-    
-        lr_init_vars()
-
-        if error:
-            raise YaccError("Unable to construct parser.")
-
-        if not lr_read_tables(tabmodule):
-
-            # Validate files
-            for filename in files.keys():
-                if not validate_file(filename):
-                    error = 1
-
-            # Validate dictionary
-            validate_dict(ldict)
-
-            if start and start not in Prodnames:
-                raise YaccError("Bad starting symbol '%s'" % start)
-        
-            augment_grammar(start)    
-            error = verify_productions(cycle_check=check_recursion)
-            otherfunc = [ldict[f] for f in ldict.keys()
-               if (type(f) in (types.FunctionType,types.MethodType) and ldict[f].__name__[:2] != 'p_')]
-
-            if error:
-                raise YaccError("Unable to construct parser.")
-            
-            build_lritems()
-            compute_first1()
-            compute_follow(start)
-        
-            if method in ['SLR','LALR']:
-                lr_parse_table(method)
-            else:
-                raise YaccError("Unknown parsing method '%s'" % method)
-
-            if write_tables:
-                lr_write_tables(tabmodule,outputdir)        
-    
-            if yaccdebug:
-                try:
-                    f = open(os.path.join(outputdir,debugfile),"w")
-                    f.write(_vfc.getvalue())
-                    f.write("\n\n")
-                    f.write(_vf.getvalue())
-                    f.close()
-                except IOError as e:
-                    print("yacc: can't create '%s'" % debugfile,e)
-        
-    # Made it here.   Create a parser object and set up its internal state.
-    # Set global parse() method to bound method of parser object.
-
-    p = Parser("xyzzy")
-    p.productions = Productions
-    p.errorfunc = Errorfunc
-    p.action = _lr_action
-    p.goto   = _lr_goto
-    p.method = _lr_method
-    p.require = Requires
-
+    # Reference to the parsing method of the last built parser
     global parse
-    parse = p.parse
 
-    global parser
-    parser = p
+    # If pickling is enabled, table files are not created
+    if picklefile:
+        write_tables = 0
 
-    # Clean up all of the globals we created
-    if (not optimize):
-        yacc_cleanup()
-    return p
+    if errorlog is None:
+        errorlog = PlyLogger(sys.stderr)
 
-# yacc_cleanup function.  Delete all of the global variables
-# used during table construction
+    # Get the module dictionary used for the parser
+    if module:
+        _items = [(k, getattr(module, k)) for k in dir(module)]
+        pdict = dict(_items)
+        # If no __file__ or __package__ attributes are available, try to obtain them
+        # from the __module__ instead
+        if '__file__' not in pdict:
+            pdict['__file__'] = sys.modules[pdict['__module__']].__file__
+        if '__package__' not in pdict and '__module__' in pdict:
+            if hasattr(sys.modules[pdict['__module__']], '__package__'):
+                pdict['__package__'] = sys.modules[pdict['__module__']].__package__
+    else:
+        pdict = get_caller_module_dict(2)
 
-def yacc_cleanup():
-    global _lr_action, _lr_goto, _lr_method, _lr_goto_cache
-    del _lr_action, _lr_goto, _lr_method, _lr_goto_cache
+    if outputdir is None:
+        # If no output directory is set, the location of the output files
+        # is determined according to the following rules:
+        #     - If tabmodule specifies a package, files go into that package directory
+        #     - Otherwise, files go in the same directory as the specifying module
+        if isinstance(tabmodule, types.ModuleType):
+            srcfile = tabmodule.__file__
+        else:
+            if '.' not in tabmodule:
+                srcfile = pdict['__file__']
+            else:
+                parts = tabmodule.split('.')
+                pkgname = '.'.join(parts[:-1])
+                exec('import %s' % pkgname)
+                srcfile = getattr(sys.modules[pkgname], '__file__', '')
+        outputdir = os.path.dirname(srcfile)
 
-    global Productions, Prodnames, Prodmap, Terminals 
-    global Nonterminals, First, Follow, Precedence, LRitems
-    global Errorfunc, Signature, Requires
-    
-    del Productions, Prodnames, Prodmap, Terminals
-    del Nonterminals, First, Follow, Precedence, LRitems
-    del Errorfunc, Signature, Requires
-    
-    global _vf, _vfc
-    del _vf, _vfc
-    
-    
-# Stub that raises an error if parsing is attempted without first calling yacc()
-def parse(*args,**kwargs):
-    raise YaccError("yacc: No parser built with yacc()")
+    # Determine if the module is package of a package or not.
+    # If so, fix the tabmodule setting so that tables load correctly
+    pkg = pdict.get('__package__')
+    if pkg and isinstance(tabmodule, str):
+        if '.' not in tabmodule:
+            tabmodule = pkg + '.' + tabmodule
 
+
+
+    # Set start symbol if it's specified directly using an argument
+    if start is not None:
+        pdict['start'] = start
+
+    # Collect parser information from the dictionary
+    pinfo = ParserReflect(pdict, log=errorlog)
+    pinfo.get_all()
+
+    if pinfo.error:
+        raise YaccError('Unable to build parser')
+
+    # Check signature against table files (if any)
+    signature = pinfo.signature()
+
+    # Read the tables
+    try:
+        lr = LRTable()
+        if picklefile:
+            read_signature = lr.read_pickle(picklefile)
+        else:
+            read_signature = lr.read_table(tabmodule)
+        if optimize or (read_signature == signature):
+            try:
+                lr.bind_callables(pinfo.pdict)
+                parser = LRParser(lr, pinfo.error_func)
+                parse = parser.parse
+                return parser
+            except Exception as e:
+                errorlog.warning('There was a problem loading the table file: %r', e)
+    except VersionError as e:
+        errorlog.warning(str(e))
+    except ImportError:
+        pass
+
+    if debuglog is None:
+        if debug:
+            try:
+                debuglog = PlyLogger(open(os.path.join(outputdir, debugfile), 'w'))
+            except IOError as e:
+                errorlog.warning("Couldn't open %r. %s" % (debugfile, e))
+                debuglog = NullLogger()
+        else:
+            debuglog = NullLogger()
+
+    debuglog.info('Created by PLY version %s (http://www.dabeaz.com/ply)', __version__)
+
+    errors = False
+
+    # Validate the parser information
+    if pinfo.validate_all():
+        raise YaccError('Unable to build parser')
+
+    if not pinfo.error_func:
+        errorlog.warning('no p_error() function is defined')
+
+    # Create a grammar object
+    grammar = Grammar(pinfo.tokens)
+
+    # Set precedence level for terminals
+    for term, assoc, level in pinfo.preclist:
+        try:
+            grammar.set_precedence(term, assoc, level)
+        except GrammarError as e:
+            errorlog.warning('%s', e)
+
+    # Add productions to the grammar
+    for funcname, gram in pinfo.grammar:
+        file, line, prodname, syms = gram
+        try:
+            grammar.add_production(prodname, syms, funcname, file, line)
+        except GrammarError as e:
+            errorlog.error('%s', e)
+            errors = True
+
+    # Set the grammar start symbols
+    try:
+        if start is None:
+            grammar.set_start(pinfo.start)
+        else:
+            grammar.set_start(start)
+    except GrammarError as e:
+        errorlog.error(str(e))
+        errors = True
+
+    if errors:
+        raise YaccError('Unable to build parser')
+
+    # Verify the grammar structure
+    undefined_symbols = grammar.undefined_symbols()
+    for sym, prod in undefined_symbols:
+        errorlog.error('%s:%d: Symbol %r used, but not defined as a token or a rule', prod.file, prod.line, sym)
+        errors = True
+
+    unused_terminals = grammar.unused_terminals()
+    if unused_terminals:
+        debuglog.info('')
+        debuglog.info('Unused terminals:')
+        debuglog.info('')
+        for term in unused_terminals:
+            errorlog.warning('Token %r defined, but not used', term)
+            debuglog.info('    %s', term)
+
+    # Print out all productions to the debug log
+    if debug:
+        debuglog.info('')
+        debuglog.info('Grammar')
+        debuglog.info('')
+        for n, p in enumerate(grammar.Productions):
+            debuglog.info('Rule %-5d %s', n, p)
+
+    # Find unused non-terminals
+    unused_rules = grammar.unused_rules()
+    for prod in unused_rules:
+        errorlog.warning('%s:%d: Rule %r defined, but not used', prod.file, prod.line, prod.name)
+
+    if len(unused_terminals) == 1:
+        errorlog.warning('There is 1 unused token')
+    if len(unused_terminals) > 1:
+        errorlog.warning('There are %d unused tokens', len(unused_terminals))
+
+    if len(unused_rules) == 1:
+        errorlog.warning('There is 1 unused rule')
+    if len(unused_rules) > 1:
+        errorlog.warning('There are %d unused rules', len(unused_rules))
+
+    if debug:
+        debuglog.info('')
+        debuglog.info('Terminals, with rules where they appear')
+        debuglog.info('')
+        terms = list(grammar.Terminals)
+        terms.sort()
+        for term in terms:
+            debuglog.info('%-20s : %s', term, ' '.join([str(s) for s in grammar.Terminals[term]]))
+
+        debuglog.info('')
+        debuglog.info('Nonterminals, with rules where they appear')
+        debuglog.info('')
+        nonterms = list(grammar.Nonterminals)
+        nonterms.sort()
+        for nonterm in nonterms:
+            debuglog.info('%-20s : %s', nonterm, ' '.join([str(s) for s in grammar.Nonterminals[nonterm]]))
+        debuglog.info('')
+
+    if check_recursion:
+        unreachable = grammar.find_unreachable()
+        for u in unreachable:
+            errorlog.warning('Symbol %r is unreachable', u)
+
+        infinite = grammar.infinite_cycles()
+        for inf in infinite:
+            errorlog.error('Infinite recursion detected for symbol %r', inf)
+            errors = True
+
+    unused_prec = grammar.unused_precedence()
+    for term, assoc in unused_prec:
+        errorlog.error('Precedence rule %r defined for unknown symbol %r', assoc, term)
+        errors = True
+
+    if errors:
+        raise YaccError('Unable to build parser')
+
+    # Run the LRGeneratedTable on the grammar
+    if debug:
+        errorlog.debug('Generating %s tables', method)
+
+    lr = LRGeneratedTable(grammar, method, debuglog)
+
+    if debug:
+        num_sr = len(lr.sr_conflicts)
+
+        # Report shift/reduce and reduce/reduce conflicts
+        if num_sr == 1:
+            errorlog.warning('1 shift/reduce conflict')
+        elif num_sr > 1:
+            errorlog.warning('%d shift/reduce conflicts', num_sr)
+
+        num_rr = len(lr.rr_conflicts)
+        if num_rr == 1:
+            errorlog.warning('1 reduce/reduce conflict')
+        elif num_rr > 1:
+            errorlog.warning('%d reduce/reduce conflicts', num_rr)
+
+    # Write out conflicts to the output file
+    if debug and (lr.sr_conflicts or lr.rr_conflicts):
+        debuglog.warning('')
+        debuglog.warning('Conflicts:')
+        debuglog.warning('')
+
+        for state, tok, resolution in lr.sr_conflicts:
+            debuglog.warning('shift/reduce conflict for %s in state %d resolved as %s',  tok, state, resolution)
+
+        already_reported = set()
+        for state, rule, rejected in lr.rr_conflicts:
+            if (state, id(rule), id(rejected)) in already_reported:
+                continue
+            debuglog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule)
+            debuglog.warning('rejected rule (%s) in state %d', rejected, state)
+            errorlog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule)
+            errorlog.warning('rejected rule (%s) in state %d', rejected, state)
+            already_reported.add((state, id(rule), id(rejected)))
+
+        warned_never = []
+        for state, rule, rejected in lr.rr_conflicts:
+            if not rejected.reduced and (rejected not in warned_never):
+                debuglog.warning('Rule (%s) is never reduced', rejected)
+                errorlog.warning('Rule (%s) is never reduced', rejected)
+                warned_never.append(rejected)
+
+    # Write the table file if requested
+    if write_tables:
+        try:
+            lr.write_table(tabmodule, outputdir, signature)
+            if tabmodule in sys.modules:
+                del sys.modules[tabmodule]
+        except IOError as e:
+            errorlog.warning("Couldn't create %r. %s" % (tabmodule, e))
+
+    # Write a pickled version of the tables
+    if picklefile:
+        try:
+            lr.pickle_table(picklefile, signature)
+        except IOError as e:
+            errorlog.warning("Couldn't create %r. %s" % (picklefile, e))
+
+    # Build the parser
+    lr.bind_callables(pinfo.pdict)
+    parser = LRParser(lr, pinfo.error_func)
+
+    parse = parser.parse
+    return parser
diff --git a/python/sepolicy/sepolicy/__init__.py b/python/sepolicy/sepolicy/__init__.py
index fbeb731..6f72947 100644
--- a/python/sepolicy/sepolicy/__init__.py
+++ b/python/sepolicy/sepolicy/__init__.py
@@ -119,16 +119,34 @@
 all_transitions = None
 
 
+def policy_sortkey(policy_path):
+    # Parse the extension of a policy path which looks like .../policy/policy.31
+    extension = policy_path.rsplit('/policy.', 1)[1]
+    try:
+        return int(extension), policy_path
+    except ValueError:
+        # Fallback with sorting on the full path
+        return 0, policy_path
+
 def get_installed_policy(root="/"):
     try:
         path = root + selinux.selinux_binary_policy_path()
         policies = glob.glob("%s.*" % path)
-        policies.sort()
+        policies.sort(key=policy_sortkey)
         return policies[-1]
     except:
         pass
     raise ValueError(_("No SELinux Policy installed"))
 
+def get_store_policy(store):
+    """Get the path to the policy file located in the given store name"""
+    policies = glob.glob("%s%s/policy/policy.*" %
+                         (selinux.selinux_path(), store))
+    if not policies:
+        return None
+    # Return the policy with the higher version number
+    policies.sort(key=policy_sortkey)
+    return policies[-1]
 
 def policy(policy_file):
     global all_domains
@@ -156,6 +174,11 @@
     except:
         raise ValueError(_("Failed to read %s policy file") % policy_file)
 
+def load_store_policy(store):
+    policy_file = get_store_policy(store)
+    if not policy_file:
+        return None
+    policy(policy_file)
 
 try:
     policy_file = get_installed_policy()
diff --git a/scripts/run-flake8 b/scripts/run-flake8
index 207edd2..6ad029f 100755
--- a/scripts/run-flake8
+++ b/scripts/run-flake8
@@ -17,10 +17,8 @@
 IGNORE_LIST="$IGNORE_LIST,E711" # comparison to None should be 'if cond is not None:'
 IGNORE_LIST="$IGNORE_LIST,E712" # comparison to False should be 'if cond is False:' or 'if not cond:'
 IGNORE_LIST="$IGNORE_LIST,E722" # do not use bare 'except'
-IGNORE_LIST="$IGNORE_LIST,E999" # TabError: inconsistent use of tabs and spaces in indentation
 
 IGNORE_LIST="$IGNORE_LIST,F401" # module imported but unused
-IGNORE_LIST="$IGNORE_LIST,F812" # list comprehension redefines 'f', in lex.py and yacc.py
 IGNORE_LIST="$IGNORE_LIST,F841" # local variable '...' is assigned to but never used
 
 
@@ -75,4 +73,4 @@
 IGNORE_LIST="$IGNORE_LIST,F811" # redefinition of unused ...
 
 
-exec flake8 --max-line-length=120 --builtins='_,unicode,lextab,parsetab' --ignore=",$IGNORE_LIST" "$@"
+exec flake8 --max-line-length=120 --builtins='_,basestring,unicode' --ignore=",$IGNORE_LIST" "$@"