Refactor the kernel update scripts.
The previous versions of the scripts did a lot of redundant changes
and were hard to follow.
I rewrote most of update_all.py so that it's clear about what's going on.
I updated clean_header.py to change the cleanupFile function so that
there is no magic about where the destination file is going to wind up.
Now the caller specifies the final location.
I updated utils.py so that if you are trying to do an update in one
location, but your lunch target is from another location, it causes
an error.
Bug: 35726570
Change-Id: Ic5a44d90c2774a627eecde34c0c403bc925a497c
Test: Ran the updater and verified it works properly.
Test: Verified that doing an update in one tree to another tree
Test: fails.
diff --git a/libc/kernel/tools/clean_header.py b/libc/kernel/tools/clean_header.py
index 99f4c7f..f474f74 100755
--- a/libc/kernel/tools/clean_header.py
+++ b/libc/kernel/tools/clean_header.py
@@ -79,55 +79,33 @@
sys.stderr.write("warning: " + msg)
-def cleanupFile(dst_dir, src_dir, rel_path, no_update = True):
+def cleanupFile(dst_file, src_file, rel_path, no_update = True):
"""reads an original header and perform the cleanup operation on it
this functions returns the destination path and the clean header
as a single string"""
- # check the header path
- full_path = os.path.join(src_dir, rel_path)
-
- if not os.path.exists(full_path):
- print_error(no_update, "file does not exist: '%s'\n" % full_path)
+ # Check the header path
+ if not os.path.exists(src_file):
+ print_error(no_update, "'%s' does not exist\n" % src_file)
return None, None
- if not os.path.isfile(full_path):
- print_error(no_update, "path is not a file: '%s'\n" % full_path)
+ if not os.path.isfile(src_file):
+ print_error(no_update, "'%s' is not a file\n" % src_file)
return None, None
- # convert into destination path, extracting architecture if needed
- # and the corresponding list of known static functions
- #
+ # Extract the architecture if found.
arch = None
statics = kernel_known_generic_statics
- m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", rel_path)
- if m and m.group(1) != 'generic':
- dst_path = "arch-%s/asm/%s" % m.groups()
- arch = m.group(1)
- statics = statics.union(kernel_known_statics.get(arch, set()))
- else:
- # process headers under the uapi directory
- # note the "asm" level has been explicitly added in the original
- # kernel header tree for architectural-dependent uapi headers
- m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", rel_path)
- if m_uapi:
- dst_path = rel_path
- m_uapi_arch = re.match(r"asm-([\w\d_\+\.\-]+)", m_uapi.group(2))
- if m_uapi_arch and m_uapi_arch.group(1) != 'generic':
- arch = m_uapi_arch.group(1)
- statics = statics.union(kernel_known_statics.get(arch, set()))
- # common headers (ie non-asm and non-uapi)
- else:
- dst_path = os.path.join("android", rel_path)
+ m = re.search(r"(^|/)asm-([\w\d_\+\.\-]+)/.*", rel_path)
+ if m and m.group(2) != 'generic':
+ arch = m.group(2)
+ statics = statics.union(kernel_known_statics.get(arch, set()))
- dst_path = os.path.join(dst_dir, dst_path)
-
- # now, let's parse the file
- #
+ # Now, let's parse the file.
parser = cpp.BlockParser()
- blocks = parser.parseFile(full_path)
+ blocks = parser.parseFile(src_file)
if not parser.parsed:
- print_error(no_update, "can't parse '%s'%" % full_path)
- return None, None
+ print_error(no_update, "Can't parse '%s'" % src_file)
+ return None
macros = kernel_known_macros.copy()
if arch and arch in kernel_default_arch_macros:
@@ -145,7 +123,7 @@
out = StringOutput()
out.write(kernel_disclaimer)
blocks.writeWithWarning(out, kernel_warning, 4)
- return dst_path, out.get()
+ return out.get()
if __name__ == "__main__":
@@ -194,22 +172,26 @@
if no_update:
for path in args:
- dst_path, newdata = cleanupFile(dst_dir, src_dir, path)
- print newdata
+ dst_file = os.path.join(dst_dir, path)
+ src_file = os.path.join(src_dir, path)
+ new_data = cleanupFile(dst_file, src_file, path)
+ print new_data
sys.exit(0)
- # now let's update our files.
+ # Now let's update our files.
b = BatchFileUpdater()
for path in args:
- dst_path, newdata = cleanupFile(dst_dir, src_dir, path, no_update)
- if not dst_path:
+ dst_file = os.path.join(dst_dir, path)
+ src_file = os.path.join(src_dir, path)
+ new_data = cleanupFile(dst_file, src_file, path, no_update)
+ if not new_data:
continue
- b.readFile(dst_path)
- r = b.editFile(dst_path, newdata)
+ b.readFile(path)
+ r = b.editFile(path, new_data)
if r == 0:
r = "unchanged"
elif r == 1:
@@ -217,7 +199,7 @@
else:
r = "added"
- print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, dst_path, r)
+ print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, path, r)
b.updateGitFiles()
diff --git a/libc/kernel/tools/update_all.py b/libc/kernel/tools/update_all.py
index 5031168..e5a07f8 100755
--- a/libc/kernel/tools/update_all.py
+++ b/libc/kernel/tools/update_all.py
@@ -24,105 +24,77 @@
""" % { "progname" : os.path.basename(sys.argv[0]) }
sys.exit(0)
+def processFiles(updater, original_dir, modified_dir, src_rel_dir, update_rel_dir):
+ # Delete the old headers before updating to the new headers.
+ update_dir = os.path.join(get_kernel_dir(), update_rel_dir)
+ shutil.rmtree(update_dir)
+ os.mkdir(update_dir, 0755)
+
+ src_dir = os.path.normpath(os.path.join(original_dir, src_rel_dir))
+ src_dir_len = len(src_dir) + 1
+ mod_src_dir = os.path.join(modified_dir, src_rel_dir)
+ update_dir = os.path.join(get_kernel_dir(), update_rel_dir)
+
+ kernel_dir = get_kernel_dir()
+ for root, _, files in os.walk(src_dir):
+ for file in sorted(files):
+ _, ext = os.path.splitext(file)
+ if ext != ".h":
+ continue
+ src_file = os.path.normpath(os.path.join(root, file))
+ rel_path = src_file[src_dir_len:]
+ # Check to see if there is a modified header to use instead.
+ if os.path.exists(os.path.join(mod_src_dir, rel_path)):
+ src_file = os.path.join(mod_src_dir, rel_path)
+ src_str = os.path.join("<modified>", src_rel_dir, rel_path)
+ else:
+ src_str = os.path.join("<original>", src_rel_dir, rel_path)
+ dst_file = os.path.join(update_dir, rel_path)
+ new_data = clean_header.cleanupFile(dst_file, src_file, rel_path)
+ if not new_data:
+ continue
+ updater.readFile(dst_file)
+ ret_val = updater.editFile(dst_file, new_data)
+ if ret_val == 0:
+ state = "unchanged"
+ elif ret_val == 1:
+ state = "edited"
+ else:
+ state = "added"
+ update_path = os.path.join(update_rel_dir, rel_path)
+ print "cleaning %s -> %s (%s)" % (src_str, update_path, state)
+
try:
optlist, args = getopt.getopt(sys.argv[1:], '')
except:
- # unrecognized option
+ # Unrecognized option
sys.stderr.write("error: unrecognized option\n")
usage()
if len(optlist) > 0 or len(args) > 2:
usage()
-modified_dir = get_kernel_headers_modified_dir()
-if len(args) == 1 or len(args) == 2:
+if len(args) > 0:
original_dir = args[0]
- if not os.path.isdir(original_dir):
- panic("Not a directory: %s\n" % original_dir)
-
- if len(args) == 2:
- modified_dir = args[1]
- if not os.path.isdir(modified_dir):
- panic("Not a directory: %s\n" % modified_dir)
else:
original_dir = get_kernel_headers_original_dir()
- if not os.path.isdir(original_dir):
- panic("Missing directory, please specify one through command-line: %s\n" % original_dir)
+
+if len(args) > 1:
+ modified_dir = args[1]
+else:
+ modified_dir = get_kernel_headers_modified_dir()
+
+if not os.path.isdir(original_dir):
+ panic("The kernel directory %s is not a directory\n" % original_dir)
if not os.path.isdir(modified_dir):
- modified_dir = None
+ panic("The kernel modified directory %s is not a directory\n" % modified_dir)
-# Find all source files in 'original'.
-sources = dict()
-original_dir = os.path.normpath(original_dir)
-original_dir_len = len(original_dir) + 1
-for root, _, files in os.walk(original_dir):
- for file in files:
- _, ext = os.path.splitext(file)
- if ext == ".h":
- rel_path = os.path.normpath(os.path.join(root, file))
- rel_path = rel_path[original_dir_len:]
- # Check to see if there is a modified header to use instead.
- if modified_dir and os.path.exists(os.path.join(modified_dir, rel_path)):
- sources[rel_path] = False
- else:
- sources[rel_path] = True
+updater = BatchFileUpdater()
+# Process the original uapi headers first.
+processFiles(updater, original_dir, modified_dir, "uapi", "uapi"),
+# Now process the special files.
+processFiles(updater, original_dir, modified_dir, "scsi", os.path.join("android", "scsi"))
-b = BatchFileUpdater()
-
-kernel_dir = get_kernel_dir()
-for arch in kernel_archs:
- b.readDir(os.path.join(kernel_dir, "arch-%s" % arch))
-
-b.readDir(os.path.join(kernel_dir, "android"))
-
-# Delete the old uapi headers before updating to handle headers that
-# get moved/deleted.
-uapi_dir = os.path.join(get_kernel_dir(), "uapi")
-shutil.rmtree(uapi_dir)
-os.mkdir(uapi_dir, 0755)
-
-oldlen = 120
-android_root_len = len(get_android_root()) + 1
-for rel_path in sorted(sources):
- if sources[rel_path]:
- src_dir = original_dir
- src_str = "<original>/"
- else:
- src_dir = modified_dir
- src_str = "<modified>/"
- dst_path, newdata = clean_header.cleanupFile(kernel_dir, src_dir, rel_path)
- if not dst_path:
- continue
-
- dst_path = os.path.join(kernel_dir, dst_path)
- b.readFile(dst_path)
- r = b.editFile(dst_path, newdata)
- if r == 0:
- state = "unchanged"
- elif r == 1:
- state = "edited"
- else:
- state = "added"
-
- # dst_path is guaranteed to include android root.
- rel_dst_path = dst_path[android_root_len:]
- str = "cleaning: %-*s -> %-*s (%s)" % (35, src_str + rel_path, 35, rel_dst_path, state)
- if sys.stdout.isatty():
- print "%-*s" % (oldlen, str),
- if (r == 0):
- print "\r",
- else:
- print "\n",
- oldlen = 0
- else:
- print str
-
- oldlen = len(str)
-
-print "%-*s" % (oldlen, "Done!")
-
-b.updateGitFiles()
-
-sys.exit(0)
+updater.updateGitFiles()
diff --git a/libc/kernel/tools/utils.py b/libc/kernel/tools/utils.py
index e2cc9ce..1b06b1b 100644
--- a/libc/kernel/tools/utils.py
+++ b/libc/kernel/tools/utils.py
@@ -31,8 +31,14 @@
def get_android_root():
if "ANDROID_BUILD_TOP" in os.environ:
+ # Verify that the current directory is in the root.
+ # If not, then print an error.
+ cwd = os.getcwd()
+ root = os.environ["ANDROID_BUILD_TOP"]
+ if len(cwd) < len(root) or not root == cwd[:len(root)]:
+ panic("Not in android tree pointed at by ANDROID_BUILD_TOP (%s)\n" % root)
return os.environ["ANDROID_BUILD_TOP"]
- panic("Unable to find root of tree, did you forget to lunch a target?")
+ panic("Unable to find root of tree, did you forget to lunch a target?\n")
class StringOutput:
