move File and Difference classes into common script
This makes them accessible from device-specific extensions (so they
can be used to send radio images as binary patches, for instance).
Change-Id: I2f2174b93b4265abf9400f9e5a0982caca0771e9
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index 6bb37ae..46cef11 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -12,16 +12,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import copy
import errno
import getopt
import getpass
import imp
import os
import re
+import sha
import shutil
import subprocess
import sys
import tempfile
+import threading
+import time
import zipfile
# missing in Python 2.4 and before
@@ -547,3 +551,124 @@
this is used to install the image for the device's baseband
processor."""
return self._DoCall("IncrementalOTA_InstallEnd")
+
+class File(object):
+ def __init__(self, name, data):
+ self.name = name
+ self.data = data
+ self.size = len(data)
+ self.sha1 = sha.sha(data).hexdigest()
+
+ def WriteToTemp(self):
+ t = tempfile.NamedTemporaryFile()
+ t.write(self.data)
+ t.flush()
+ return t
+
+ def AddToZip(self, z):
+ ZipWriteStr(z, self.name, self.data)
+
+DIFF_PROGRAM_BY_EXT = {
+ ".gz" : "imgdiff",
+ ".zip" : ["imgdiff", "-z"],
+ ".jar" : ["imgdiff", "-z"],
+ ".apk" : ["imgdiff", "-z"],
+ ".img" : "imgdiff",
+ }
+
+class Difference(object):
+ def __init__(self, tf, sf):
+ self.tf = tf
+ self.sf = sf
+ self.patch = None
+
+ def ComputePatch(self):
+ """Compute the patch (as a string of data) needed to turn sf into
+ tf. Returns the same tuple as GetPatch()."""
+
+ tf = self.tf
+ sf = self.sf
+
+ ext = os.path.splitext(tf.name)[1]
+ diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
+
+ ttemp = tf.WriteToTemp()
+ stemp = sf.WriteToTemp()
+
+ ext = os.path.splitext(tf.name)[1]
+
+ try:
+ ptemp = tempfile.NamedTemporaryFile()
+ if isinstance(diff_program, list):
+ cmd = copy.copy(diff_program)
+ else:
+ cmd = [diff_program]
+ cmd.append(stemp.name)
+ cmd.append(ttemp.name)
+ cmd.append(ptemp.name)
+ p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ _, err = p.communicate()
+ if err or p.returncode != 0:
+ print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
+ return None
+ diff = ptemp.read()
+ finally:
+ ptemp.close()
+ stemp.close()
+ ttemp.close()
+
+ self.patch = diff
+ return self.tf, self.sf, self.patch
+
+
+ def GetPatch(self):
+ """Return a tuple (target_file, source_file, patch_data).
+ patch_data may be None if ComputePatch hasn't been called, or if
+ computing the patch failed."""
+ return self.tf, self.sf, self.patch
+
+
+def ComputeDifferences(diffs):
+ """Call ComputePatch on all the Difference objects in 'diffs'."""
+ print len(diffs), "diffs to compute"
+
+ # Do the largest files first, to try and reduce the long-pole effect.
+ by_size = [(i.tf.size, i) for i in diffs]
+ by_size.sort(reverse=True)
+ by_size = [i[1] for i in by_size]
+
+ lock = threading.Lock()
+ diff_iter = iter(by_size) # accessed under lock
+
+ def worker():
+ try:
+ lock.acquire()
+ for d in diff_iter:
+ lock.release()
+ start = time.time()
+ d.ComputePatch()
+ dur = time.time() - start
+ lock.acquire()
+
+ tf, sf, patch = d.GetPatch()
+ if sf.name == tf.name:
+ name = tf.name
+ else:
+ name = "%s (%s)" % (tf.name, sf.name)
+ if patch is None:
+ print "patching failed! %s" % (name,)
+ else:
+ print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
+ dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
+ lock.release()
+ except Exception, e:
+ print e
+ raise
+
+ # start worker threads; wait for them all to finish.
+ threads = [threading.Thread(target=worker)
+ for i in range(OPTIONS.worker_threads)]
+ for th in threads:
+ th.start()
+ while threads:
+ threads.pop().join()
diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files
index 29911bb..ad7e0e4 100755
--- a/tools/releasetools/ota_from_target_files
+++ b/tools/releasetools/ota_from_target_files
@@ -59,7 +59,6 @@
import sha
import subprocess
import tempfile
-import threading
import time
import zipfile
@@ -308,7 +307,7 @@
executable.
"""
- d = Difference(recovery_img, boot_img)
+ d = common.Difference(recovery_img, boot_img)
_, _, patch = d.ComputePatch()
common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
Item.Get("system/recovery-from-boot.p", dir=False)
@@ -374,9 +373,9 @@
symlinks = CopySystemFiles(input_zip, output_zip)
script.MakeSymlinks(symlinks)
- boot_img = File("boot.img", common.BuildBootableImage(
+ boot_img = common.File("boot.img", common.BuildBootableImage(
os.path.join(OPTIONS.input_tmp, "BOOT")))
- recovery_img = File("recovery.img", common.BuildBootableImage(
+ recovery_img = common.File("recovery.img", common.BuildBootableImage(
os.path.join(OPTIONS.input_tmp, "RECOVERY")))
MakeRecoveryPatch(output_zip, recovery_img, boot_img)
@@ -407,21 +406,6 @@
for kv in sorted(metadata.iteritems())]))
-class File(object):
- def __init__(self, name, data):
- self.name = name
- self.data = data
- self.size = len(data)
- self.sha1 = sha.sha(data).hexdigest()
-
- def WriteToTemp(self):
- t = tempfile.NamedTemporaryFile()
- t.write(self.data)
- t.flush()
- return t
-
- def AddToZip(self, z):
- common.ZipWriteStr(z, self.name, self.data)
def LoadSystemFiles(z):
@@ -432,117 +416,10 @@
if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
fn = "system/" + info.filename[7:]
data = z.read(info.filename)
- out[fn] = File(fn, data)
+ out[fn] = common.File(fn, data)
return out
-DIFF_PROGRAM_BY_EXT = {
- ".gz" : "imgdiff",
- ".zip" : ["imgdiff", "-z"],
- ".jar" : ["imgdiff", "-z"],
- ".apk" : ["imgdiff", "-z"],
- ".img" : "imgdiff",
- }
-
-
-class Difference(object):
- def __init__(self, tf, sf):
- self.tf = tf
- self.sf = sf
- self.patch = None
-
- def ComputePatch(self):
- """Compute the patch (as a string of data) needed to turn sf into
- tf. Returns the same tuple as GetPatch()."""
-
- tf = self.tf
- sf = self.sf
-
- ext = os.path.splitext(tf.name)[1]
- diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
-
- ttemp = tf.WriteToTemp()
- stemp = sf.WriteToTemp()
-
- ext = os.path.splitext(tf.name)[1]
-
- try:
- ptemp = tempfile.NamedTemporaryFile()
- if isinstance(diff_program, list):
- cmd = copy.copy(diff_program)
- else:
- cmd = [diff_program]
- cmd.append(stemp.name)
- cmd.append(ttemp.name)
- cmd.append(ptemp.name)
- p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- _, err = p.communicate()
- if err or p.returncode != 0:
- print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
- return None
- diff = ptemp.read()
- finally:
- ptemp.close()
- stemp.close()
- ttemp.close()
-
- self.patch = diff
- return self.tf, self.sf, self.patch
-
-
- def GetPatch(self):
- """Return a tuple (target_file, source_file, patch_data).
- patch_data may be None if ComputePatch hasn't been called, or if
- computing the patch failed."""
- return self.tf, self.sf, self.patch
-
-
-def ComputeDifferences(diffs):
- """Call ComputePatch on all the Difference objects in 'diffs'."""
- print len(diffs), "diffs to compute"
-
- # Do the largest files first, to try and reduce the long-pole effect.
- by_size = [(i.tf.size, i) for i in diffs]
- by_size.sort(reverse=True)
- by_size = [i[1] for i in by_size]
-
- lock = threading.Lock()
- diff_iter = iter(by_size) # accessed under lock
-
- def worker():
- try:
- lock.acquire()
- for d in diff_iter:
- lock.release()
- start = time.time()
- d.ComputePatch()
- dur = time.time() - start
- lock.acquire()
-
- tf, sf, patch = d.GetPatch()
- if sf.name == tf.name:
- name = tf.name
- else:
- name = "%s (%s)" % (tf.name, sf.name)
- if patch is None:
- print "patching failed! %s" % (name,)
- else:
- print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
- dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
- lock.release()
- except Exception, e:
- print e
- raise
-
- # start worker threads; wait for them all to finish.
- threads = [threading.Thread(target=worker)
- for i in range(OPTIONS.worker_threads)]
- for th in threads:
- th.start()
- while threads:
- threads.pop().join()
-
-
def GetBuildProp(property, z):
"""Return the fingerprint of the build of a given target-files
ZipFile object."""
@@ -616,12 +493,12 @@
verbatim_targets.append((fn, tf.size))
elif tf.sha1 != sf.sha1:
# File is different; consider sending as a patch
- diffs.append(Difference(tf, sf))
+ diffs.append(common.Difference(tf, sf))
else:
# Target file identical to source.
pass
- ComputeDifferences(diffs)
+ common.ComputeDifferences(diffs)
for diff in diffs:
tf, sf, d = diff.GetPatch()
@@ -642,20 +519,20 @@
script.Mount("MTD", "system", "/system")
script.AssertSomeFingerprint(source_fp, target_fp)
- source_boot = File("/tmp/boot.img",
- common.BuildBootableImage(
- os.path.join(OPTIONS.source_tmp, "BOOT")))
- target_boot = File("/tmp/boot.img",
- common.BuildBootableImage(
- os.path.join(OPTIONS.target_tmp, "BOOT")))
+ source_boot = common.File("/tmp/boot.img",
+ common.BuildBootableImage(
+ os.path.join(OPTIONS.source_tmp, "BOOT")))
+ target_boot = common.File("/tmp/boot.img",
+ common.BuildBootableImage(
+ os.path.join(OPTIONS.target_tmp, "BOOT")))
updating_boot = (source_boot.data != target_boot.data)
- source_recovery = File("system/recovery.img",
- common.BuildBootableImage(
- os.path.join(OPTIONS.source_tmp, "RECOVERY")))
- target_recovery = File("system/recovery.img",
- common.BuildBootableImage(
- os.path.join(OPTIONS.target_tmp, "RECOVERY")))
+ source_recovery = common.File("system/recovery.img",
+ common.BuildBootableImage(
+ os.path.join(OPTIONS.source_tmp, "RECOVERY")))
+ target_recovery = common.File("system/recovery.img",
+ common.BuildBootableImage(
+ os.path.join(OPTIONS.target_tmp, "RECOVERY")))
updating_recovery = (source_recovery.data != target_recovery.data)
# Here's how we divide up the progress bar:
@@ -681,7 +558,7 @@
script.SetProgress(so_far / total_verify_size)
if updating_boot:
- d = Difference(target_boot, source_boot)
+ d = common.Difference(target_boot, source_boot)
_, _, d = d.ComputePatch()
print "boot target: %d source: %d diff: %d" % (
target_boot.size, source_boot.size, len(d))
