ipkg-utils: Merge in tweaked version of patches from #1504 (from Paul Sokolovsky) to massively speedup ipkg-make-index

8 files changed, 420 insertions, 71 deletions

diff --git a/packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty b/packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty
deleted file mode 100644
index e69de29bb2..0000000000
--- a/packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty
+++ /dev/null
diff --git a/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb b/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
index 03a381387e..a0964d65b3 100644
--- a/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
+++ b/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
@@ -1,15 +1,14 @@
 require ipkg-utils_${PV}.bb
-SRC_URI += "file://ipkg-utils-fix.patch;patch=1 \
-            file://ipkg-py-sane-vercompare.patch;patch=1"
 
 RDEPENDS = ""
-PR = "r9"
+PR = "r11"
 
 inherit native
 
 # Avoid circular dependencies from package_ipk.bbclass
 PACKAGES = ""
 FILESDIR = "${@os.path.dirname(bb.data.getVar('FILE',d,1))}/ipkg-utils"
+INSTALL += "arfile.py"
 
 do_stage() {
         for i in ${INSTALL}; do
diff --git a/packages/ipkg-utils/ipkg-utils/index_speedup.patch b/packages/ipkg-utils/ipkg-utils/index_speedup.patch
deleted file mode 100644
index bef28a9df1..0000000000
--- a/packages/ipkg-utils/ipkg-utils/index_speedup.patch
+++ /dev/null
@@ -1,53 +0,0 @@
-Index: ipkg-utils/ipkg-make-index
-===================================================================
---- ipkg-utils.orig/ipkg-make-index	2005-03-20 18:10:54.000000000 +0000
-+++ ipkg-utils/ipkg-make-index	2006-07-24 09:18:16.000000000 +0100
-@@ -37,6 +37,21 @@
-      if os.path.exists(pkg_dir + "/" + filename + ".asc"):
-           os.rename(pkg_dir + "/" + filename + ".asc", locale_dir + "/" + filename + ".asc")
- 
-+def md5sum(file):
-+     import md5
-+     sum = md5.new()
-+     f = open(file, "r")
-+     while 1:
-+          data = f.read(1024)
-+          if not data: break
-+          sum.update(data)
-+     f.close()
-+     if sys.version[:1] > '2':
-+          # when using Python 2.0 or newer
-+          return sum.hexdigest()
-+     else:
-+          return string.join(map((lambda x:"%02x" % ord(x)),sum.digest()),'')
-+
- old_filename = None
- packages_filename = None
- filelist_filename = "Packages.filelist"
-@@ -87,7 +102,7 @@
- files.sort()
- for filename in files:
-      basename = os.path.basename(filename)
--     if old_pkg_hash.has_key(basename):
-+     if old_pkg_hash.has_key(basename) and old_pkg_hash[basename].md5 == md5sum(filename) and old_pkg_hash[basename].size == os.stat(filename)[6]:
-           if (verbose):
-                sys.stderr.write("Found %s in Packages\n" % (filename,))
-           pkg = old_pkg_hash[basename]
-Index: ipkg-utils/ipkg.py
-===================================================================
---- ipkg-utils.orig/ipkg.py	2005-01-20 23:09:10.000000000 +0000
-+++ ipkg-utils/ipkg.py	2006-07-24 09:16:44.000000000 +0100
-@@ -210,8 +210,13 @@
-                     value = value + '\n' + line
-                 if name == 'size':
-                     self.size = int(value)
-+	        elif name == 'md5sum':
-+                    self.md5 = value
-                 elif self.__dict__.has_key(name):
-                     self.__dict__[name] = value
-+		else:
-+		    print "Lost field %s, %s" % (name,value)
-+
-                 if line[0] == '\n':
-                     return # consumes one blank line at end of package descriptoin
-             else:
diff --git a/packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch b/packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch
new file mode 100644
index 0000000000..9f9b9359ce
--- /dev/null
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch
@@ -0,0 +1,97 @@
+---
+ ipkg-make-index |   31 +++++++++++++++++++++++++++----
+ 1 file changed, 27 insertions(+), 4 deletions(-)
+
+Index: ipkg-utils/ipkg-make-index
+===================================================================
+--- ipkg-utils.orig/ipkg-make-index	2007-05-26 23:45:56.000000000 +0100
++++ ipkg-utils/ipkg-make-index	2007-05-26 23:47:25.000000000 +0100
+@@ -40,6 +40,7 @@ def to_locale(filename, locale):
+ old_filename = None
+ packages_filename = None
+ filelist_filename = "Packages.filelist"
++stamplist_filename = "Packages.stamps"
+ opt_s = 0
+ opt_m = 0
+ (opts, remaining_args) = getopt.getopt(sys.argv[1:], "hl:p:vsmr:L:")
+@@ -50,6 +51,7 @@ for (optkey, optval) in opts:
+           opt_s = 1
+      if optkey == '-p': 
+           packages_filename = optval
++          stamplist_filename = optval + ".stamps"
+      if optkey == '-l':
+           filelist_filename = optval
+      if optkey == '-v':
+@@ -72,6 +74,7 @@ old_pkg_hash = {}
+ if packages_filename and not old_filename and os.path.exists(packages_filename):
+      old_filename = packages_filename
+ 
++pkgsStamps = {}
+ if old_filename:
+      if (verbose):
+           sys.stderr.write("Reading package list from " + old_filename + "\n")
+@@ -80,6 +83,14 @@ if old_filename:
+      for k in old_packages.packages.keys():
+           p = old_packages.packages[k]
+           old_pkg_hash[p.filename] = p
++     try:
++        f = open(stamplist_filename, "r")
++        for l in f:
++            l = l.strip()
++            s, f = l.split(" ", 1)
++            pkgsStamps[f] = int(s)
++     except IOError:
++        pass
+ 
+ if (verbose):
+      sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, ))
+@@ -87,11 +98,17 @@ files=glob(pkg_dir + '/*.ipk') + glob(pk
+ files.sort()
+ for filename in files:
+      basename = os.path.basename(filename)
++     pkg = None
++     fnameStat = os.stat(filename)
+      if old_pkg_hash.has_key(basename):
+-          if (verbose):
++          if pkgsStamps.has_key(basename) and fnameStat.st_mtime == pkgsStamps[basename]:
++            if (verbose):
+                sys.stderr.write("Found %s in Packages\n" % (filename,))
+-          pkg = old_pkg_hash[basename]
+-     else:
++            pkg = old_pkg_hash[basename]
++          else:
++               sys.stderr.write("Found %s in Packages, but mtime differs - re-reading\n" % (filename,))
++
++     if not pkg:
+           if (verbose):
+                sys.stderr.write("Reading info for package %s\n" % (filename,))
+           pkg = ipkg.Package(filename)
+@@ -101,6 +118,7 @@ for filename in files:
+      else:
+           old_filename = ""
+      s = packages.add_package(pkg)
++     pkgsStamps[basename] = fnameStat.st_mtime
+      if s == 0:
+           if old_filename:
+                # old package was displaced by newer
+@@ -114,6 +132,11 @@ for filename in files:
+           if opt_s:
+                print filename
+ 
++pkgsStampsFile = open(stamplist_filename, "w")
++for f in pkgsStamps.keys():
++    pkgsStampsFile.write("%d %s\n" % (pkgsStamps[f], f))
++pkgsStampsFile.close()
++
+ if opt_s:
+      sys.exit(0)
+ 
+@@ -154,7 +177,7 @@ if packages_filename:
+      os.rename(tmp_packages_filename, packages_filename)
+      os.rename(tmp_gzip_filename, gzip_filename)
+ 
+-if verbose:	
++if verbose:
+      sys.stderr.write("Generate Packages.filelist file\n")
+ files = {}
+ names = packages.packages.keys()
diff --git a/packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch b/packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch
index 00ee391405..00ee391405 100644
--- a/packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch
diff --git a/packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch b/packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch
new file mode 100644
index 0000000000..94c57f63f2
--- /dev/null
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch
@@ -0,0 +1,306 @@
+---
+ arfile.py |  124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ ipkg.py   |  106 ++++++++++++++++++++++++++---------------------------
+ setup.py  |    2 -
+ 3 files changed, 177 insertions(+), 55 deletions(-)
+
+Index: ipkg-utils/arfile.py
+===================================================================
+--- /dev/null	1970-01-01 00:00:00.000000000 +0000
++++ ipkg-utils/arfile.py	2007-05-26 23:46:59.000000000 +0100
+@@ -0,0 +1,124 @@
++"""
++arfile - A module to parse GNU ar archives.
++
++Copyright (c) 2006-7 Paul Sokolovsky
++This file is released under the terms 
++of GNU General Public License v2 or later.
++"""
++import sys
++import os
++import tarfile 
++
++
++class FileSection:
++    "A class which allows to treat portion of file as separate file object."
++
++    def __init__(self, f, offset, size):
++        self.f = f
++        self.offset = offset
++        self.size = size
++        self.seek(0, 0)
++
++    def seek(self, offset, whence = 0):
++#        print "seek(%x, %d)" % (offset, whence)
++        if whence == 0:
++            return self.f.seek(offset + self.offset, whence)
++        elif whence == 1:
++            return self.f.seek(offset, whence)
++        elif whence == 2:
++            return self.f.seek(self.offset + self.size + offset, 0)
++        else:
++            assert False
++
++    def tell(self):
++#        print "tell()"
++        return self.f.tell() - self.offset
++
++    def read(self, size = -1):
++#        print "read(%d)" % size
++        return self.f.read(size)
++
++class ArFile:
++
++    def __init__(self, f):
++        self.f = f
++        self.directory = {}
++        self.directoryRead = False
++
++        signature = self.f.readline()
++        assert signature == "!<arch>\n"
++        self.directoryOffset = self.f.tell()
++
++    def open(self, fname):
++        if self.directory.has_key(fname):
++            return FileSection(self.f, self.directory[fname][-1], int(self.directory[fname][5]))
++
++        if self.directoryRead:
++            raise IOError, (2, "AR member not found: " + fname)
++
++        f = self._scan(fname)
++        if f == None:
++            raise IOError, (2, "AR member not found: " + fname)
++        return f
++
++
++    def _scan(self, fname):
++        self.f.seek(self.directoryOffset, 0)
++
++        while True:
++            l = self.f.readline()
++            if not l: 
++                self.directoryRead = True
++                return None
++
++            if l == "\n":
++                l = self.f.readline()
++                if not l: break
++            descriptor = l.split()
++#            print descriptor
++            size = int(descriptor[5])
++            memberName = descriptor[0][:-1]
++            self.directory[memberName] = descriptor + [self.f.tell()]
++#            print "read:", memberName
++            if memberName == fname:
++                # Record directory offset to start from next time
++                self.directoryOffset = self.f.tell() + size
++                return FileSection(self.f, self.f.tell(), size)
++
++            # Skip data and loop
++            data = self.f.seek(size, 1)
++#            print hex(f.tell())
++
++
++if __name__ == "__main__":
++    if None:
++        f = open(sys.argv[1], "rb")
++
++        ar = ArFile(f)
++        tarStream = ar.open("data.tar.gz")
++        print "--------"
++        tarStream = ar.open("data.tar.gz")
++        print "--------"
++        tarStream = ar.open("control.tar.gz")
++        print "--------"
++        tarStream = ar.open("control.tar.gz2")
++
++        sys.exit(0)
++
++
++    dir = "."
++    if len(sys.argv) > 1:
++        dir = sys.argv[1]
++    for f in os.listdir(dir):
++        if not f.endswith(".ipk"): continue
++
++        print "=== %s ===" % f
++        f = open(dir + "/" + f, "rb")
++
++        ar = ArFile(f)
++        tarStream = ar.open("control.tar.gz")
++        tarf = tarfile.open("control.tar.gz", "r", tarStream)
++        #tarf.list()
++
++        f2 = tarf.extractfile("control")
++        print f2.read()
+Index: ipkg-utils/setup.py
+===================================================================
+--- ipkg-utils.orig/setup.py	2007-05-26 23:45:55.000000000 +0100
++++ ipkg-utils/setup.py	2007-05-26 23:46:59.000000000 +0100
+@@ -16,6 +16,6 @@ distutils.core.setup( name = 'ipkg-utils
+ 		      platforms = 'POSIX',
+ 		      keywords = 'ipkg familiar',
+                       url = 'http://www.handhelds.org/sources.html/',
+-                      py_modules = [ 'ipkg' ],
++                      py_modules = [ 'ipkg', 'arfile' ],
+ 		      scripts = ['ipkg-compare-indexes', 'ipkg-make-index', 'ipkg-update-index', 'ipkg-build', 'ipkg-unbuild', 'ipkg-upload']
+                       )
+Index: ipkg-utils/ipkg.py
+===================================================================
+--- ipkg-utils.orig/ipkg.py	2007-05-26 23:46:55.000000000 +0100
++++ ipkg-utils/ipkg.py	2007-05-26 23:45:20.000000000 +0100
+@@ -41,6 +41,8 @@ import re
+ import string
+ import commands
+ from stat import ST_SIZE
++import arfile
++import tarfile
+ 
+ class Version:
+     """A class for holding parsed package version information."""
+@@ -131,78 +133,58 @@ class Package:
+ 	self.section = None
+         self.filename_header = None
+ 	self.file_list = []
+-        self.md5 = None
++        # md5 is lazy attribute, computed on demand
++        #self.md5 = None
+         self.size = None
+         self.installed_size = None
+         self.filename = None
+         self.isdeb = 0
++        self.fn = fn
+ 
+ 	if fn:
+             # see if it is deb format
+-            f = open(fn, "r")
++            f = open(fn, "rb")
+             magic = f.read(4)
+-            f.close()
++            f.seek(0, 0)
+             if (magic == "!<ar"):
+                 self.isdeb = 1
+ 
+-            # compute the MD5.
+-            f = open(fn, "r")
+-            sum = md5.new()
+-            while 1:
+-                data = f.read(1024)
+-                if not data: break
+-                sum.update(data)
+-            f.close()
+-            if sys.version[:1] > '2':
+-                # when using Python 2.0 or newer
+-                self.md5 = sum.hexdigest() 
+-            else:
+-                self.md5 = string.join(map((lambda x:"%02x" % ord(x)),sum.digest()),'')
+-            stat = os.stat(fn)
+-            self.size = stat[ST_SIZE]
++
+             self.filename = os.path.basename(fn)
++            assert self.isdeb == 1, "Old ipk format (non-deb) is unsupported"
++
+ 	    ## sys.stderr.write("  extracting control.tar.gz from %s\n"% (fn,)) 
+-            if self.isdeb:
+-                control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - './control'","r")
+-            else:
+-                control = os.popen("tar xfzO "+fn+" 'control.tar.gz' | tar xfzO - './control'","r")
+-            line = control.readline()
+-            while 1:
+-                if not line: break
+-                line = string.rstrip(line)
+-                lineparts = re.match(r'([\w-]*?):\s*(.*)', line)
+-		if lineparts:
+-                    name = string.lower(lineparts.group(1))
+-		    value = lineparts.group(2)
+-		    while 1:
+-			line = control.readline()
+-			if not line: break
+-			if line[0] != ' ': break
+-                        line = string.rstrip(line)
+-			value = value + '\n' + line
+-                    # don't allow package to override its own filename
+-                    if name == "filename":
+-                        self.filename_header = value
+-                    else:
+-                        if self.__dict__.has_key(name):
+-                            self.__dict__[name] = value
+-                else:
+-                    line = control.readline()
++
++            ar = arfile.ArFile(f)
++            tarStream = ar.open("control.tar.gz")
++            tarf = tarfile.open("control.tar.gz", "r", tarStream)
++
++            control = tarf.extractfile("control")
++            self.read_control(control)
+             control.close()
+-            if self.isdeb:
+-                data = os.popen("ar p "+fn+" data.tar.gz | tar tfz -","r")
+-            else:
+-                data = os.popen("tar xfzO "+fn+" '*data.tar.gz' | tar tfz -","r")
+-            while 1:
+-                line = data.readline()
+-                if not line: break
+-                self.file_list.append(string.rstrip(line))
+-            data.close()
+ 
+ 	self.scratch_dir = None
+ 	self.file_dir = None
+ 	self.meta_dir = None
+ 
++    def __getattr__(self, name):
++        if name == "md5":
++            self._computeFileMD5()
++            return self.md5
++        else:
++            raise AttributeError, name
++
++    def _computeFileMD5(self):
++        # compute the MD5.
++        f = open(self.fn, "rb")
++        sum = md5.new()
++        while 1:
++            data = f.read(1024)
++            if not data: break
++            sum.update(data)
++        f.close()
++        self.md5 = sum.hexdigest()
++
+     def read_control(self, control):
+         import os
+ 
+@@ -221,9 +203,15 @@ class Package:
+                     value = value + '\n' + line
+                 if name == 'size':
+                     self.size = int(value)
++	        elif name == 'md5sum':
++                    self.md5 = value
+                 elif self.__dict__.has_key(name):
+                     self.__dict__[name] = value
+-                if line[0] == '\n':
++		else:
++		    #print "Lost field %s, %s" % (name,value)
++                    pass
++
++                if line and line[0] == '\n':
+                     return # consumes one blank line at end of package descriptoin
+             else:
+                 line = control.readline()
+@@ -314,6 +302,16 @@ class Package:
+ 	return self.section
+ 
+     def get_file_list(self):
++        if not self.fn:
++            return []
++        f = open(self.fn, "rb")
++        ar = arfile.ArFile(f)
++        tarStream = ar.open("data.tar.gz")
++        tarf = tarfile.open("data.tar.gz", "r", tarStream)
++        self.file_list = tarf.getnames()
++        self.file_list = map(lambda a: ["./", ""][a.startswith("./")] + a, self.file_list)
++
++        f.close()
+ 	return self.file_list
+ 
+     def write_package(self, dirname):
diff --git a/packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch b/packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch
index a9e6fc4d05..b3e0d62cd9 100644
--- a/packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch
@@ -1,11 +1,14 @@
---- ../../../work/i686-linux/ipkg-utils-native-1.6+cvs20050404-r6/ipkg-utils/ipkg.py	2005-01-08 19:08:52.000000000 +0100
-+++ ipkg-utils/ipkg.py	2006-06-29 12:52:58.466446000 +0200
-@@ -150,11 +150,11 @@
-             stat = os.stat(fn)
-             self.size = stat[ST_SIZE]
+---
+ ipkg.py |    4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+Index: ipkg-utils/ipkg.py
+===================================================================
+--- ipkg-utils.orig/ipkg.py	2007-05-26 23:49:05.000000000 +0100
++++ ipkg-utils/ipkg.py	2007-05-26 23:49:23.000000000 +0100
+@@ -152,9 +152,9 @@ class Package:
              self.filename = os.path.basename(fn)
--	    ## sys.stderr.write("  extracting control.tar.gz from %s\n"% (fn,)) 
-+	    sys.stderr.write("  extracting control.tar.gz from %s\n"% (fn,)) 
+ 	    ## sys.stderr.write("  extracting control.tar.gz from %s\n"% (fn,)) 
              if self.isdeb:
 -                control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - '*control'","r")
 +                control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - './control'","r")
diff --git a/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb b/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
index 48c7445828..6ffe6fe679 100644
--- a/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
+++ b/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
@@ -5,10 +5,13 @@ LICENSE = "GPL"
 CONFLICTS = "ipkg-link"
 RDEPENDS = "python"
 SRCDATE = "20050404"
-PR = "r12"
+PR = "r14"
 
 SRC_URI = "${HANDHELDS_CVS};module=ipkg-utils \
-           file://index_speedup.patch;patch=1"
+           file://ipkg-utils-fix.patch;patch=1 \
+           file://ipkg-py-sane-vercompare.patch;patch=1 \
+           file://ipkg-py-tarfile.patch;patch=1 \
+           file://ipkg-make-index-track-stamps.patch;patch=1"
 
 S = "${WORKDIR}/ipkg-utils"
 
@@ -26,9 +29,3 @@ do_install() {
 	done
 }
 
-#FIXME: ipkg-utils is not allowed to have packages or else a ipkg-native -> ipkg-utils -> ipkg-utils
-#       recursive dependency is triggered. This has been fixed by installing the ipkg-link script in
-#       a dedicated package.
-#PACKAGES_prepend = "ipkg-link "
-#FILES_ipkg-link = "${bindir}/ipkg-link"
-