summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@rpsys.net>2007-05-26 23:13:13 +0000
committerRichard Purdie <rpurdie@rpsys.net>2007-05-26 23:13:13 +0000
commita30543818b99bc9e6ebfbc0d8996e6ca7c28fd26 (patch)
tree00909f0452ff8d4bb20f85439b85ac4ec537a5d7
parent18fbbaab563f7efe8053e493f23660d19e5596c1 (diff)
ipkg-utils: Merge in tweaked version of patches from #1504 (from Paul Sokolovsky) to massively speedup ipkg-make-index
-rw-r--r--packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty0
-rw-r--r--packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb5
-rw-r--r--packages/ipkg-utils/ipkg-utils/index_speedup.patch53
-rw-r--r--packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch97
-rw-r--r--packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch (renamed from packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch)0
-rw-r--r--packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch306
-rw-r--r--packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch (renamed from packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch)17
-rw-r--r--packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb13
8 files changed, 420 insertions, 71 deletions
diff --git a/packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty b/packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty
deleted file mode 100644
index e69de29bb2..0000000000
--- a/packages/ipkg-utils/ipkg-utils-native/.mtn2git_empty
+++ /dev/null
diff --git a/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb b/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
index 03a381387e..a0964d65b3 100644
--- a/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
+++ b/packages/ipkg-utils/ipkg-utils-native_1.6+cvs20050404.bb
@@ -1,15 +1,14 @@
require ipkg-utils_${PV}.bb
-SRC_URI += "file://ipkg-utils-fix.patch;patch=1 \
- file://ipkg-py-sane-vercompare.patch;patch=1"
RDEPENDS = ""
-PR = "r9"
+PR = "r11"
inherit native
# Avoid circular dependencies from package_ipk.bbclass
PACKAGES = ""
FILESDIR = "${@os.path.dirname(bb.data.getVar('FILE',d,1))}/ipkg-utils"
+INSTALL += "arfile.py"
do_stage() {
for i in ${INSTALL}; do
diff --git a/packages/ipkg-utils/ipkg-utils/index_speedup.patch b/packages/ipkg-utils/ipkg-utils/index_speedup.patch
deleted file mode 100644
index bef28a9df1..0000000000
--- a/packages/ipkg-utils/ipkg-utils/index_speedup.patch
+++ /dev/null
@@ -1,53 +0,0 @@
-Index: ipkg-utils/ipkg-make-index
-===================================================================
---- ipkg-utils.orig/ipkg-make-index 2005-03-20 18:10:54.000000000 +0000
-+++ ipkg-utils/ipkg-make-index 2006-07-24 09:18:16.000000000 +0100
-@@ -37,6 +37,21 @@
- if os.path.exists(pkg_dir + "/" + filename + ".asc"):
- os.rename(pkg_dir + "/" + filename + ".asc", locale_dir + "/" + filename + ".asc")
-
-+def md5sum(file):
-+ import md5
-+ sum = md5.new()
-+ f = open(file, "r")
-+ while 1:
-+ data = f.read(1024)
-+ if not data: break
-+ sum.update(data)
-+ f.close()
-+ if sys.version[:1] > '2':
-+ # when using Python 2.0 or newer
-+ return sum.hexdigest()
-+ else:
-+ return string.join(map((lambda x:"%02x" % ord(x)),sum.digest()),'')
-+
- old_filename = None
- packages_filename = None
- filelist_filename = "Packages.filelist"
-@@ -87,7 +102,7 @@
- files.sort()
- for filename in files:
- basename = os.path.basename(filename)
-- if old_pkg_hash.has_key(basename):
-+ if old_pkg_hash.has_key(basename) and old_pkg_hash[basename].md5 == md5sum(filename) and old_pkg_hash[basename].size == os.stat(filename)[6]:
- if (verbose):
- sys.stderr.write("Found %s in Packages\n" % (filename,))
- pkg = old_pkg_hash[basename]
-Index: ipkg-utils/ipkg.py
-===================================================================
---- ipkg-utils.orig/ipkg.py 2005-01-20 23:09:10.000000000 +0000
-+++ ipkg-utils/ipkg.py 2006-07-24 09:16:44.000000000 +0100
-@@ -210,8 +210,13 @@
- value = value + '\n' + line
- if name == 'size':
- self.size = int(value)
-+ elif name == 'md5sum':
-+ self.md5 = value
- elif self.__dict__.has_key(name):
- self.__dict__[name] = value
-+ else:
-+ print "Lost field %s, %s" % (name,value)
-+
- if line[0] == '\n':
- return # consumes one blank line at end of package descriptoin
- else:
diff --git a/packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch b/packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch
new file mode 100644
index 0000000000..9f9b9359ce
--- /dev/null
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-make-index-track-stamps.patch
@@ -0,0 +1,97 @@
+---
+ ipkg-make-index | 31 +++++++++++++++++++++++++++----
+ 1 file changed, 27 insertions(+), 4 deletions(-)
+
+Index: ipkg-utils/ipkg-make-index
+===================================================================
+--- ipkg-utils.orig/ipkg-make-index 2007-05-26 23:45:56.000000000 +0100
++++ ipkg-utils/ipkg-make-index 2007-05-26 23:47:25.000000000 +0100
+@@ -40,6 +40,7 @@ def to_locale(filename, locale):
+ old_filename = None
+ packages_filename = None
+ filelist_filename = "Packages.filelist"
++stamplist_filename = "Packages.stamps"
+ opt_s = 0
+ opt_m = 0
+ (opts, remaining_args) = getopt.getopt(sys.argv[1:], "hl:p:vsmr:L:")
+@@ -50,6 +51,7 @@ for (optkey, optval) in opts:
+ opt_s = 1
+ if optkey == '-p':
+ packages_filename = optval
++ stamplist_filename = optval + ".stamps"
+ if optkey == '-l':
+ filelist_filename = optval
+ if optkey == '-v':
+@@ -72,6 +74,7 @@ old_pkg_hash = {}
+ if packages_filename and not old_filename and os.path.exists(packages_filename):
+ old_filename = packages_filename
+
++pkgsStamps = {}
+ if old_filename:
+ if (verbose):
+ sys.stderr.write("Reading package list from " + old_filename + "\n")
+@@ -80,6 +83,14 @@ if old_filename:
+ for k in old_packages.packages.keys():
+ p = old_packages.packages[k]
+ old_pkg_hash[p.filename] = p
++ try:
++ f = open(stamplist_filename, "r")
++ for l in f:
++ l = l.strip()
++ s, f = l.split(" ", 1)
++ pkgsStamps[f] = int(s)
++ except IOError:
++ pass
+
+ if (verbose):
+ sys.stderr.write("Reading in all the package info from %s\n" % (pkg_dir, ))
+@@ -87,11 +98,17 @@ files=glob(pkg_dir + '/*.ipk') + glob(pk
+ files.sort()
+ for filename in files:
+ basename = os.path.basename(filename)
++ pkg = None
++ fnameStat = os.stat(filename)
+ if old_pkg_hash.has_key(basename):
+- if (verbose):
++ if pkgsStamps.has_key(basename) and fnameStat.st_mtime == pkgsStamps[basename]:
++ if (verbose):
+ sys.stderr.write("Found %s in Packages\n" % (filename,))
+- pkg = old_pkg_hash[basename]
+- else:
++ pkg = old_pkg_hash[basename]
++ else:
++ sys.stderr.write("Found %s in Packages, but mtime differs - re-reading\n" % (filename,))
++
++ if not pkg:
+ if (verbose):
+ sys.stderr.write("Reading info for package %s\n" % (filename,))
+ pkg = ipkg.Package(filename)
+@@ -101,6 +118,7 @@ for filename in files:
+ else:
+ old_filename = ""
+ s = packages.add_package(pkg)
++ pkgsStamps[basename] = fnameStat.st_mtime
+ if s == 0:
+ if old_filename:
+ # old package was displaced by newer
+@@ -114,6 +132,11 @@ for filename in files:
+ if opt_s:
+ print filename
+
++pkgsStampsFile = open(stamplist_filename, "w")
++for f in pkgsStamps.keys():
++ pkgsStampsFile.write("%d %s\n" % (pkgsStamps[f], f))
++pkgsStampsFile.close()
++
+ if opt_s:
+ sys.exit(0)
+
+@@ -154,7 +177,7 @@ if packages_filename:
+ os.rename(tmp_packages_filename, packages_filename)
+ os.rename(tmp_gzip_filename, gzip_filename)
+
+-if verbose:
++if verbose:
+ sys.stderr.write("Generate Packages.filelist file\n")
+ files = {}
+ names = packages.packages.keys()
diff --git a/packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch b/packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch
index 00ee391405..00ee391405 100644
--- a/packages/ipkg-utils/ipkg-utils-native/ipkg-py-sane-vercompare.patch
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-py-sane-vercompare.patch
diff --git a/packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch b/packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch
new file mode 100644
index 0000000000..94c57f63f2
--- /dev/null
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-py-tarfile.patch
@@ -0,0 +1,306 @@
+---
+ arfile.py | 124 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ ipkg.py | 106 ++++++++++++++++++++++++++---------------------------
+ setup.py | 2 -
+ 3 files changed, 177 insertions(+), 55 deletions(-)
+
+Index: ipkg-utils/arfile.py
+===================================================================
+--- /dev/null 1970-01-01 00:00:00.000000000 +0000
++++ ipkg-utils/arfile.py 2007-05-26 23:46:59.000000000 +0100
+@@ -0,0 +1,124 @@
++"""
++arfile - A module to parse GNU ar archives.
++
++Copyright (c) 2006-7 Paul Sokolovsky
++This file is released under the terms
++of GNU General Public License v2 or later.
++"""
++import sys
++import os
++import tarfile
++
++
++class FileSection:
++ "A class which allows to treat portion of file as separate file object."
++
++ def __init__(self, f, offset, size):
++ self.f = f
++ self.offset = offset
++ self.size = size
++ self.seek(0, 0)
++
++ def seek(self, offset, whence = 0):
++# print "seek(%x, %d)" % (offset, whence)
++ if whence == 0:
++ return self.f.seek(offset + self.offset, whence)
++ elif whence == 1:
++ return self.f.seek(offset, whence)
++ elif whence == 2:
++ return self.f.seek(self.offset + self.size + offset, 0)
++ else:
++ assert False
++
++ def tell(self):
++# print "tell()"
++ return self.f.tell() - self.offset
++
++ def read(self, size = -1):
++# print "read(%d)" % size
++ return self.f.read(size)
++
++class ArFile:
++
++ def __init__(self, f):
++ self.f = f
++ self.directory = {}
++ self.directoryRead = False
++
++ signature = self.f.readline()
++ assert signature == "!<arch>\n"
++ self.directoryOffset = self.f.tell()
++
++ def open(self, fname):
++ if self.directory.has_key(fname):
++ return FileSection(self.f, self.directory[fname][-1], int(self.directory[fname][5]))
++
++ if self.directoryRead:
++ raise IOError, (2, "AR member not found: " + fname)
++
++ f = self._scan(fname)
++ if f == None:
++ raise IOError, (2, "AR member not found: " + fname)
++ return f
++
++
++ def _scan(self, fname):
++ self.f.seek(self.directoryOffset, 0)
++
++ while True:
++ l = self.f.readline()
++ if not l:
++ self.directoryRead = True
++ return None
++
++ if l == "\n":
++ l = self.f.readline()
++ if not l: break
++ descriptor = l.split()
++# print descriptor
++ size = int(descriptor[5])
++ memberName = descriptor[0][:-1]
++ self.directory[memberName] = descriptor + [self.f.tell()]
++# print "read:", memberName
++ if memberName == fname:
++ # Record directory offset to start from next time
++ self.directoryOffset = self.f.tell() + size
++ return FileSection(self.f, self.f.tell(), size)
++
++ # Skip data and loop
++ data = self.f.seek(size, 1)
++# print hex(f.tell())
++
++
++if __name__ == "__main__":
++ if None:
++ f = open(sys.argv[1], "rb")
++
++ ar = ArFile(f)
++ tarStream = ar.open("data.tar.gz")
++ print "--------"
++ tarStream = ar.open("data.tar.gz")
++ print "--------"
++ tarStream = ar.open("control.tar.gz")
++ print "--------"
++ tarStream = ar.open("control.tar.gz2")
++
++ sys.exit(0)
++
++
++ dir = "."
++ if len(sys.argv) > 1:
++ dir = sys.argv[1]
++ for f in os.listdir(dir):
++ if not f.endswith(".ipk"): continue
++
++ print "=== %s ===" % f
++ f = open(dir + "/" + f, "rb")
++
++ ar = ArFile(f)
++ tarStream = ar.open("control.tar.gz")
++ tarf = tarfile.open("control.tar.gz", "r", tarStream)
++ #tarf.list()
++
++ f2 = tarf.extractfile("control")
++ print f2.read()
+Index: ipkg-utils/setup.py
+===================================================================
+--- ipkg-utils.orig/setup.py 2007-05-26 23:45:55.000000000 +0100
++++ ipkg-utils/setup.py 2007-05-26 23:46:59.000000000 +0100
+@@ -16,6 +16,6 @@ distutils.core.setup( name = 'ipkg-utils
+ platforms = 'POSIX',
+ keywords = 'ipkg familiar',
+ url = 'http://www.handhelds.org/sources.html/',
+- py_modules = [ 'ipkg' ],
++ py_modules = [ 'ipkg', 'arfile' ],
+ scripts = ['ipkg-compare-indexes', 'ipkg-make-index', 'ipkg-update-index', 'ipkg-build', 'ipkg-unbuild', 'ipkg-upload']
+ )
+Index: ipkg-utils/ipkg.py
+===================================================================
+--- ipkg-utils.orig/ipkg.py 2007-05-26 23:46:55.000000000 +0100
++++ ipkg-utils/ipkg.py 2007-05-26 23:45:20.000000000 +0100
+@@ -41,6 +41,8 @@ import re
+ import string
+ import commands
+ from stat import ST_SIZE
++import arfile
++import tarfile
+
+ class Version:
+ """A class for holding parsed package version information."""
+@@ -131,78 +133,58 @@ class Package:
+ self.section = None
+ self.filename_header = None
+ self.file_list = []
+- self.md5 = None
++ # md5 is lazy attribute, computed on demand
++ #self.md5 = None
+ self.size = None
+ self.installed_size = None
+ self.filename = None
+ self.isdeb = 0
++ self.fn = fn
+
+ if fn:
+ # see if it is deb format
+- f = open(fn, "r")
++ f = open(fn, "rb")
+ magic = f.read(4)
+- f.close()
++ f.seek(0, 0)
+ if (magic == "!<ar"):
+ self.isdeb = 1
+
+- # compute the MD5.
+- f = open(fn, "r")
+- sum = md5.new()
+- while 1:
+- data = f.read(1024)
+- if not data: break
+- sum.update(data)
+- f.close()
+- if sys.version[:1] > '2':
+- # when using Python 2.0 or newer
+- self.md5 = sum.hexdigest()
+- else:
+- self.md5 = string.join(map((lambda x:"%02x" % ord(x)),sum.digest()),'')
+- stat = os.stat(fn)
+- self.size = stat[ST_SIZE]
++
+ self.filename = os.path.basename(fn)
++ assert self.isdeb == 1, "Old ipk format (non-deb) is unsupported"
++
+ ## sys.stderr.write(" extracting control.tar.gz from %s\n"% (fn,))
+- if self.isdeb:
+- control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - './control'","r")
+- else:
+- control = os.popen("tar xfzO "+fn+" 'control.tar.gz' | tar xfzO - './control'","r")
+- line = control.readline()
+- while 1:
+- if not line: break
+- line = string.rstrip(line)
+- lineparts = re.match(r'([\w-]*?):\s*(.*)', line)
+- if lineparts:
+- name = string.lower(lineparts.group(1))
+- value = lineparts.group(2)
+- while 1:
+- line = control.readline()
+- if not line: break
+- if line[0] != ' ': break
+- line = string.rstrip(line)
+- value = value + '\n' + line
+- # don't allow package to override its own filename
+- if name == "filename":
+- self.filename_header = value
+- else:
+- if self.__dict__.has_key(name):
+- self.__dict__[name] = value
+- else:
+- line = control.readline()
++
++ ar = arfile.ArFile(f)
++ tarStream = ar.open("control.tar.gz")
++ tarf = tarfile.open("control.tar.gz", "r", tarStream)
++
++ control = tarf.extractfile("control")
++ self.read_control(control)
+ control.close()
+- if self.isdeb:
+- data = os.popen("ar p "+fn+" data.tar.gz | tar tfz -","r")
+- else:
+- data = os.popen("tar xfzO "+fn+" '*data.tar.gz' | tar tfz -","r")
+- while 1:
+- line = data.readline()
+- if not line: break
+- self.file_list.append(string.rstrip(line))
+- data.close()
+
+ self.scratch_dir = None
+ self.file_dir = None
+ self.meta_dir = None
+
++ def __getattr__(self, name):
++ if name == "md5":
++ self._computeFileMD5()
++ return self.md5
++ else:
++ raise AttributeError, name
++
++ def _computeFileMD5(self):
++ # compute the MD5.
++ f = open(self.fn, "rb")
++ sum = md5.new()
++ while 1:
++ data = f.read(1024)
++ if not data: break
++ sum.update(data)
++ f.close()
++ self.md5 = sum.hexdigest()
++
+ def read_control(self, control):
+ import os
+
+@@ -221,9 +203,15 @@ class Package:
+ value = value + '\n' + line
+ if name == 'size':
+ self.size = int(value)
++ elif name == 'md5sum':
++ self.md5 = value
+ elif self.__dict__.has_key(name):
+ self.__dict__[name] = value
+- if line[0] == '\n':
++ else:
++ #print "Lost field %s, %s" % (name,value)
++ pass
++
++ if line and line[0] == '\n':
+ return # consumes one blank line at end of package descriptoin
+ else:
+ line = control.readline()
+@@ -314,6 +302,16 @@ class Package:
+ return self.section
+
+ def get_file_list(self):
++ if not self.fn:
++ return []
++ f = open(self.fn, "rb")
++ ar = arfile.ArFile(f)
++ tarStream = ar.open("data.tar.gz")
++ tarf = tarfile.open("data.tar.gz", "r", tarStream)
++ self.file_list = tarf.getnames()
++ self.file_list = map(lambda a: ["./", ""][a.startswith("./")] + a, self.file_list)
++
++ f.close()
+ return self.file_list
+
+ def write_package(self, dirname):
diff --git a/packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch b/packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch
index a9e6fc4d05..b3e0d62cd9 100644
--- a/packages/ipkg-utils/ipkg-utils-native/ipkg-utils-fix.patch
+++ b/packages/ipkg-utils/ipkg-utils/ipkg-utils-fix.patch
@@ -1,11 +1,14 @@
---- ../../../work/i686-linux/ipkg-utils-native-1.6+cvs20050404-r6/ipkg-utils/ipkg.py 2005-01-08 19:08:52.000000000 +0100
-+++ ipkg-utils/ipkg.py 2006-06-29 12:52:58.466446000 +0200
-@@ -150,11 +150,11 @@
- stat = os.stat(fn)
- self.size = stat[ST_SIZE]
+---
+ ipkg.py | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+Index: ipkg-utils/ipkg.py
+===================================================================
+--- ipkg-utils.orig/ipkg.py 2007-05-26 23:49:05.000000000 +0100
++++ ipkg-utils/ipkg.py 2007-05-26 23:49:23.000000000 +0100
+@@ -152,9 +152,9 @@ class Package:
self.filename = os.path.basename(fn)
-- ## sys.stderr.write(" extracting control.tar.gz from %s\n"% (fn,))
-+ sys.stderr.write(" extracting control.tar.gz from %s\n"% (fn,))
+ ## sys.stderr.write(" extracting control.tar.gz from %s\n"% (fn,))
if self.isdeb:
- control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - '*control'","r")
+ control = os.popen("ar p "+fn+" control.tar.gz | tar xfzO - './control'","r")
diff --git a/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb b/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
index 48c7445828..6ffe6fe679 100644
--- a/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
+++ b/packages/ipkg-utils/ipkg-utils_1.6+cvs20050404.bb
@@ -5,10 +5,13 @@ LICENSE = "GPL"
CONFLICTS = "ipkg-link"
RDEPENDS = "python"
SRCDATE = "20050404"
-PR = "r12"
+PR = "r14"
SRC_URI = "${HANDHELDS_CVS};module=ipkg-utils \
- file://index_speedup.patch;patch=1"
+ file://ipkg-utils-fix.patch;patch=1 \
+ file://ipkg-py-sane-vercompare.patch;patch=1 \
+ file://ipkg-py-tarfile.patch;patch=1 \
+ file://ipkg-make-index-track-stamps.patch;patch=1"
S = "${WORKDIR}/ipkg-utils"
@@ -26,9 +29,3 @@ do_install() {
done
}
-#FIXME: ipkg-utils is not allowed to have packages or else a ipkg-native -> ipkg-utils -> ipkg-utils
-# recursive dependency is triggered. This has been fixed by installing the ipkg-link script in
-# a dedicated package.
-#PACKAGES_prepend = "ipkg-link "
-#FILES_ipkg-link = "${bindir}/ipkg-link"
-