diff options
author | Mike Westerhof <mwester@dls.net> | 2008-12-27 00:32:13 -0600 |
---|---|---|
committer | Mike Westerhof <mwester@dls.net> | 2008-12-27 00:32:13 -0600 |
commit | 1bbdc74a5384c81d0bdeb96f6acdfef91430d35d (patch) | |
tree | 002c64a551d3d7142c4f5cfcd7ca40d2a92e0f0b | |
parent | 0faec81b54ed7bee58647c14ba18ed28200bef39 (diff) |
opkg_wget_nogpg: dramatically reduce memory footprint, too dodge OOM killer.
- eliminate unnecessary libopkg.so
- use vfork() instead of fork() and system()
- make specifying of alternate tmpdir actually work.
4 files changed, 554 insertions, 2 deletions
diff --git a/packages/opkg/files/opkg_wget_nogpg_01_use_vfork_gunzip.patch b/packages/opkg/files/opkg_wget_nogpg_01_use_vfork_gunzip.patch new file mode 100644 index 0000000000..00bd0bdd29 --- /dev/null +++ b/packages/opkg/files/opkg_wget_nogpg_01_use_vfork_gunzip.patch @@ -0,0 +1,163 @@ +# This patchset updates the libbb stuff to use a vfork() version of +# gz_open, called gzvopen. This is done because a standard fork will +# duplicate the entire address space. This will invoke the OOM +# (out of memory) killer on small-memory machines, because most often +# by the time we unzip any package, we've read the entire package +# database into memory already. By using vfork() and immediatly +# execing the external gunzip utility, we avoid the need to clone the +# entire address space. +# +# Yes, this is actually **LESS** efficient than the original way! +# But there is no way to (currently) dodge the OOM killer on a +# per-process basis, so the alternatives are to either change the +# OOM killer behavior system-wide, or to use this workaround. +# +# Mike Westerhof, Dec 2008 + +--- orig/libbb/gz_open.c 2008-03-01 12:55:33.000000000 -0600 ++++ opkg/libbb/gz_open.c 2008-12-21 21:41:13.000000000 -0600 +@@ -56,3 +56,78 @@ + } + return(fdopen(unzip_pipe[0], "r")); + } ++ ++/* gz_open implementation using gunzip and a vfork/exec -- dodges OOM killer */ ++extern FILE *gzvopen(FILE *compressed_file, int *pid) ++{ ++ int unzip_pipe[2]; ++ off_t floc; ++ int cfile; ++ ++ /* create a new file descriptor for the input stream ++ * (it *must* be associated with a file) ++ * and seek to the same position in that fd as the stream. ++ */ ++ cfile = dup(fileno(compressed_file)); ++ floc = ftello(compressed_file); ++ lseek(cfile, floc, SEEK_SET); ++ ++ /* create the pipe */ ++ if (pipe(unzip_pipe)!=0) { ++ error_msg("gzvopen(): pipe error"); ++ return(NULL); ++ } ++ ++ *pid = vfork(); ++ ++ if (*pid < 0) { ++ error_msg("gzvopen(): fork failed"); ++ return(NULL); ++ } ++ ++ if (*pid==0) { ++ /* child process - reads STDIN, writes to pipe */ ++ ++ /* close unused read end of pipe */ ++ close(unzip_pipe[0]); ++ ++ /* connect child's stdout to the pipe write end */ ++ dup2(unzip_pipe[1], 1); ++ ++ /* connect child's stdin to the fd passed in to us */ ++ dup2(cfile, 0); ++ ++ /* execute the gunzip utility */ ++ execlp("gunzip","gunzip",NULL); ++ ++ /* if we get here, we had a failure - since we are ++ * using vfork(), we cannot call exit(), must call _exit(). ++ */ ++ _exit(-1); ++ } ++ ++ /* Parent process is executing here */ ++ ++ /* we have no more need of the duplicate fd */ ++ close(cfile); ++ ++ /* close the write end of the pipe */ ++ close(unzip_pipe[1]); ++ ++ /* return the read end of the pipe as a FILE */ ++ return(fdopen(unzip_pipe[0], "r")); ++} ++ ++extern void gzvclose(int gunzip_pid) ++{ ++ if (kill(gunzip_pid, SIGTERM) == -1) { ++ perror("gzvclose()"); ++ fprintf(stderr,"%s: unable to kill gunzip pid.\n", ++ __FUNCTION__); ++ } ++ ++ if (waitpid(gunzip_pid, NULL, 0) == -1) { ++ fprintf(stderr,"%s: unable to wait on gunzip pid.\n", ++ __FUNCTION__); ++ } ++} +--- orig/libbb/libbb.h 2008-12-20 15:06:51.000000000 -0600 ++++ opkg/libbb/libbb.h 2008-12-21 21:41:13.000000000 -0600 +@@ -270,6 +270,8 @@ + extern int unzip(FILE *l_in_file, FILE *l_out_file); + extern void gz_close(int gunzip_pid); + extern FILE *gz_open(FILE *compressed_file, int *pid); ++extern void gzvclose(int gunzip_pid); ++extern FILE *gzvopen(FILE *compressed_file, int *pid); + + extern struct hostent *xgethostbyname(const char *name); + extern int create_icmp_socket(void); +--- orig/libbb/unarchive.c 2008-09-10 11:48:23.000000000 -0500 ++++ opkg/libbb/unarchive.c 2008-12-21 21:41:13.000000000 -0600 +@@ -749,7 +749,7 @@ + while ((ar_header = get_header_ar(deb_stream)) != NULL) { + if (strcmp(ared_file, ar_header->name) == 0) { + /* open a stream of decompressed data */ +- uncompressed_stream = gz_open(deb_stream, &gunzip_pid); ++ uncompressed_stream = gzvopen(deb_stream, &gunzip_pid); + archive_offset = 0; + output_buffer = unarchive(uncompressed_stream, out_stream, get_header_tar, free_header_tar, extract_function, prefix, file_list); + } +@@ -757,7 +757,7 @@ + free (ar_header->name); + free (ar_header); + } +- gz_close(gunzip_pid); ++ gzvclose(gunzip_pid); + fclose(deb_stream); + fclose(uncompressed_stream); + free(ared_file); +@@ -769,7 +769,7 @@ + file_header_t *tar_header; + archive_offset = 0; + fseek(deb_stream, 0, SEEK_SET); +- unzipped_opkg_stream = gz_open(deb_stream, &unzipped_opkg_pid); ++ unzipped_opkg_stream = gzvopen(deb_stream, &unzipped_opkg_pid); + + /*fprintf(stderr, __FUNCTION__ ": processing opkg %s -- ared_file=%s\n", package_filename, ared_file);*/ + /* walk through outer tar file to find ared_file */ +@@ -779,7 +779,7 @@ + name_offset = 2; + if (strcmp(ared_file, tar_header->name+name_offset) == 0) { + /* open a stream of decompressed data */ +- uncompressed_stream = gz_open(unzipped_opkg_stream, &gunzip_pid); ++ uncompressed_stream = gzvopen(unzipped_opkg_stream, &gunzip_pid); + archive_offset = 0; + /*fprintf(stderr, __FUNCTION__ ":%d: here -- found file\n", __LINE__);*/ + output_buffer = unarchive(uncompressed_stream, +@@ -791,14 +791,14 @@ + file_list); + /*fprintf(stderr, __FUNCTION__ ":%d: unarchive complete\n", __LINE__);*/ + free_header_tar(tar_header); +- gz_close(gunzip_pid); ++ gzvclose(gunzip_pid); + fclose(uncompressed_stream); + break; + } + seek_sub_file(unzipped_opkg_stream, tar_header->size); + free_header_tar(tar_header); + } +- gz_close(unzipped_opkg_pid); ++ gzvclose(unzipped_opkg_pid); + fclose(unzipped_opkg_stream); + fclose(deb_stream); + free(ared_file); diff --git a/packages/opkg/files/opkg_wget_nogpg_02_use_vfork_system.patch b/packages/opkg/files/opkg_wget_nogpg_02_use_vfork_system.patch new file mode 100644 index 0000000000..5eb9517df8 --- /dev/null +++ b/packages/opkg/files/opkg_wget_nogpg_02_use_vfork_system.patch @@ -0,0 +1,200 @@ +# This patchset implements the suggestion found in xsystem.c in the +# original opkg sources -- use vfork and exec instead of system(). +# +# This, like the preceding patch in this patchset is vital to dodging +# the OOM killer on small-memory machines. +# +# Mike Westerhof, Dec 2008 + +--- orig/libopkg/opkg.c 2008-12-20 15:07:04.000000000 -0600 ++++ opkg/libopkg/opkg.c 2008-12-21 21:41:07.000000000 -0600 +@@ -31,6 +31,7 @@ + + #include "sprintf_alloc.h" + #include "file_util.h" ++#include "xsystem.h" + + #include <libbb/libbb.h> + +--- orig/libopkg/xsystem.h 2008-03-01 12:55:29.000000000 -0600 ++++ opkg/libopkg/xsystem.h 2008-12-21 21:41:07.000000000 -0600 +@@ -29,6 +29,7 @@ + as defined in <sys/wait.h>. + */ + int xsystem(const char *cmd); ++int xvsystem(const char *cmd, char *const argv[]); + + #endif + +--- orig/libopkg/xsystem.c 2008-04-17 11:00:51.000000000 -0500 ++++ opkg/libopkg/xsystem.c 2008-12-21 21:41:07.000000000 -0600 +@@ -62,3 +62,58 @@ + return -1; + } + ++int xvsystem(const char *cmd, char *const argv[]) ++{ ++ int err; ++ pid_t pid; ++ ++#ifdef DO_DEBUG ++ int i = 0; ++ char *p; ++ printf("DEBUG: xvsystem: execute "); ++ while (p = argv[i++]) { ++ printf("\"%s\" ", p); ++ } ++ printf("\n"); ++#endif ++ ++ pid = vfork(); ++ ++ if (pid == 0) { /* Child */ ++ execvp(cmd, argv); ++ _exit(1); ++ ++ } else if (pid == -1) { /* Error on fork() */ ++ perror("vfork()"); ++ fprintf(stderr, "%s: ERROR: fork failed before execution: `%s'\n", ++ __FUNCTION__, cmd); ++ return -1; ++ ++ } else { /* Parent */ ++ if (waitpid(pid, &err, 0) != pid) { /* wait for child */ ++ perror("waitpid()"); ++ fprintf(stderr, "%s: ERROR: failed to execute: `%s'\n", ++ __FUNCTION__, cmd); ++ return -1; ++ } ++ } ++ ++#ifdef DO_DEBUG ++ printf("DEBUG: xvsystem: child exit status %d\n", err); ++#endif ++ ++ if (WIFSIGNALED(err)) { ++ fprintf(stderr, "%s: ERROR: Child process died due to signal %d: `%s'\n", ++ __FUNCTION__, WTERMSIG(err), cmd); ++ return -1; ++ } ++ ++ if (WIFEXITED(err)) { ++ /* Normal child exit */ ++ return WEXITSTATUS(err); ++ } ++ ++ fprintf(stderr, "%s: ERROR: Received unintelligible return value from system: %d", ++ __FUNCTION__, err); ++ return -1; ++} +--- orig/libopkg/opkg_conf.c 2008-12-20 15:06:50.000000000 -0600 ++++ opkg/libopkg/opkg_conf.c 2008-12-21 21:41:07.000000000 -0600 +@@ -316,10 +320,8 @@ + err = rmdir(conf->tmp_dir); + if (err) { + if (errno == ENOTEMPTY) { +- char *cmd; +- sprintf_alloc(&cmd, "rm -fr %s\n", conf->tmp_dir); +- err = xsystem(cmd); +- free(cmd); ++ char *argv[] = {"rm", "-fr", conf->tmp_dir, NULL}; ++ err = xvsystem(argv[0], argv); + } + if (err) + fprintf(stderr, "WARNING: Unable to remove temporary directory: %s: %s\n", conf->tmp_dir, strerror(errno)); +--- orig/libopkg/opkg_install.c 2008-12-20 15:06:51.000000000 -0600 ++++ opkg/libopkg/opkg_install.c 2008-12-21 21:41:07.000000000 -0600 +@@ -1617,13 +1617,9 @@ + } + + if (strcmp(response, "d") == 0) { +- char *cmd; +- ++ char *argv[] = {"diff", "-u", backup, file_name, NULL}; + free(response); +- /* XXX: BUG rewrite to use exec or busybox's internal diff */ +- sprintf_alloc(&cmd, "diff -u %s %s", backup, file_name); +- xsystem(cmd); +- free(cmd); ++ xvsystem(argv[0], argv); + printf(" [Press ENTER to continue]\n"); + response = file_read_line_alloc(stdin); + free(response); +--- orig/libopkg/pkg.c 2008-12-20 15:06:50.000000000 -0600 ++++ opkg/libopkg/pkg.c 2008-12-21 23:31:39.000000000 -0600 +@@ -1534,7 +1564,12 @@ + sprintf_alloc(&cmd, "%s %s", path, args); + free(path); + +- err = xsystem(cmd); ++// err = xsystem(cmd); ++ { ++ char *argv[] = {"sh", "-c", cmd, NULL}; ++ err = xvsystem(argv[0], argv); ++ } ++ + free(cmd); + + if (err) { +--- orig/libopkg/opkg_cmd.c 2008-12-20 15:07:04.000000000 -0600 ++++ opkg/libopkg/opkg_cmd.c 2008-12-21 21:41:07.000000000 -0600 +@@ -33,6 +33,7 @@ + #include "libbb/libbb.h" + #include "opkg_utils.h" + #include "opkg_defines.h" ++#include "xsystem.h" + + #include <fnmatch.h> + +@@ -371,7 +373,7 @@ + + sprintf_alloc (&path, "%s/%s", ctx->statedir, de->d_name); + if (access (path, X_OK) == 0) { +- if (system (path)) { ++ if (system (path)) { /* FIXME FIXME */ + err = errno; + perror (de->d_name); + } +@@ -382,7 +384,7 @@ + perror (ctx->statedir); + + sprintf_alloc (&cmd, "rm -rf %s", ctx->statedir); +- err = system (cmd); ++ err = system (cmd); /* FIXME FIXME */ + free (cmd); + + free (ctx->statedir); +--- orig/libopkg/opkg_download.c 2008-12-20 15:07:04.000000000 -0600 ++++ opkg/libopkg/opkg_download.c 2008-12-21 21:44:30.000000000 -0600 +@@ -129,16 +129,21 @@ + #else + { + int res; +- char *wgetcmd; +- char *wgetopts; +- wgetopts = getenv("OPKG_WGETOPTS"); +- sprintf_alloc(&wgetcmd, "wget -q %s%s -O \"%s\" \"%s\"", +- (conf->http_proxy || conf->ftp_proxy) ? "-Y on " : "", +- (wgetopts!=NULL) ? wgetopts : "", +- tmp_file_location, src); +- opkg_message(conf, OPKG_INFO, "Executing: %s\n", wgetcmd); +- res = xsystem(wgetcmd); +- free(wgetcmd); ++ char *argv[16]; ++ int i = 0; ++ ++ argv[i++] = "wget"; ++ argv[i++] = "-q"; ++ if (conf->http_proxy || conf->ftp_proxy) { ++ argv[i++] = "-Y"; ++ argv[i++] = "on"; ++ } ++ argv[i++] = "-O"; ++ argv[i++] = tmp_file_location; ++ argv[i++] = src; ++ argv[i++] = NULL; ++ res = xvsystem(argv[0], argv); ++ + if (res) { + opkg_message(conf, OPKG_ERROR, "Failed to download %s, error %d\n", src, res); + free(tmp_file_location); diff --git a/packages/opkg/files/opkg_wget_nogpg_03_fix_tmpdirs.patch b/packages/opkg/files/opkg_wget_nogpg_03_fix_tmpdirs.patch new file mode 100644 index 0000000000..e619562039 --- /dev/null +++ b/packages/opkg/files/opkg_wget_nogpg_03_fix_tmpdirs.patch @@ -0,0 +1,186 @@ +# Further compounding the problem on small-memory machines is that +# opkg does not really honor the TMPDIR environment variable nor +# the command-line option to select the temporary file directory. +# The result is that when /tmp is an in-memory filesystem, the temp +# files it creates further reduce the amount of available memory. +# +# Mike Westerhof, Dec 2008 + +--- orig/libopkg/opkg.c 2008-12-20 15:07:04.000000000 -0600 ++++ opkg/libopkg/opkg.c 2008-12-21 21:41:07.000000000 -0600 +@@ -773,15 +773,17 @@ + } + } + +- tmp = strdup ("/tmp/opkg.XXXXXX"); ++ sprintf_alloc(&tmp, "%s-XXXXXX", opkg->conf->tmp_dir); + + if (mkdtemp (tmp) == NULL) + { + /* XXX: Error: could not create temporary file name */ ++ perror("mkdtemp()"); + free (lists_dir); + free (tmp); + return 1; + } ++ /* printf("%s: Using tmp_dir %s\n", __FUNCTION__, tmp); */ + + /* count the number of sources so we can give some progress updates */ + sources_list_count = 0; +--- orig/libopkg/pkg_extract.c 2008-07-27 16:41:58.000000000 -0500 ++++ opkg/libopkg/pkg_extract.c 2008-12-21 23:32:32.000000000 -0600 +@@ -24,6 +24,8 @@ + #include "file_util.h" + #include "sprintf_alloc.h" + ++extern char *opkg_conf_tmp_name; /* Hack - workaround tmpfile issue */ ++ + /* assuage libb functions */ + const char *applet_name = "opkg"; + +@@ -91,6 +93,8 @@ + char *line, *data_file; + FILE *file; + FILE *tmp; ++ char *tmp_fname; ++ int tmp_f; + + file = fopen(file_name, "w"); + if (file == NULL) { +@@ -99,7 +103,19 @@ + return EINVAL; + } + +- tmp = tmpfile(); ++// tmp = tmpfile(); ++ sprintf_alloc(&tmp_fname, "%s-XXXXXX", opkg_conf_tmp_name); ++ tmp_f = mkstemp(tmp_fname); ++ if (tmp_f == 0) { ++ perror("mkstemp()"); ++ free(tmp_fname); ++ return errno; ++ } ++ /* printf("%s: Using tmp_fname %s\n", __FUNCTION__, tmp_fname); */ ++ unlink(tmp_fname); ++ free(tmp_fname); ++ tmp = fdopen(tmp_f, "w"); ++ + if (pkg->installed_files) { + str_list_elt_t *elt; + for (elt = pkg->installed_files->head; elt; elt = elt->next) { +--- orig/libopkg/opkg_conf.c 2008-12-20 15:06:50.000000000 -0600 ++++ opkg/libopkg/opkg_conf.c 2008-12-21 21:41:07.000000000 -0600 +@@ -101,6 +101,8 @@ + } + } + ++char *opkg_conf_tmp_name; ++ + int opkg_conf_init(opkg_conf_t *conf, const args_t *args) + { + int err; +@@ -153,6 +155,8 @@ + __FUNCTION__, conf->tmp_dir, strerror(errno)); + return OPKG_CONF_ERR_TMP_DIR; + } ++ /* printf("%s: Using tmp_dir %s\n", __FUNCTION__, conf->tmp_dir); */ ++ opkg_conf_tmp_name = strdup(conf->tmp_dir); + + pkg_hash_init("pkg-hash", &conf->pkg_hash, OPKG_CONF_DEFAULT_HASH_LEN); + hash_table_init("file-hash", &conf->file_hash, OPKG_CONF_DEFAULT_HASH_LEN); +--- orig/libopkg/pkg.c 2008-12-20 15:06:50.000000000 -0600 ++++ opkg/libopkg/pkg.c 2008-12-21 23:31:39.000000000 -0600 +@@ -34,6 +34,8 @@ + #include "xsystem.h" + #include "opkg_conf.h" + ++extern char *opkg_conf_tmp_name; /* Hack - workaround tmpfile issue */ ++ + typedef struct enum_map enum_map_t; + struct enum_map + { +@@ -278,13 +280,27 @@ + int err; + char **raw; + FILE *control_file; ++ char *tmp_fname; ++ int tmp_f; + + err = pkg_init(pkg); + if (err) { return err; } + + pkg->local_filename = strdup(filename); + +- control_file = tmpfile(); ++// control_file = tmpfile(); ++ sprintf_alloc(&tmp_fname, "%s-XXXXXX", opkg_conf_tmp_name); ++ tmp_f = mkstemp(tmp_fname); ++ if (tmp_f == 0) { ++ perror("mkstemp()"); ++ free(tmp_fname); ++ return errno; ++ } ++ /* printf("%s: Using tmp_fname %s\n", __FUNCTION__, tmp_fname); */ ++ unlink(tmp_fname); ++ free(tmp_fname); ++ control_file = fdopen(tmp_f, "w"); ++ + err = pkg_extract_control_file_to_stream(pkg, control_file); + if (err) { return err; } + +@@ -1302,6 +1318,8 @@ + char *line; + char *installed_file_name; + int rootdirlen; ++ char *tmp_fname; ++ int tmp_f; + + pkg->installed_files_ref_cnt++; + +@@ -1326,7 +1344,19 @@ + file. In other words, change deb_extract so that it can + simply return the file list as a char *[] rather than + insisting on writing in to a FILE * as it does now. */ +- list_file = tmpfile(); ++// list_file = tmpfile(); ++ sprintf_alloc(&tmp_fname, "%s-XXXXXX", opkg_conf_tmp_name); ++ tmp_f = mkstemp(tmp_fname); ++ if (tmp_f == 0) { ++ perror("mkstemp()"); ++ free(tmp_fname); ++ return pkg->installed_files; ++ } ++ /* printf("%s: Using tmp_fname %s\n", __FUNCTION__, tmp_fname); */ ++ unlink(tmp_fname); ++ free(tmp_fname); ++ list_file = fdopen(tmp_f, "w"); ++ + err = pkg_extract_data_file_names_to_stream(pkg, list_file); + if (err) { + fclose(list_file); +--- orig/libopkg/opkg_cmd.c 2008-12-20 15:07:04.000000000 -0600 ++++ opkg/libopkg/opkg_cmd.c 2008-12-21 21:41:07.000000000 -0600 +@@ -210,12 +210,13 @@ + failures = 0; + + +- tmp = strdup ("/tmp/opkg.XXXXXX"); ++ sprintf_alloc(&tmp, "%s-XXXXXX", conf->tmp_dir); + + if (mkdtemp (tmp) == NULL) { + perror ("mkdtemp"); + failures++; + } ++ /* printf("%s: Using tmp_dir %s\n", __FUNCTION__, tmp); */ + + + for (iter = conf->pkg_src_list.head; iter; iter = iter->next) { +@@ -334,7 +335,7 @@ + + gen = 0; + retry: +- sprintf_alloc (&ctx->statedir, "/tmp/opkg-intercept-%d-%d", getpid (), gen); ++ sprintf_alloc (&ctx->statedir, "%s-intercept-%d-%d", conf->tmp_dir, getpid (), gen); + if (mkdir (ctx->statedir, 0770) < 0) { + if (errno == EEXIST) { + free (ctx->statedir); diff --git a/packages/opkg/opkg-nogpg-nocurl_svn.bb b/packages/opkg/opkg-nogpg-nocurl_svn.bb index 5fa459630d..8560178287 100644 --- a/packages/opkg/opkg-nogpg-nocurl_svn.bb +++ b/packages/opkg/opkg-nogpg-nocurl_svn.bb @@ -5,12 +5,15 @@ PROVIDES += "opkg" SRC_URI += "file://opkg_wget.patch;patch=1 \ file://reduce-nogpg-noise.patch;patch=1 \ + file://opkg_wget_nogpg_01_use_vfork_gunzip.patch;patch=1 \ + file://opkg_wget_nogpg_02_use_vfork_system.patch;patch=1 \ + file://opkg_wget_nogpg_03_fix_tmpdirs.patch;patch=1 \ " -PR = "r0" +PR = "r1" SRCREV = "${SRCREV_pn-opkg}" -EXTRA_OECONF += "--disable-gpg" +EXTRA_OECONF += "--disable-gpg --enable-static --disable-shared" # The nogpg version isn't getting much love and has an unused variable which trips up -Werror do_configure_prepend() { |