From 9396bd4a4c84bde6b55ac3c47c90b4804e51adaf Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 13 Feb 2017 18:07:04 +0100 Subject: [PATCH] libopkg: store checksums in binary form, use integer index for architecture Instead of storing a copy of the architecture string and architecture priority value in each pkg_t instance, declare a 3 bit wide field which allows referencing the architecture in the global config array by index. The 3 bit field allows referencing up to 8 different architectures which is more than enough for the systems we target with opkg. Another nice side effect is that we can coalesce this field with other flag values in pkg_t, saving 4 bytes for an int member. Also convert the hexadecimal checksums to binary format before storing them in pkg_t's blob buffer to save 50% of the space per checksum. Signed-off-by: Jo-Philipp Wich --- libopkg/file_util.c | 52 +++++++++++++++++++++++++++++++ libopkg/file_util.h | 3 ++ libopkg/opkg.c | 2 +- libopkg/opkg_install.c | 2 +- libopkg/pkg.c | 69 +++++++++++++++++++++++++++++++++++++----- libopkg/pkg.h | 9 ++++-- libopkg/pkg_depends.c | 4 +-- libopkg/pkg_hash.c | 15 +++++---- libopkg/pkg_parse.c | 50 +++++++++++++++++------------- libopkg/pkg_vec.c | 4 +-- tests/libopkg_test.c | 4 +-- 11 files changed, 167 insertions(+), 47 deletions(-) diff --git a/libopkg/file_util.c b/libopkg/file_util.c index 4f949ac..912b147 100644 --- a/libopkg/file_util.c +++ b/libopkg/file_util.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "sprintf_alloc.h" #include "file_util.h" @@ -228,6 +229,57 @@ char *file_sha256sum_alloc(const char *file_name) #endif +char *checksum_bin2hex(const char *src, size_t len) +{ + char *p; + static char buf[65]; + static const unsigned char bin2hex[16] = { + '0', '1', '2', '3', + '4', '5', '6', '7', + '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f' + }; + + if (len > 32) + return NULL; + + for (p = buf; len > 0; src++, len--) { + *p++ = bin2hex[*src / 16]; + *p++ = bin2hex[*src % 16]; + } + + *p = 0; + + return buf; +} + +char *checksum_hex2bin(const char *src, size_t *len) +{ + char *p; + size_t slen; + static char buf[32]; + + while (isspace(*src)) + src++; + + slen = strlen(src); + + if (slen > 64) { + *len = 0; + return NULL; + } + +#define hex(c) \ + (c >= 'a' ? (c - 'a') : (c >= 'A' ? (c - 'A') : (c - '0'))) + + for (p = buf, *len = 0; + slen > 0 && isxdigit(src[0]) && isxdigit(src[1]); + slen--, src += 2, (*len)++) + *p++ = hex(src[0]) * 16 + hex(src[1]); + + return buf; +} + int rm_r(const char *path) { int ret = 0; diff --git a/libopkg/file_util.h b/libopkg/file_util.h index a7e2a38..d8e6767 100644 --- a/libopkg/file_util.h +++ b/libopkg/file_util.h @@ -28,4 +28,7 @@ char *file_md5sum_alloc(const char *file_name); char *file_sha256sum_alloc(const char *file_name); int rm_r(const char *path); +char *checksum_bin2hex(const char *src, size_t len); +char *checksum_hex2bin(const char *src, size_t *len); + #endif diff --git a/libopkg/opkg.c b/libopkg/opkg.c index 9e278cd..d8c17cd 100644 --- a/libopkg/opkg.c +++ b/libopkg/opkg.c @@ -749,7 +749,7 @@ pkg_t *opkg_find_package(const char *name, const char *ver, const char *arch, /* check architecture */ if (arch) { - if (sstrcmp(pkg_get_string(pkg, PKG_ARCHITECTURE), arch)) + if (sstrcmp(pkg_get_architecture(pkg), arch)) continue; } diff --git a/libopkg/opkg_install.c b/libopkg/opkg_install.c index 55c124b..85159bf 100644 --- a/libopkg/opkg_install.c +++ b/libopkg/opkg_install.c @@ -1285,7 +1285,7 @@ int opkg_install_pkg(pkg_t * pkg, int from_upgrade) if (!pkg_arch_supported(pkg)) { opkg_msg(ERROR, "INTERNAL ERROR: architecture %s for pkg %s is unsupported.\n", - pkg_get_string(pkg, PKG_ARCHITECTURE), pkg->name); + pkg_get_architecture(pkg), pkg->name); return -1; } if (pkg->state_status == SS_INSTALLED && conf->nodeps == 0) { diff --git a/libopkg/pkg.c b/libopkg/pkg.c index a1ee5a2..0731bd0 100644 --- a/libopkg/pkg.c +++ b/libopkg/pkg.c @@ -85,6 +85,8 @@ static void pkg_init(pkg_t * pkg) pkg->essential = 0; pkg->provided_by_hand = 0; + pkg->arch_index = 0; + blob_buf_init(&pkg->blob, 0); } @@ -157,6 +159,59 @@ char *pkg_set_string(pkg_t *pkg, int id, const char *s) return p; } +char *pkg_get_architecture(const pkg_t *pkg) +{ + nv_pair_list_elt_t *l; + int n = 1; + + list_for_each_entry(l, &conf->arch_list.head, node) { + nv_pair_t *nv = (nv_pair_t *) l->data; + if (n++ == pkg->arch_index) + return nv->name; + } + + return NULL; +} + +char *pkg_set_architecture(pkg_t *pkg, const char *architecture, ssize_t len) +{ + nv_pair_list_elt_t *l; + int n = 1; + + list_for_each_entry(l, &conf->arch_list.head, node) { + nv_pair_t *nv = (nv_pair_t *) l->data; + + if (!strncmp(nv->name, architecture, len) && nv->name[len] == '\0') { + if (n >= 8) { + opkg_msg(ERROR, "Internal error: too many different architectures\n"); + break; + } + + pkg->arch_index = n; + return nv->name; + } + + n++; + } + + pkg->arch_index = 0; + return NULL; +} + +int pkg_get_arch_priority(const pkg_t *pkg) +{ + nv_pair_list_elt_t *l; + int n = 1; + + list_for_each_entry(l, &conf->arch_list.head, node) { + nv_pair_t *nv = (nv_pair_t *) l->data; + if (n++ == pkg->arch_index) + return strtol(nv->value, NULL, 0); + } + + return 0; +} + static void compound_depend_deinit(compound_depend_t * depends) { @@ -292,10 +347,8 @@ int pkg_merge(pkg_t * oldpkg, pkg_t * newpkg) oldpkg->src = newpkg->src; if (!oldpkg->dest) oldpkg->dest = newpkg->dest; - if (!pkg_get_string(oldpkg, PKG_ARCHITECTURE)) - pkg_set_string(oldpkg, PKG_ARCHITECTURE, pkg_get_string(newpkg, PKG_ARCHITECTURE)); - if (!pkg_get_int(oldpkg, PKG_ARCH_PRIORITY)) - pkg_set_int(oldpkg, PKG_ARCH_PRIORITY, pkg_get_int(newpkg, PKG_ARCH_PRIORITY)); + if (!oldpkg->arch_index) + oldpkg->arch_index = newpkg->arch_index; if (!pkg_get_string(oldpkg, PKG_SECTION)) pkg_set_string(oldpkg, PKG_SECTION, pkg_get_string(newpkg, PKG_SECTION)); if (!pkg_get_string(oldpkg, PKG_MAINTAINER)) @@ -544,7 +597,7 @@ void pkg_formatted_field(FILE * fp, pkg_t * pkg, const char *field) case 'a': case 'A': if (strcasecmp(field, "Architecture") == 0) { - p = pkg_get_string(pkg, PKG_ARCHITECTURE); + p = pkg_get_architecture(pkg); if (p) { fprintf(fp, "Architecture: %s\n", p); @@ -983,8 +1036,8 @@ int pkg_name_version_and_architecture_compare(const void *p1, const void *p2) vercmp = pkg_compare_versions(a, b); if (vercmp) return vercmp; - arch_prio1 = pkg_get_int(a, PKG_ARCH_PRIORITY); - arch_prio2 = pkg_get_int(b, PKG_ARCH_PRIORITY); + arch_prio1 = pkg_get_arch_priority(a); + arch_prio2 = pkg_get_arch_priority(b); if (!arch_prio1 || !arch_prio2) { opkg_msg(ERROR, "Internal error: a->arch_priority=%i b->arch_priority=%i.\n", @@ -1301,7 +1354,7 @@ int pkg_run_script(pkg_t * pkg, const char *script, const char *args) int pkg_arch_supported(pkg_t * pkg) { nv_pair_list_elt_t *l; - char *architecture = pkg_get_string(pkg, PKG_ARCHITECTURE); + char *architecture = pkg_get_architecture(pkg); if (!architecture) return 1; diff --git a/libopkg/pkg.h b/libopkg/pkg.h index 2c645aa..df5d7a7 100644 --- a/libopkg/pkg.h +++ b/libopkg/pkg.h @@ -89,8 +89,6 @@ enum pkg_fields { PKG_LOCAL_FILENAME, PKG_VERSION, PKG_REVISION, - PKG_ARCHITECTURE, - PKG_ARCH_PRIORITY, PKG_DESCRIPTION, PKG_MD5SUM, PKG_SHA256SUM, @@ -168,6 +166,8 @@ struct pkg { int auto_installed:1; int is_upgrade:1; + int arch_index:3; + struct blob_buf blob; }; @@ -206,6 +206,11 @@ static inline void * pkg_get_ptr(const pkg_t *pkg, int id) return ptr ? *ptr : NULL; } +char *pkg_set_architecture(pkg_t *pkg, const char *architecture, ssize_t len); +char *pkg_get_architecture(const pkg_t *pkg); +int pkg_get_arch_priority(const pkg_t *pkg); + + abstract_pkg_t *abstract_pkg_new(void); /* diff --git a/libopkg/pkg_depends.c b/libopkg/pkg_depends.c index ebd763a..b2e0df0 100644 --- a/libopkg/pkg_depends.c +++ b/libopkg/pkg_depends.c @@ -510,10 +510,10 @@ static int is_pkg_in_pkg_vec(pkg_vec_t * vec, pkg_t * pkg) int i; char *arch1, *arch2; pkg_t **pkgs = vec->pkgs; - arch1 = pkg_get_string(pkg, PKG_ARCHITECTURE); + arch1 = pkg_get_architecture(pkg); for (i = 0; i < vec->len; i++) { - arch2 = pkg_get_string(*(pkgs + i), PKG_ARCHITECTURE); + arch2 = pkg_get_architecture(*(pkgs + i)); if ((strcmp(pkg->name, (*(pkgs + i))->name) == 0) && (pkg_compare_versions(pkg, *(pkgs + i)) == 0) diff --git a/libopkg/pkg_hash.c b/libopkg/pkg_hash.c index a8809a3..f5c0495 100644 --- a/libopkg/pkg_hash.c +++ b/libopkg/pkg_hash.c @@ -141,8 +141,7 @@ pkg_hash_add_from_file(const char *file_name, continue; } - if (!pkg_get_string(pkg, PKG_ARCHITECTURE) || - !pkg_get_int(pkg, PKG_ARCH_PRIORITY)) { + if (!pkg_get_architecture(pkg) || !pkg_get_arch_priority(pkg)) { char *version_str = pkg_version_str_alloc(pkg); opkg_msg(NOTICE, "Package %s version %s has no " "valid architecture, ignoring.\n", @@ -319,11 +318,11 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg, /* count packages matching max arch priority and keep track of last one */ for (j = 0; j < vec->len; j++) { pkg_t *maybe = vec->pkgs[j]; - arch_priority = pkg_get_int(maybe, PKG_ARCH_PRIORITY); + arch_priority = pkg_get_arch_priority(maybe); opkg_msg(DEBUG, "%s arch=%s arch_priority=%d version=%s.\n", - maybe->name, pkg_get_string(maybe, PKG_ARCHITECTURE), + maybe->name, pkg_get_architecture(maybe), arch_priority, pkg_get_string(maybe, PKG_VERSION)); /* We make sure not to add the same package twice. Need to search for the reason why they show up twice sometimes. */ @@ -393,7 +392,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg, int prio = 0; for (i = 0; i < matching_pkgs->len; i++) { pkg_t *matching = matching_pkgs->pkgs[i]; - arch_priority = pkg_get_int(matching, PKG_ARCH_PRIORITY); + arch_priority = pkg_get_arch_priority(matching); if (arch_priority > prio) { priorized_matching = matching; prio = arch_priority; @@ -411,7 +410,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg, pkg_t *matching = matching_pkgs->pkgs[i]; opkg_msg(INFO, "%s %s %s\n", matching->name, pkg_get_string(matching, PKG_VERSION), - pkg_get_string(matching, PKG_ARCHITECTURE)); + pkg_get_architecture(matching)); } } @@ -437,7 +436,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg, if (priorized_matching) { opkg_msg(INFO, "Using priorized matching %s %s %s.\n", priorized_matching->name, pkg_get_string(priorized_matching, PKG_VERSION), - pkg_get_string(priorized_matching, PKG_ARCHITECTURE)); + pkg_get_architecture(priorized_matching)); return priorized_matching; } if (nmatching > 1) { @@ -448,7 +447,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg, if (latest_matching) { opkg_msg(INFO, "Using latest matching %s %s %s.\n", latest_matching->name, pkg_get_string(latest_matching, PKG_VERSION), - pkg_get_string(latest_matching, PKG_ARCHITECTURE)); + pkg_get_architecture(latest_matching)); return latest_matching; } return NULL; diff --git a/libopkg/pkg_parse.c b/libopkg/pkg_parse.c index cfa551e..e0c0069 100644 --- a/libopkg/pkg_parse.c +++ b/libopkg/pkg_parse.c @@ -28,6 +28,7 @@ #include "pkg_depends.h" #include "libbb/libbb.h" +#include "file_util.h" #include "parse_util.h" static void parse_status(pkg_t * pkg, const char *sstr) @@ -94,16 +95,20 @@ int parse_version(pkg_t * pkg, const char *vstr) return 0; } -static int get_arch_priority(const char *arch) +static char *parse_architecture(pkg_t *pkg, const char *str) { - nv_pair_list_elt_t *l; + const char *s = str; + const char *e; - list_for_each_entry(l, &conf->arch_list.head, node) { - nv_pair_t *nv = (nv_pair_t *) l->data; - if (strcmp(nv->name, arch) == 0) - return strtol(nv->value, NULL, 0); - } - return 0; + while (isspace(*s)) + s++; + + e = s + strlen(s); + + while (e > s && isspace(*e)) + e--; + + return pkg_set_architecture(pkg, s, e - s); } int pkg_parse_line(void *ptr, const char *line, uint mask) @@ -114,6 +119,7 @@ int pkg_parse_line(void *ptr, const char *line, uint mask) /* these flags are a bit hackish... */ static int reading_conffiles = 0, reading_description = 0; static char *description = NULL; + char *s; int ret = 0; /* Exclude globally masked fields. */ @@ -124,11 +130,9 @@ int pkg_parse_line(void *ptr, const char *line, uint mask) switch (*line) { case 'A': - if ((mask & PFM_ARCHITECTURE) && is_field("Architecture", line)) { - pkg_set_int(pkg, PKG_ARCH_PRIORITY, get_arch_priority( - pkg_set_string(pkg, PKG_ARCHITECTURE, line + strlen("Architecture") + 1))); - - } else if ((mask & PFM_AUTO_INSTALLED) + if ((mask & PFM_ARCHITECTURE) && is_field("Architecture", line)) + parse_architecture(pkg, line + strlen("Architecture") + 1); + else if ((mask & PFM_AUTO_INSTALLED) && is_field("Auto-Installed", line)) { char *tmp = parse_simple("Auto-Installed", line); if (strcmp(tmp, "yes") == 0) @@ -182,12 +186,12 @@ int pkg_parse_line(void *ptr, const char *line, uint mask) break; case 'M': - if ((mask & PFM_MD5SUM) && is_field("MD5sum:", line)) - pkg_set_string(pkg, PKG_MD5SUM, line + strlen("MD5sum") + 1); - /* The old opkg wrote out status files with the wrong - * case for MD5sum, let's parse it either way */ - else if ((mask & PFM_MD5SUM) && is_field("MD5Sum:", line)) - pkg_set_string(pkg, PKG_MD5SUM, line + strlen("MD5Sum") + 1); + if ((mask & PFM_MD5SUM) && (is_field("MD5sum:", line) || is_field("MD5Sum:", line))) { + size_t len; + char *cksum = checksum_hex2bin(line + strlen("MD5sum") + 1, &len); + if (cksum && len == 16) + pkg_set_raw(pkg, PKG_MD5SUM, cksum, len); + } else if ((mask & PFM_MAINTAINER) && is_field("Maintainer", line)) pkg_set_string(pkg, PKG_MAINTAINER, line + strlen("Maintainer") + 1); @@ -216,8 +220,12 @@ int pkg_parse_line(void *ptr, const char *line, uint mask) if ((mask & PFM_SECTION) && is_field("Section", line)) pkg_set_string(pkg, PKG_SECTION, line + strlen("Section") + 1); #ifdef HAVE_SHA256 - else if ((mask & PFM_SHA256SUM) && is_field("SHA256sum", line)) - pkg_set_string(pkg, PKG_SHA256SUM, line + strlen("SHA256sum") + 1); + else if ((mask & PFM_SHA256SUM) && is_field("SHA256sum", line)) { + size_t len; + char *cksum = checksum_hex2bin(line + strlen("SHA256sum") + 1, &len); + if (cksum && len == 32) + pkg_set_raw(pkg, PKG_SHA256SUM, cksum, len); + } #endif else if ((mask & PFM_SIZE) && is_field("Size", line)) { pkg_set_int(pkg, PKG_SIZE, strtoul(line + strlen("Size") + 1, NULL, 0)); diff --git a/libopkg/pkg_vec.c b/libopkg/pkg_vec.c index d37a185..a60c5ab 100644 --- a/libopkg/pkg_vec.c +++ b/libopkg/pkg_vec.c @@ -53,12 +53,12 @@ void pkg_vec_insert_merge(pkg_vec_t * vec, pkg_t * pkg, int set_status) int i; int found = 0; char *pkg_version = pkg_get_string(pkg, PKG_VERSION); - char *pkg_architecture = pkg_get_string(pkg, PKG_ARCHITECTURE); + char *pkg_architecture = pkg_get_architecture(pkg); char *vec_architecture; /* look for a duplicate pkg by name, version, and architecture */ for (i = 0; i < vec->len; i++) { - vec_architecture = pkg_get_string(vec->pkgs[i], PKG_ARCHITECTURE); + vec_architecture = pkg_get_architecture(vec->pkgs[i]); opkg_msg(DEBUG2, "%s %s arch=%s vs. %s %s arch=%s.\n", pkg->name, pkg_version, pkg_architecture, diff --git a/tests/libopkg_test.c b/tests/libopkg_test.c index ab15eb7..0d6a703 100644 --- a/tests/libopkg_test.c +++ b/tests/libopkg_test.c @@ -70,7 +70,7 @@ void print_package(pkg_t * pkg) pkg->name, v, pkg->src->name, - pkg_get_string(pkg, PKG_ARCHITECTURE), + pkg_get_architecture(pkg), pkg_get_string(pkg, PKG_DESCRIPTION), tags ? tags : "", (unsigned long) pkg_get_int(pkg, PKG_SIZE), pkg->state_status); @@ -93,7 +93,7 @@ void opkg_test(void) pkg = opkg_find_package(find_pkg->name, pkg_get_string(find_pkg, PKG_VERSION), - pkg_get_string(find_pkg, PKG_ARCHITECTURE), + pkg_get_architecture(find_pkg), find_pkg->src->name); if (pkg) { print_package(pkg); -- 2.25.1