Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core: Compare rpm file coloring within layered packages #3161

Merged
merged 5 commits into from
Oct 14, 2021
Merged

core: Compare rpm file coloring within layered packages #3161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5ae0aeb
core: Inline back add_te_files_to_ht function
jlebon Nov 13, 2020
862e2e9
core: Track file disposition by NEVRA
jlebon Oct 6, 2021
ab2a143
core: Compare rpm file coloring within layered packages
jlebon Oct 6, 2021
d76aafa
core: Free files_skip_add hash table after we're done
jlebon Oct 8, 2021
19ccb7d
core: Handle RPM file colors more efficiently
jlebon Oct 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 113 additions & 44 deletions src/libpriv/rpmostree-core.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2443,22 +2443,6 @@ rpmostree_context_consume_package (RpmOstreeContext *self,
return TRUE;
}

/* Builds a mapping from filename to rpm color */
static void
add_te_files_to_ht (rpmte te,
GHashTable *ht)
{
g_auto(rpmfiles) files = rpmteFiles (te);
g_auto(rpmfi) fi = rpmfilesIter (files, RPMFI_ITER_FWD);

while (rpmfiNext (fi) >= 0)
{
const char *fn = rpmfiFN (fi);
rpm_color_t color = rpmfiFColor (fi);
g_hash_table_insert (ht, g_strdup (fn), GUINT_TO_POINTER (color));
}
}

static char*
canonicalize_rpmfi_path (const char *path)
{
Expand All @@ -2485,6 +2469,18 @@ canonicalize_non_usrmove_path (RpmOstreeContext *self,
return g_build_filename (link, slash + 1, NULL);
}

static void
ht_insert_path_for_nevra (GHashTable *ht, const char *nevra, char *path, gpointer v)
{
auto paths = static_cast<GHashTable*>(g_hash_table_lookup (ht, nevra));
if (!paths)
{
paths = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_hash_table_insert (ht, g_strdup (nevra), paths);
}
g_hash_table_insert (paths, path, v);
}

/* This is a lighter version of calculations that librpm calls "file disposition".
* Essentially, we determine which file removals/installations should be skipped. The librpm
* functions and APIs for these are unfortunately private since they're just run as part of
Expand All @@ -2499,41 +2495,68 @@ handle_file_dispositions (RpmOstreeContext *self,
GCancellable *cancellable,
GError **error)
{
/* we deal with color similarly to librpm (compare with skipInstallFiles()) */
rpm_color_t ts_color = rpmtsColor (ts);
rpm_color_t ts_prefcolor = rpmtsPrefColor (ts);

if (ts_color == 0)
return TRUE; /* this architecture doesn't do colours; done */

g_autoptr(GHashTable) pkgs_deleted = g_hash_table_new (g_direct_hash, g_direct_equal);

/* note these entries are *not* canonicalized for ostree conventions */
g_autoptr(GHashTable) files_deleted =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_autoptr(GHashTable) files_added =
g_autoptr(GHashTable) files_deleted = /* set{paths} */
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_autoptr(GHashTable) files_added = /* map{nevra -> map{path -> color}} */
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_hash_table_unref);

/* first pass to just collect added and removed files */
const guint n_rpmts_elements = (guint)rpmtsNElements (ts);
for (guint i = 0; i < n_rpmts_elements; i++)
{
rpmte te = rpmtsElement (ts, i);
rpmElementType type = rpmteType (te);
if (type == TR_REMOVED)
g_hash_table_add (pkgs_deleted, GUINT_TO_POINTER (rpmteDBInstance (te)));
add_te_files_to_ht (te, type == TR_REMOVED ? files_deleted : files_added);

g_auto(rpmfiles) files = rpmteFiles (te);
g_auto(rpmfi) fi = rpmfilesIter (files, RPMFI_ITER_FWD);

if (type == TR_REMOVED)
{
while (rpmfiNext (fi) >= 0)
g_hash_table_add (files_deleted, g_strdup (rpmfiFN (fi)));
}
else
{
DnfPackage *pkg = (DnfPackage*)rpmteKey (te);
const char *nevra = dnf_package_get_nevra (pkg);
while (rpmfiNext (fi) >= 0)
{
char *fn = g_strdup (rpmfiFN (fi));
rpm_color_t color = rpmfiFColor (fi);
if (color)
ht_insert_path_for_nevra (files_added, nevra, fn, GUINT_TO_POINTER (color));
}
}
}

/* this we *do* canonicalize since we'll be comparing against ostree paths */
g_autoptr(GHashTable) files_skip_add =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_autoptr(GHashTable) files_skip_add = /* map{nevra -> set{files}} */
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_hash_table_unref);
/* this one we *don't* canonicalize since we'll be comparing against rpmfi paths */
g_autoptr(GHashTable) files_skip_delete =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);

/* we deal with color similarly to librpm (compare with skipInstallFiles()) */
rpm_color_t ts_color = rpmtsColor (ts);
rpm_color_t ts_prefcolor = rpmtsPrefColor (ts);

/* ignore colored files not in our rainbow */
GLNX_HASH_TABLE_FOREACH_IT (files_added, it, const char*, fn, gpointer, colorp)
/* skip added files whose colors aren't in our rainbow */
GLNX_HASH_TABLE_FOREACH_KV (files_added, const char*, nevra, GHashTable*, paths)
{
rpm_color_t color = GPOINTER_TO_UINT (colorp);
if (color && ts_color && !(ts_color & color))
g_hash_table_add (files_skip_add, canonicalize_rpmfi_path (fn));
GLNX_HASH_TABLE_FOREACH_KV (paths, const char*, path, gpointer, colorp)
{
rpm_color_t color = GPOINTER_TO_UINT (colorp);
if (!(ts_color & color))
ht_insert_path_for_nevra (files_skip_add, nevra, canonicalize_rpmfi_path (path), NULL);
}
}

g_auto(rpmdbMatchIterator) it = rpmtsInitIterator (ts, RPMDBI_PACKAGES, NULL, 0);
Expand Down Expand Up @@ -2562,6 +2585,8 @@ handle_file_dispositions (RpmOstreeContext *self,
g_hash_table_add (files_skip_delete, g_strdup (fn));

rpm_color_t color = (rpmfiFColor (fi) & ts_color);
if (!color)
continue;

/* let's make the safe assumption that the color mess is only an issue for /usr */
const char *fn_rel = fn + strspn (fn, "/");
Expand All @@ -2574,24 +2599,64 @@ handle_file_dispositions (RpmOstreeContext *self,
if (!g_str_has_prefix (fn_rel, "usr/"))
continue;

/* check if one of the pkgs to install wants to overwrite our file */
rpm_color_t other_color =
GPOINTER_TO_UINT (g_hash_table_lookup (files_added, fn));
other_color &= ts_color;
/* check if any of the pkgs to install want to overwrite our file */
GLNX_HASH_TABLE_FOREACH_KV (files_added, const char*, nevra, GHashTable*, paths)
{
gpointer other_colorp = NULL;
if (!g_hash_table_lookup_extended (paths, fn, NULL, &other_colorp))
continue;

rpm_color_t other_color = GPOINTER_TO_UINT (other_colorp);
other_color &= ts_color;

/* see handleColorConflict() */
if (color && other_color && (color != other_color))
{
/* do we already have the preferred color installed? */
if (color & ts_prefcolor)
ht_insert_path_for_nevra (files_skip_add, nevra, canonicalize_rpmfi_path (fn), NULL);
else if (other_color & ts_prefcolor)
{
/* the new pkg is bringing our favourite color, give way now so we let
* checkout silently write into it */
if (!glnx_shutil_rm_rf_at (tmprootfs_dfd, fn_rel, cancellable, error))
return FALSE;
}
}
}
}
}

/* and finally, iterate over added pkgs only to search for duplicate files of
* differing rpm colors for which we have to pick one */
g_autoptr(GHashTable) path_to_nevra = g_hash_table_new (g_str_hash, g_str_equal);
g_autoptr(GHashTable) path_to_color = g_hash_table_new (g_str_hash, g_str_equal);
GLNX_HASH_TABLE_FOREACH_KV (files_added, const char*, nevra, GHashTable*, paths)
{
GLNX_HASH_TABLE_FOREACH_KV (paths, const char*, path, gpointer, colorp)
{
if (!g_hash_table_contains (path_to_nevra, path))
{
g_hash_table_insert (path_to_nevra, (gpointer)path, (gpointer)nevra);
g_hash_table_insert (path_to_color, (gpointer)path, colorp);
continue;
}

rpm_color_t color = GPOINTER_TO_UINT (colorp) & ts_color;
const char *other_nevra = static_cast<const char*>(g_hash_table_lookup (path_to_nevra, path));
rpm_color_t other_color = GPOINTER_TO_UINT (g_hash_table_lookup (path_to_color, path)) & ts_color;

/* see handleColorConflict() */
if (color && other_color && (color != other_color))
{
/* do we already have the preferred color installed? */
if (color & ts_prefcolor)
g_hash_table_add (files_skip_add, canonicalize_rpmfi_path (fn));
else if (other_color & ts_prefcolor)
{
/* the new pkg is bringing our favourite color, give way now so we let
* checkout silently write into it */
if (!glnx_shutil_rm_rf_at (tmprootfs_dfd, fn_rel, cancellable, error))
return FALSE;
ht_insert_path_for_nevra (files_skip_add, other_nevra, canonicalize_rpmfi_path (path), NULL);
g_hash_table_insert (path_to_nevra, (gpointer)path, (gpointer)nevra);
g_hash_table_insert (path_to_color, (gpointer)path, colorp);
}
else if (other_color & ts_prefcolor)
ht_insert_path_for_nevra (files_skip_add, nevra, canonicalize_rpmfi_path (path), NULL);
}
}
}
Expand Down Expand Up @@ -2724,8 +2789,11 @@ checkout_package_into_root (RpmOstreeContext *self,
}
}

GHashTable *pkg_files_skip = NULL;
if (files_skip != NULL)
pkg_files_skip = static_cast<GHashTable*>(g_hash_table_lookup (files_skip, dnf_package_get_nevra (pkg)));
if (!checkout_package (pkgcache_repo, dfd, path,
devino_cache, pkg_commit, files_skip, files_remove_regex, ovwmode,
devino_cache, pkg_commit, pkg_files_skip, files_remove_regex, ovwmode,
!self->enable_rofiles,
cancellable, error))
return glnx_prefix_error (error, "Checkout %s", dnf_package_get_nevra (pkg));
Expand Down Expand Up @@ -2837,7 +2905,7 @@ delete_package_from_root (RpmOstreeContext *self,
S_ISDIR (mode)))
continue;

if (g_hash_table_contains (files_skip, fn))
if (files_skip && g_hash_table_contains (files_skip, fn))
continue;

g_assert (fn != NULL);
Expand Down Expand Up @@ -4097,6 +4165,7 @@ rpmostree_context_assemble (RpmOstreeContext *self,
n_rpmts_done++;
progress->nitems_update(n_rpmts_done);
}
g_clear_pointer (&files_skip_add, g_hash_table_unref);

progress->end("");

Expand Down
7 changes: 7 additions & 0 deletions tests/vmcheck/test-layering-rpmdb.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,13 @@ check_bloop_color() {
# we use -nostdlib so we don't pull in glibc.i686
vm_build_rpm bloop arch i686 build 'echo "void main() {}" | gcc -m32 -nostdlib -o bloop -xc -'
vm_build_rpm bloop arch x86_64 build 'echo "void main() {}" | gcc -m64 -nostdlib -o bloop -xc -'

# first try to install both of them
vm_rpmostree install bloop.i686 bloop.x86_64
check_bloop_color 64-bit
vm_rpmostree cleanup -p
echo "ok coloring layered"

# now embed the x86_64 version into it
vm_rpmostree install bloop.x86_64
vm_cmd ostree commit -b vmcheck --tree=ref=$(vm_get_pending_csum) --fsync=no
Expand Down