Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(web): synchronise metadata and album membership between duplicate images #13851

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+189 −19
Open

feat(web): synchronise metadata and album membership between duplicate images #13851

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bcd98d7
feat(web): Synchronize albums from deduplicated images
EinToni Oct 30, 2024
5f07eee
Fix mixup of descriptions
EinToni Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions i18n/en.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1192,6 +1192,7 @@
"show_slideshow_transition": "Show slideshow transition",
"show_supporter_badge": "Supporter badge",
"show_supporter_badge_description": "Show a supporter badge",
"show_synchronise_albums": "Synchronize the albums of all images.",
"shuffle": "Shuffle",
"sidebar": "Sidebar",
"sidebar_display_description": "Display a link to the view in the sidebar",
Expand Down Expand Up @@ -1237,6 +1238,12 @@
"support_third_party_description": "Your Immich installation was packaged by a third-party. Issues you experience may be caused by that package, so please raise issues with them in the first instance using the links below.",
"swap_merge_direction": "Swap merge direction",
"sync": "Sync",
"synchronize_albums" : "Synchronize Albums",
"synchronize_albums_description" : "All deduplicates get put in all albums of all duplicates.",
"synchronize_archives" : "Synchronize Archives",
"synchronize_archives_description" : "If one of the duplicates is a favorite, mark all deduplicated as favorites.",
"synchronize_favorites" : "Synchronize Favorites",
"synchronize_favorites_description" : "If one of the duplicates is already archived, also archive the deduplicated result.",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mix up of "synchronize_favorites_description" and "synchronize_archives_description".

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed it

"tag": "Tag",
"tag_assets": "Tag assets",
"tag_created": "Created tag: {tag}",
Expand Down
52 changes: 45 additions & 7 deletions web/src/lib/components/utilities-page/duplicates/duplicate-asset.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,47 @@
import { getAssetResolution, getFileSize } from '$lib/utils/asset-utils';
import { getAltText } from '$lib/utils/thumbnail-util';
import { type AssetResponseDto, getAllAlbums } from '@immich/sdk';
import { mdiHeart, mdiImageMultipleOutline, mdiMagnifyPlus } from '@mdi/js';
import { mdiHeart, mdiImageMultipleOutline, mdiMagnifyPlus, mdiArchiveArrowDownOutline } from '@mdi/js';
import { t } from 'svelte-i18n';
import type { SelectedSyncData } from '$lib/components/utilities-page/duplicates/duplicates-compare-control.svelte';

interface Props {
asset: AssetResponseDto;
isSelected: boolean;
isSynchronizeAlbumsActive: boolean;
isSynchronizeFavoritesActive: boolean;
isSynchronizeArchivesActive: boolean;
selectedSyncData: SelectedSyncData;
onSelectAsset: (asset: AssetResponseDto) => void;
onViewAsset: (asset: AssetResponseDto) => void;
}

let { asset, isSelected, onSelectAsset, onViewAsset }: Props = $props();
let {
asset,
isSelected,
isSynchronizeAlbumsActive,
isSynchronizeFavoritesActive,
isSynchronizeArchivesActive,
selectedSyncData = $bindable(),
onSelectAsset,
onViewAsset,
}: Props = $props();

let isFromExternalLibrary = $derived(!!asset.libraryId);
let assetData = $derived(JSON.stringify(asset, null, 2));

selectedSyncData.isFavorite = selectedSyncData.isFavorite || asset.isFavorite;
selectedSyncData.isArchived = selectedSyncData.isArchived || asset.isArchived;
let displayedFavorite = $derived(
isSelected && isSynchronizeFavoritesActive && selectedSyncData?.isFavorite
? selectedSyncData.isFavorite
: asset.isFavorite,
);
let displayedArchived = $derived(
isSelected && isSynchronizeArchivesActive && selectedSyncData?.isArchived
? selectedSyncData.isArchived
: asset.isArchived,
);
</script>

<div
Expand All @@ -43,11 +70,22 @@
/>

<!-- FAVORITE ICON -->
{#if asset.isFavorite}
<div class="absolute bottom-2 left-2">
<Icon path={mdiHeart} size="24" class="text-white" />
</div>
{/if}
<div class="absolute bottom-2 left-2">
{#if displayedFavorite}
{#if asset.isFavorite}
<Icon path={mdiHeart} size="24" class="text-white inline-block" />
{:else}
<Icon path={mdiHeart} size="24" color="red" class="text-white inline-block" />
{/if}
{/if}
{#if displayedArchived}
{#if asset.isArchived}
<Icon path={mdiArchiveArrowDownOutline} size="24" class="text-white inline-block" />
{:else}
<Icon path={mdiArchiveArrowDownOutline} size="24" color="red" class="text-white inline-block" />
{/if}
{/if}
</div>

<!-- OVERLAY CHIP -->
<div
Expand Down
49 changes: 49 additions & 0 deletions web/src/lib/components/utilities-page/duplicates/duplicate-options.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<script lang="ts">
import FullScreenModal from '$lib/components/shared-components/full-screen-modal.svelte';
import SettingSwitch from '$lib/components/shared-components/settings/setting-switch.svelte';
import { t } from 'svelte-i18n';

interface Props {
synchronizeAlbums: boolean;
synchronizeArchives: boolean;
synchronizeFavorites: boolean;
onClose: () => void;
onToggleSyncAlbum: () => void;
onToggleSyncArchives: () => void;
onToggleSyncFavorites: () => void;
}
let {
synchronizeAlbums,
synchronizeArchives,
synchronizeFavorites,
onClose,
onToggleSyncAlbum,
onToggleSyncArchives,
onToggleSyncFavorites,
}: Props = $props();
</script>

<FullScreenModal title={$t('options')} width="auto" {onClose}>
<div class="items-center justify-center">
<div class="grid p-2 gap-y-2">
<SettingSwitch
title={$t('synchronize_albums')}
subtitle={$t('synchronize_albums_description')}
checked={synchronizeAlbums}
onToggle={onToggleSyncAlbum}
/>
<SettingSwitch
title={$t('synchronize_favorites')}
subtitle={$t('synchronize_favorites_description')}
checked={synchronizeFavorites}
onToggle={onToggleSyncFavorites}
/>
<SettingSwitch
title={$t('synchronize_archives')}
subtitle={$t('synchronize_archives_description')}
checked={synchronizeArchives}
onToggle={onToggleSyncArchives}
/>
</div>
</div>
</FullScreenModal>
29 changes: 26 additions & 3 deletions web/src/lib/components/utilities-page/duplicates/duplicates-compare-control.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,35 @@

interface Props {
assets: AssetResponseDto[];
onResolve: (duplicateAssetIds: string[], trashIds: string[]) => void;
isSynchronizeAlbumsActive: boolean;
isSynchronizeArchivesActive: boolean;
isSynchronizeFavoritesActive: boolean;
onResolve: (duplicateAssetIds: string[], trashIds: string[], selectedDataToSync: SelectedSyncData) => void;
onStack: (assets: AssetResponseDto[]) => void;
}

let { assets, onResolve, onStack }: Props = $props();
let {
assets,
isSynchronizeAlbumsActive,
isSynchronizeArchivesActive,
isSynchronizeFavoritesActive,
onResolve,
onStack,
}: Props = $props();

const { isViewing: showAssetViewer, asset: viewingAsset, setAsset } = assetViewingStore;
const getAssetIndex = (id: string) => assets.findIndex((asset) => asset.id === id);

let selectedAssetIds = $state(new SvelteSet<string>());
let trashCount = $derived(assets.length - selectedAssetIds.size);
export interface SelectedSyncData {
isArchived: boolean | null;
isFavorite: boolean | null;
}
let selectedSyncData: SelectedSyncData = $state({
isArchived: null,
isFavorite: null,
});

onMount(() => {
const suggestedAsset = suggestDuplicate(assets);
Expand Down Expand Up @@ -89,7 +108,7 @@
const handleResolve = () => {
const trashIds = assets.map((asset) => asset.id).filter((id) => !selectedAssetIds.has(id));
const duplicateAssetIds = assets.map((asset) => asset.id);
onResolve(duplicateAssetIds, trashIds);
onResolve(duplicateAssetIds, trashIds, selectedSyncData);
};

const handleStack = () => {
Expand Down Expand Up @@ -118,8 +137,12 @@
<DuplicateAsset
{asset}
{onSelectAsset}
bind:selectedSyncData
isSelected={selectedAssetIds.has(asset.id)}
onViewAsset={(asset) => setAsset(asset)}
{isSynchronizeAlbumsActive}
{isSynchronizeFavoritesActive}
{isSynchronizeArchivesActive}
/>
{/each}
</div>
Expand Down
71 changes: 62 additions & 9 deletions web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,38 @@
} from '$lib/components/shared-components/notification/notification';
import ShowShortcuts from '$lib/components/shared-components/show-shortcuts.svelte';
import DuplicatesCompareControl from '$lib/components/utilities-page/duplicates/duplicates-compare-control.svelte';
import { locale } from '$lib/stores/preferences.store';
import type { AssetResponseDto, AlbumResponseDto, AssetBulkUpdateDto } from '@immich/sdk';
import { deleteAssets, updateAssets, getAllAlbums } from '@immich/sdk';
import { featureFlags } from '$lib/stores/server-config.store';
import { stackAssets } from '$lib/utils/asset-utils';
import { stackAssets, addAssetsToAlbum } from '$lib/utils/asset-utils';
import { suggestDuplicate } from '$lib/utils/duplicate-utils';
import { handleError } from '$lib/utils/handle-error';
import type { AssetResponseDto } from '@immich/sdk';
import { deleteAssets, updateAssets } from '@immich/sdk';
import { Button, HStack, IconButton, Text } from '@immich/ui';
import { mdiCheckOutline, mdiInformationOutline, mdiKeyboard, mdiTrashCanOutline } from '@mdi/js';
import { t } from 'svelte-i18n';
import type { PageData } from './$types';
import { mdiCheckOutline, mdiInformationOutline, mdiKeyboard, mdiTrashCanOutline, mdiCogOutline } from '@mdi/js';
import DuplicateOptions from '$lib/components/utilities-page/duplicates/duplicate-options.svelte';
import { locale } from '$lib/stores/preferences.store';
import type { SelectedSyncData } from '$lib/components/utilities-page/duplicates/duplicates-compare-control.svelte';

interface Props {
data: PageData;
isShowKeyboardShortcut?: boolean;
isShowDuplicateInfo?: boolean;
isShowOptions?: boolean;
}

let {
data = $bindable(),
isShowKeyboardShortcut = $bindable(false),
isShowDuplicateInfo = $bindable(false),
isShowOptions = $bindable(false),
}: Props = $props();

let isSynchronizeAlbumsActive = $state(true);
let isSynchronizeFavoritesActive = $state(true);
let isSynchronizeArchivesActive = $state(true);

interface Shortcuts {
general: ExplainedShortcut[];
actions: ExplainedShortcut[];
Expand Down Expand Up @@ -84,11 +92,29 @@
});
};

const handleResolve = async (duplicateId: string, duplicateAssetIds: string[], trashIds: string[]) => {
const handleResolve = async (
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

handleDeduplicateAll isn't updated to respect the new settings. Ideally, both bulk and individual resolutions should behave the same.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

locally hacked up a version that does this. Definitely unpolished and imperfect, but sharing here in case there's anything useful.

I'm not really convinced that 'fetch all the album memberships up front' is the most efficient approach vs requesting per photo. Probably depends on the number of dupes vs overall album size. Might make sense when there are tons of dupes, but otherwise not.

const handleDeduplicateAll = async () => {
    console.log("dedupe all");
    
    const albumIdToAddedAssets = new Map<string, Set<string>>();
    const assetIdToAlbums = new Map<string, Set<string>>();

    {
      const relevantAssetIds = new Set<string>();
      duplicates.forEach((group) => group.assets.forEach((asset) => relevantAssetIds.add(asset.id)));

      const allAlbums1 = await getAllAlbums({});
      const allAlbums = await Promise.all(allAlbums1.map((album) => getAlbumInfo({
          id: album.id,
          withoutAssets: false
        })));
      allAlbums.forEach((album) => {
        albumIdToAddedAssets.set(album.id, new Set<string>());
        album.assets.forEach((asset) => {
          if (relevantAssetIds.has(asset.id)) {
            if (!assetIdToAlbums.has(asset.id)) {
              assetIdToAlbums.set(asset.id, new Set<string>());
            }

            assetIdToAlbums.get(asset.id)?.add(album.id);
          }

        })
      });
    }

    debugger;

    let newFavorites = new Array();
    const idsToDelete = new Set<string>();
    const idsToKeep = new Array();

    duplicates.forEach((group) => {
      
      const assetToKeep = suggestDuplicateByFileSize(group.assets);
      const assetsToDelete = group.assets.filter((asset) => asset.id !== assetToKeep.id);
      
      if (assetToKeep.exifInfo.dateTimeOriginal > '2017-01-01'
         && assetsToDelete.some((asset) => asset.originalFileName !== assetToKeep.originalFileName)
         && assetsToDelete.some((asset) => asset.exifInfo.dateTimeOriginal !== assetToKeep.exifInfo.dateTimeOriginal)) {
        return;
      }

      idsToKeep.push(assetToKeep);

      assetsToDelete.forEach((asset) => idsToDelete.add(asset.id));

      const albumIds = [...new Set(assetsToDelete.filter((asset) => assetIdToAlbums.has(asset.id)).flatMap((asset) => [...assetIdToAlbums.get(asset.id)]))];

      albumIds.forEach((albumId) => {
        if (assetIdToAlbums.has(assetToKeep.id) && assetIdToAlbums.get(assetToKeep.id)?.has(albumId)) {
          return;
        }

        albumIdToAddedAssets.get(albumId).add(assetToKeep.id);
      });

      if (!assetToKeep.isFavorite && assetsToDelete.some(asset => asset.isFavorite)) {
        newFavorites.push(assetToKeep.id);
      }
    });

    debugger;
    await updateAssets({assetBulkUpdateDto: { ids: newFavorites, isFavorite: true}});

    debugger;
    await Promise.all(albumIdToAddedAssets.entries().filter((entry) => entry[1].size > 0).map((entry) => addAssetsToAlbum(entry[0], [...entry[1]], false)));

    debugger;

    await deleteAssets({ assetBulkDeleteDto: { ids: Array.from(idsToDelete), force: !$featureFlags.trash } });
    debugger;
    await updateAssets({
      assetBulkUpdateDto: {
        ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)],
        duplicateId: null,
      },
    });

    duplicates = [];

    deletedNotification(idsToDelete.length);

    return;
  };

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Honestly, this should just be handled in the backend. All these queries and set operations can be handled much more efficiently with a single specialized DB query.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But if we automatically sync album membership while deduplicating single images, it should also be done when deduplicating all. Otherwise this would be quite unexpected behavior for the end user.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not saying we shouldn't handle syncing when deduplicating all assets, just that the client is not a great place to be doing this. The DB is much better at doing these operations than JS, can do it without moving all of this data back and forth, and can make the entire process atomic. The bulk dedupe arguably should have been a server endpoint from the start, but with this extra layer of functionality it's just too much to keep in the client.

duplicateId: string,
duplicateAssetIds: string[],
trashIds: string[],
selectedDataToSync: SelectedSyncData,
) => {
return withConfirmation(
async () => {
let assetBulkUpdate: AssetBulkUpdateDto = {
ids: duplicateAssetIds,
duplicateId: null,
};
if (isSynchronizeAlbumsActive) {
await synchronizeAlbums(duplicateAssetIds);
}
if (isSynchronizeArchivesActive) {
assetBulkUpdate.isArchived = selectedDataToSync.isArchived;
}
if (isSynchronizeFavoritesActive) {
assetBulkUpdate.isFavorite = selectedDataToSync.isFavorite;
}
await deleteAssets({ assetBulkDeleteDto: { ids: trashIds, force: !$featureFlags.trash } });
await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } });
await updateAssets({ assetBulkUpdateDto: assetBulkUpdate });

duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);

Expand All @@ -99,6 +125,17 @@
);
};

const synchronizeAlbums = async (assetIds: string[]) => {
const allAlbums: AlbumResponseDto[] = await Promise.all(
assetIds.map((assetId) => getAllAlbums({ assetId: assetId })),
);
const albumIds = [...new Set(allAlbums.flat().map((album) => album.id))];

albumIds.forEach((albumId) => {
addAssetsToAlbum(albumId, assetIds, false);
});
};

const handleStack = async (duplicateId: string, assets: AssetResponseDto[]) => {
await stackAssets(assets, false);
const duplicateAssetIds = assets.map((asset) => asset.id);
Expand Down Expand Up @@ -191,6 +228,7 @@
title={$t('show_keyboard_shortcuts')}
onclick={() => (isShowKeyboardShortcut = !isShowKeyboardShortcut)}
/>
<CircleIconButton icon={mdiCogOutline} title={$t('options')} onclick={() => (isShowOptions = !isShowOptions)} />
</HStack>
{/snippet}

Expand All @@ -212,9 +250,12 @@
{#key duplicates[0].duplicateId}
<DuplicatesCompareControl
assets={duplicates[0].assets}
onResolve={(duplicateAssetIds, trashIds) =>
handleResolve(duplicates[0].duplicateId, duplicateAssetIds, trashIds)}
onResolve={(duplicateAssetIds, trashIds, selectedDataToSync) =>
handleResolve(duplicates[0].duplicateId, duplicateAssetIds, trashIds, selectedDataToSync)}
onStack={(assets) => handleStack(duplicates[0].duplicateId, assets)}
{isSynchronizeAlbumsActive}
{isSynchronizeFavoritesActive}
{isSynchronizeArchivesActive}
/>
{/key}
{:else}
Expand All @@ -225,6 +266,18 @@
</div>
</UserPageLayout>

{#if isShowOptions}
<DuplicateOptions
synchronizeAlbums={isSynchronizeAlbumsActive}
synchronizeFavorites={isSynchronizeFavoritesActive}
synchronizeArchives={isSynchronizeArchivesActive}
onClose={() => (isShowOptions = false)}
onToggleSyncAlbum={() => (isSynchronizeAlbumsActive = !isSynchronizeAlbumsActive)}
onToggleSyncFavorites={() => (isSynchronizeFavoritesActive = !isSynchronizeFavoritesActive)}
onToggleSyncArchives={() => (isSynchronizeArchivesActive = !isSynchronizeArchivesActive)}
/>
{/if}

{#if isShowKeyboardShortcut}
<ShowShortcuts shortcuts={duplicateShortcuts} onClose={() => (isShowKeyboardShortcut = false)} />
{/if}
Expand Down