From a92717987a94bb65db06a7c504ff53011be600a8 Mon Sep 17 00:00:00 2001 From: Wes Bonelli Date: Mon, 16 May 2022 22:56:43 -0400 Subject: [PATCH] persistent migration status, better migration UI, various fixes (#301) --- .../front_end/src/components/navigation.vue | 188 ++++++++++++------ plantit/front_end/src/scss/main.sass | 3 +- plantit/front_end/src/store/user.js | 36 +--- plantit/plantit/admin.py | 11 + plantit/plantit/celery_tasks.py | 89 ++++----- plantit/plantit/queries.py | 13 +- plantit/plantit/users/admin.py | 24 --- plantit/plantit/users/models.py | 13 +- plantit/plantit/users/views.py | 34 ++-- 9 files changed, 220 insertions(+), 191 deletions(-) delete mode 100644 plantit/plantit/users/admin.py diff --git a/plantit/front_end/src/components/navigation.vue b/plantit/front_end/src/components/navigation.vue index 42232170..bb0421fb 100644 --- a/plantit/front_end/src/components/navigation.vue +++ b/plantit/front_end/src/components/navigation.vue @@ -621,10 +621,10 @@ : 'text-dark' " > - + DIRT Migration @@ -759,12 +759,17 @@ hide-footer busy > - + + -

+

You haven't migrated your datasets from DIRT yet.

- + You already have a collection with path /iplant/home/{{ @@ -776,10 +781,9 @@ {{ migrationSubmitting || - migrationData.started !== null - ? 'Running migration' - : 'Start Migration' + profile.migration.started !== null + ? 'Running' + : 'Start' }} -

+


- Started: {{ prettify(migrationData.started) }} + You have + {{ profile.migration.num_folders }} dataset(s) to + migrate. +
+ Started: + {{ prettify(profile.migration.started) }}
Collection: - {{ migrationData.target_path }} + {{ profile.migration.target_path }} +
+ Downloading: + {{ downloadedFolders.length }}/{{ + profile.migration.num_folders === null + ? '?' + : profile.migration.num_folders + }}
- Downloaded files: - {{ migrationData.downloads.length }} - + + - {{ file.folder }}/{{ file.name }} + {{ folder.name }}, + {{ folder.files.length }} file(s) - Uploaded datasets: - {{ migrationData.uploads.length }}/{{ - migrationData.num_folders === null + Uploading: + {{ uploadedFolders.length }}/{{ + profile.migration.num_folders === null ? '?' - : migrationData.num_folders + : profile.migration.num_folders }}
+ + + + {{ folder.name }}, + {{ folder.files.length }} file(s) + +

@@ -853,13 +901,13 @@

Started: - {{ prettify(profile.dirtMigrationStarted) }} + {{ prettify(profile.migration.started) }}
Completed: - {{ prettify(profile.dirtMigrationCompleted) }} + {{ prettify(profile.migration.completed) }}
Collection: - {{ profile.dirtMigrationPath }} + {{ profile.migration.target_path }}

@@ -974,15 +1022,7 @@ export default { dismissCountDown: 0, maintenanceWindows: [], // DIRT migration - migrationData: { - started: null, - completed: null, - target_path: null, - num_folders: null, - downloads: [], - uploads: [], - duplicate: false, - }, + migrationDataDuplicate: false, migrationSubmitting: false, }; }, @@ -1001,6 +1041,26 @@ export default { 'notificationsRead', 'notificationsUnread', ]), + downloadedFolders() { + if (this.profile.migration.downloads.length === 0) return []; + let grouped = this.groupBy( + this.profile.migration.downloads, + 'folder' + ); + return Object.entries(grouped).map((pair) => { + return { name: pair[0], files: pair[1] }; + }); + }, + uploadedFolders() { + if (this.profile.migration.uploads.length === 0) return []; + let grouped = this.groupBy( + this.profile.migration.uploads, + 'folder' + ); + return Object.entries(grouped).map((pair) => { + return { name: pair[0], files: pair[1] }; + }); + }, maintenance() { let now = moment(); return this.maintenanceWindows.find((w) => { @@ -1074,11 +1134,15 @@ export default { alerts() { this.dismissCountDown = this.dismissSecs; }, - migrationData() { - // noop - }, }, methods: { + // https://stackoverflow.com/a/34890276 + groupBy(xs, key) { + return xs.reduce(function (rv, x) { + (rv[x[key]] = rv[x[key]] || []).push(x); + return rv; + }, {}); + }, showDirtMigrationModal() { this.$bvModal.show('migration'); }, @@ -1086,19 +1150,15 @@ export default { this.$bvModal.hide('migration'); }, startDirtMigration() { + this.migrationDataDuplicate = false; this.migrationSubmitting = true; axios .get(`/apis/v1/users/start_dirt_migration/`) .then(async (response) => { - this.migrationData = response.data.migration; await Promise.all([ this.$store.dispatch( - 'user/setDirtMigrationStarted', - this.migrationData.started - ), - this.$store.dispatch( - 'user/setDirtMigrationPath', - this.migrationData.target_path + 'user/setDirtMigration', + response.data.migration ), this.$store.dispatch('alerts/add', { variant: 'success', @@ -1116,7 +1176,7 @@ export default { 'migration collection already exists' ) ) { - this.migrationData.duplicate = true; + this.migrationDataDuplicate = true; } this.$store.dispatch('alerts/add', { variant: 'danger', @@ -1206,6 +1266,10 @@ export default { response.data.django_profile.push_notifications ); this.$store.dispatch('user/setStats', response.data.stats); + this.$store.dispatch( + 'user/setDirtMigration', + response.data.migration + ); this.$store.dispatch('user/setProfileLoading', false); // load notifications into Vuex @@ -1439,16 +1503,16 @@ export default { } }, async handleMigrationEvent(migration) { - this.migrationData = migration; + await this.$store.dispatch('user/setDirtMigration', migration); // check if completed and update user profile & create an alert if so let completed = migration.completed; - if (completed !== null && completed !== undefined) { - await this.$store.dispatch( - 'user/setDirtMigrationCompleted', - completed - ); - } + if (completed !== null && completed !== undefined) + await this.$store.dispatch('alerts/add', { + variant: 'success', + message: `DIRT migration completed (target collection: ${migration.target_path})`, + guid: guid().toString(), + }); }, async handleNotificationEvent(notification) { await this.$store.dispatch('notifications/update', notification); diff --git a/plantit/front_end/src/scss/main.sass b/plantit/front_end/src/scss/main.sass index dc8a9ac9..92464bb6 100644 --- a/plantit/front_end/src/scss/main.sass +++ b/plantit/front_end/src/scss/main.sass @@ -457,4 +457,5 @@ a:link code font-family: "Cutive-Mono", "Menlo", "Courier New", monospace - font-size: 12px \ No newline at end of file + font-size: 12px + diff --git a/plantit/front_end/src/store/user.js b/plantit/front_end/src/store/user.js index 903e73f7..fd0d052f 100644 --- a/plantit/front_end/src/store/user.js +++ b/plantit/front_end/src/store/user.js @@ -19,9 +19,7 @@ export const user = { projects: [], hints: false, stats: null, - dirtMigrationStarted: null, - dirtMigrationCompleted: null, - dirtMigrationPath: null, + migration: null, }, profileLoading: true, }), @@ -62,14 +60,8 @@ export const user = { setProfileLoading(state, loading) { state.profileLoading = loading; }, - setDirtMigrationStarted(state, started) { - state.profile.dirtMigrationStarted = started; - }, - setDirtMigrationCompleted(state, completed) { - state.profile.dirtMigrationCompleted = completed; - }, - setDirtMigrationPath(state, path) { - state.profile.dirtMigrationPath = path; + setDirtMigration(state, migration) { + state.profile.migration = migration; }, }, actions: { @@ -110,16 +102,8 @@ export const user = { ); commit('setStats', response.data.stats); commit( - 'setDirtMigrationStarted', - response.data.django_profile.dirt_migration_started - ); - commit( - 'setDirtMigrationCompleted', - response.data.django_profile.dirt_migration_completed - ); - commit( - 'setDirtMigrationPath', - response.data.django_profile.dirt_migration_path + 'setDirtMigration', + response.data.django_profile.migration ); commit('setProfileLoading', false); }) @@ -209,14 +193,8 @@ export const user = { setStats({ commit }, stats) { commit('setStats', stats); }, - setDirtMigrationStarted({ commit }, started) { - commit('setDirtMigrationStarted', started); - }, - setDirtMigrationCompleted({ commit }, completed) { - commit('setDirtMigrationCompleted', completed); - }, - setDirtMigrationPath({ commit }, path) { - commit('setDirtMigrationPath', path); + setDirtMigration({ commit }, migration) { + commit('setDirtMigration', migration); }, }, getters: { diff --git a/plantit/plantit/admin.py b/plantit/plantit/admin.py index 0a7d6f90..75858c50 100644 --- a/plantit/plantit/admin.py +++ b/plantit/plantit/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin +from plantit.users.models import Profile, Migration from plantit.misc.models import NewsUpdate, MaintenanceWindow, FeaturedWorkflow from plantit.agents.models import Agent, AgentAccessPolicy from plantit.datasets.models import DatasetAccessPolicy, DatasetSession @@ -7,6 +8,16 @@ from plantit.miappe.models import Investigation, Study +@admin.register(Profile) +class ProfileAdmin(admin.ModelAdmin): + pass + + +@admin.register(Migration) +class MigrationAdmin(admin.ModelAdmin): + pass + + @admin.register(NewsUpdate) class NewsUpdateAdmin(admin.ModelAdmin): pass diff --git a/plantit/plantit/celery_tasks.py b/plantit/plantit/celery_tasks.py index 47eda433..cca9095f 100644 --- a/plantit/plantit/celery_tasks.py +++ b/plantit/plantit/celery_tasks.py @@ -23,7 +23,7 @@ from plantit.ssh import SSH from plantit.keypairs import get_user_private_key_path from plantit import settings -from plantit.users.models import Profile +from plantit.users.models import Profile, Migration from plantit.agents.models import Agent from plantit.celery import app from plantit.healthchecks import is_healthy @@ -931,24 +931,15 @@ class DownloadedFile(TypedDict): name: str -class UploadedFolder(TypedDict): - path: str - id: str - - -class Migration(TypedDict): - started: datetime - completed: Optional[datetime] - target_path: str - num_folders: Optional[int] - downloads: List[DownloadedFile] - uploads: List[UploadedFolder] +class UploadedFile(TypedDict): + folder: str + name: str async def push_migration_event(user: User, migration: Migration): await get_channel_layer().group_send(f"{user.username}", { 'type': 'migration_event', - 'migration': migration, + 'migration': q.migration_to_dict(migration), }) @@ -957,15 +948,12 @@ def migrate_dirt_datasets(self, username: str): try: user = User.objects.get(username=username) profile = Profile.objects.get(user=user) + migration = Migration.objects.get(profile=profile) except: logger.warning(f"Could not find user {username}") self.request.callbacks = None return - # get started time - start = profile.dirt_migration_started - - # create SSH/SFTP client and open SFTP connection ssh = SSH( host=settings.DIRT_MIGRATION_HOST, port=settings.DIRT_MIGRATION_PORT, @@ -974,13 +962,17 @@ def migrate_dirt_datasets(self, username: str): with ssh: with ssh.client.open_sftp() as sftp: - # list the user's datasets on the DIRT server + # check how many datasets the user has on the DIRT server user_dir = join(settings.DIRT_MIGRATION_DATA_DIR, username, 'root-images') datasets = [folder for folder in sftp.listdir(user_dir)] logger.info(f"User {username} has {len(datasets)} DIRT folders: {', '.join(datasets)}") + # persist number of datasets + migration.num_folders = len(datasets) + migration.save() + # create a client for the CyVerse APIs and create a collection for the migrated DIRT data - client = TerrainClient(access_token=profile.cyverse_access_token) + client = TerrainClient(access_token=profile.cyverse_access_token, timeout_seconds=60) root_collection_path = f"/iplant/home/{user.username}/dirt_migration" if client.dir_exists(root_collection_path): logger.warning(f"Collection {root_collection_path} already exists, aborting DIRT migration for {user.username}") @@ -998,23 +990,20 @@ def migrate_dirt_datasets(self, username: str): logger.info(f"User {username} folder {folder} has {len(files)} files: {', '.join(files)}") # create temp local folder for this dataset - staging_dir = join(settings.DIRT_MIGRATION_STAGING_DIR, folder) + staging_dir = join(settings.DIRT_MIGRATION_STAGING_DIR, user.username, folder) Path(staging_dir).mkdir(parents=True, exist_ok=True) # download files for file in files: - file_path = join(folder_name, file) - sftp.get(file_path, join(settings.DIRT_MIGRATION_STAGING_DIR, folder, file)) + sftp.get(join(folder_name, file), join(staging_dir, file)) - # push download status update to UI + # push download update to UI downloads.append(DownloadedFile(name=file, folder=folder)) - async_to_sync(push_migration_event)(user, Migration( - started=start.isoformat(), - completed=None, - num_folders=len(datasets), - target_path=root_collection_path, - downloads=downloads, - uploads=[])) + migration.downloads = json.dumps(downloads) + async_to_sync(push_migration_event)(user, migration) + + # persist downloaded dataset + migration.save() # create subcollection for this folder collection_path = join(root_collection_path, folder.rpartition('/')[2]) @@ -1025,9 +1014,16 @@ def migrate_dirt_datasets(self, username: str): client.mkdir(collection_path) # upload all files to collection - client.upload_directory( - from_path=join(settings.DIRT_MIGRATION_STAGING_DIR, folder), - to_prefix=collection_path) + for file in os.listdir(staging_dir): + client.upload(from_path=join(staging_dir, str(file)), to_prefix=collection_path) + + # push upload update to UI + uploads.append(UploadedFile(folder=folder, name=str(file))) + migration.uploads = json.dumps(uploads) + async_to_sync(push_migration_event)(user, migration) + + # persist uploaded dataset + migration.save() # get ID of newly created collection stat = client.stat(collection_path) @@ -1039,16 +1035,6 @@ def migrate_dirt_datasets(self, username: str): # TODO: anything else we need to add here? ], []) - # push upload status update to UI - uploads.append(UploadedFolder(path=collection_path, id=id)) - async_to_sync(push_migration_event)(user, Migration( - started=start.isoformat(), - completed=None, - num_folders=len(datasets), - target_path=root_collection_path, - downloads=downloads, - uploads=uploads)) - # get ID of newly created collection root_collection_id = client.stat(root_collection_path)['id'] @@ -1066,20 +1052,13 @@ def migrate_dirt_datasets(self, username: str): # f"Duration: {str(end - start)}", # {}) - # mark user's profile that DIRT transfer has been completed + # persist completion end = timezone.now() - profile.dirt_migration_completed = end - profile.save() - user.save() + migration.completed = end + migration.save() # push completion update to the UI - async_to_sync(push_migration_event)(user, Migration( - started=start.isoformat(), - completed=end.isoformat(), - num_folders=len(datasets), - target_path=root_collection_path, - downloads=downloads, - uploads=uploads)) + async_to_sync(push_migration_event)(user, migration) # see https://stackoverflow.com/a/41119054/6514033 diff --git a/plantit/plantit/queries.py b/plantit/plantit/queries.py index a1f23e40..67eaca5b 100644 --- a/plantit/plantit/queries.py +++ b/plantit/plantit/queries.py @@ -31,7 +31,7 @@ from plantit.misc.models import NewsUpdate, FeaturedWorkflow from plantit.datasets.models import DatasetAccessPolicy from plantit.tasks.models import Task, DelayedTask, RepeatingTask, TriggeredTask, TaskCounter, TaskStatus -from plantit.users.models import Profile +from plantit.users.models import Profile, Migration from plantit.utils.misc import del_none from plantit.utils.tasks import get_task_orchestrator_log_file_path, has_output_target @@ -749,6 +749,17 @@ def person_to_dict(user: User, role: str) -> dict: } +def migration_to_dict(migration: Migration) -> dict: + return { + 'started': None if migration.started is None else migration.started.isoformat(), + 'completed': None if migration.completed is None else migration.completed.isoformat(), + 'target_path': migration.target_path, + 'num_folders': migration.num_folders, + 'downloads': json.loads(migration.downloads), + 'uploads': json.loads(migration.uploads) + } + + # usage stats/demographics info diff --git a/plantit/plantit/users/admin.py b/plantit/plantit/users/admin.py deleted file mode 100644 index 868f5bc3..00000000 --- a/plantit/plantit/users/admin.py +++ /dev/null @@ -1,24 +0,0 @@ -from django.contrib import admin -from django.contrib.auth.admin import UserAdmin -from django.contrib.auth.models import User -from plantit.users.models import Profile - - -class ProfileInline(admin.StackedInline): - model = Profile - can_delete = False - verbose_name_plural = 'Profile' - fk_name = 'user' - - -class UserProfileAdmin(UserAdmin): - inlines = (ProfileInline,) - - def get_inline_instances(self, request, obj=None): - if not obj: - return list() - return super(UserProfileAdmin, self).get_inline_instances(request, obj) - - -admin.site.unregister(User) -admin.site.register(User, UserProfileAdmin) diff --git a/plantit/plantit/users/models.py b/plantit/plantit/users/models.py index 85a9f1a6..d8d14a64 100644 --- a/plantit/plantit/users/models.py +++ b/plantit/plantit/users/models.py @@ -19,6 +19,13 @@ class Profile(models.Model): hints = models.BooleanField(default=False) created = models.DateField(null=True, blank=True) first_login = models.BooleanField(default=True) - dirt_migration_started = models.DateTimeField(null=True, blank=True) - dirt_migration_completed = models.DateTimeField(null=True, blank=True) - dirt_migration_path = models.CharField(max_length=255, null=True, blank=True) + + +class Migration(models.Model): + profile: Profile = models.OneToOneField(Profile, on_delete=models.CASCADE) + started = models.DateTimeField(null=True, blank=True) + completed = models.DateTimeField(null=True, blank=True) + target_path = models.CharField(max_length=255, null=True, blank=True) + num_folders = models.IntegerField(null=True, blank=True) + downloads = models.JSONField(null=True, blank=True) + uploads = models.JSONField(null=True, blank=True) diff --git a/plantit/plantit/users/views.py b/plantit/plantit/users/views.py index 7c07f1ab..739e3695 100644 --- a/plantit/plantit/users/views.py +++ b/plantit/plantit/users/views.py @@ -259,6 +259,12 @@ def get_current(self, request): user.profile.save() user.save() + migration, created = Migration.objects.get_or_create(profile=user.profile) + if created: + migration.downloads = json.dumps([]) + migration.uploads = json.dumps([]) + migration.save() + response = { 'django_profile': { 'username': user.username, @@ -271,9 +277,7 @@ def get_current(self, request): 'cyverse_token': user.profile.cyverse_access_token, 'hints': user.profile.hints, 'first': user.profile.first_login, - 'dirt_migration_started': user.profile.dirt_migration_started, - 'dirt_migration_completed': user.profile.dirt_migration_completed, - 'dirt_migration_path': user.profile.dirt_migration_path + 'migration': q.migration_to_dict(migration) }, 'stats': async_to_sync(q.get_user_statistics)(user), 'users': q.list_users(), @@ -384,10 +388,7 @@ def get_key(self, request): def start_dirt_migration(self, request): user = self.get_object() profile = Profile.objects.get(user=user) - - # don't allow duplicate starts - if profile.dirt_migration_started is not None: - return HttpResponseBadRequest(f"DIRT migration already started for user {user.username}") + migration, created = Migration.objects.get_or_create(profile=profile) # make sure a `dirt_migration` collection doesn't already exist client = TerrainClient(access_token=profile.cyverse_access_token) @@ -396,10 +397,17 @@ def start_dirt_migration(self, request): self.logger.warning(f"Collection {root_collection_path} already exists, aborting DIRT migration for {user.username}") return HttpResponseBadRequest(f"DIRT migration collection already exists for {user.username}") + # if already started, just return it + if not created and migration.started is not None: + return JsonResponse({'migration': q.migration_to_dict(migration)}) + # record starting time start = timezone.now() - profile.dirt_migration_started = start - profile.dirt_migration_path = root_collection_path + migration.started = start + migration.target_path = root_collection_path + migration.downloads = json.dumps([]) + migration.uploads = json.dumps([]) + migration.save() profile.save() user.save() @@ -407,10 +415,4 @@ def start_dirt_migration(self, request): migrate_dirt_datasets.s(user.username).apply_async(countdown=5) # return status to client - return JsonResponse({'migration': Migration( - started=start.isoformat(), - completed=None, - num_folders=None, - target_path=f"/iplant/home/{user.username}/dirt_migration", - downloads=[], - uploads=[])}) + return JsonResponse({'migration': q.migration_to_dict(migration)})