Implement File Storage

lib/livebook/file_system/s3.ex

@@ -43,12 +43,10 @@ defmodule Livebook.FileSystem.S3 do
     bucket_url = String.trim_trailing(bucket_url, "/")
     region = opts[:region] || region_from_uri(bucket_url)
 
-    hash = :crypto.hash(:sha256, bucket_url) |> Base.url_encode64(padding: false)
-
     id =
       if prefix = opts[:prefix],
-        do: "#{prefix}-s3-#{hash}",
-        else: "s3-#{hash}"
+        do: "#{prefix}-#{id(bucket_url)}",
+        else: id(bucket_url)
 
     %__MODULE__{
       id: id,
@@ -134,14 +132,18 @@ defmodule Livebook.FileSystem.S3 do
 
   defp put_id(changeset) do
     if bucket_url = get_field(changeset, :bucket_url) do
-      hash = :crypto.hash(:sha256, bucket_url)
-      encrypted_hash = Base.url_encode64(hash, padding: false)
-
-      put_change(changeset, :id, "s3-#{encrypted_hash}")
+      put_change(changeset, :id, id(bucket_url))
     else
       changeset
     end
   end
+
+  defp id(bucket_url) do
+    hash = :crypto.hash(:sha256, bucket_url)
+    encrypted_hash = Base.url_encode64(hash, padding: false)
+
+    "s3-#{encrypted_hash}"
+  end
 end
 
 defimpl Livebook.FileSystem, for: Livebook.FileSystem.S3 do

lib/livebook/hubs.ex

@@ -161,9 +161,9 @@ defmodule Livebook.Hubs do
 
   Topic `hubs:file_systems`:
 
-    * `{:file_system_created, t:FileSystem}`
-    * `{:file_system_updated, t:FileSystem}`
-    * `{:file_system_deleted, t:FileSystem}`
+    * `{:file_system_created, FileSystem.t()}`
+    * `{:file_system_updated, FileSystem.t()}`
+    * `{:file_system_deleted, FileSystem.t()}`
 
   """
   @spec subscribe(atom() | list(atom())) :: :ok | {:error, term()}
@@ -316,12 +316,14 @@ defmodule Livebook.Hubs do
   @doc """
   Gets a list of file systems for given hub.
   """
-  @spec get_file_systems(Provider.t()) :: list(FileSystem.t())
-  def get_file_systems(hub) do
+  @spec get_file_systems(Provider.t(), keyword()) :: list(FileSystem.t())
+  def get_file_systems(hub, opts \\ []) do
     hub_file_systems = Provider.get_file_systems(hub)
-    local_file_system = Livebook.Config.local_file_system()
+    sorted_hub_file_systems = Enum.sort_by(hub_file_systems, & &1.id)
 
-    [local_file_system | Enum.sort_by(hub_file_systems, & &1.id)]
+    if opts[:hub_only],
+      do: sorted_hub_file_systems,
+      else: [Livebook.Config.local_file_system() | sorted_hub_file_systems]
   end
 
   @doc """

lib/livebook/session.ex

@@ -2661,15 +2661,20 @@ defmodule Livebook.Session do
         {:error, "no file exists at path #{inspect(file.path)}"}
       end
     else
-      type = Livebook.FileSystems.type(file.file_system)
-      dumped_data = FileSystem.dump(file.file_system)
-      "/" <> key = file.path
-
       spec =
-        Map.merge(dumped_data, %{
-          type: String.to_existing_atom(type),
-          key: key
-        })
+        case file.file_system do
+          %FileSystem.S3{} = file_system ->
+            "/" <> key = file.path
+
+            %{
+              type: :s3,
+              bucket_url: file_system.bucket_url,
+              region: file_system.region,
+              access_key_id: file_system.access_key_id,
+              secret_access_key: file_system.secret_access_key,
+              key: key
+            }
+        end
 
       {:ok, spec}
     end

lib/livebook_web/live/file_select_component.ex

@@ -71,12 +71,12 @@ defmodule LivebookWeb.FileSelectComponent do
       |> assign(assigns)
       |> update_file_infos(force_reload? or running_files_changed?)
 
-    configure_path = ~p"/hub/#{Livebook.Hubs.Personal.id()}/file-systems/new"
-
-    {file_systems, configure_path} =
+    {file_systems, configure_hub_id} =
       if hub = socket.assigns[:hub],
-        do: {Livebook.Hubs.get_file_systems(hub), ~p"/hub/#{hub.id}/file-systems/new"},
-        else: {Livebook.Hubs.get_file_systems(), configure_path}
+        do: {Livebook.Hubs.get_file_systems(hub), hub.id},
+        else: {Livebook.Hubs.get_file_systems(), Livebook.Hubs.Personal.id()}
+
+    configure_path = ~p"/hub/#{configure_hub_id}/file-systems/new"
 
     {:ok, assign(socket, file_systems: file_systems, configure_path: configure_path)}
   end

lib/livebook_web/live/hub/edit/personal_component.ex

@@ -11,7 +11,7 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
     socket = assign(socket, assigns)
     changeset = Personal.change_hub(assigns.hub)
     secrets = Hubs.get_secrets(assigns.hub)
-    [_local | file_systems] = Hubs.get_file_systems(assigns.hub)
+    file_systems = Hubs.get_file_systems(assigns.hub, hub_only: true)
     secret_name = assigns.params["secret_name"]
     file_system_id = assigns.params["file_system_id"]
 
@@ -101,7 +101,6 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
               id="hub-secrets-list"
               hub={@hub}
               secrets={@secrets}
-              target={@myself}
             />
           </div>
 
@@ -119,7 +118,6 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
               id="hub-file-systems-list"
               hub_id={@hub.id}
               file_systems={@file_systems}
-              target={@myself}
             />
           </div>
 
@@ -239,25 +237,6 @@ defmodule LivebookWeb.Hub.Edit.PersonalComponent do
     {:noreply, validate(params, :stamp_changeset, socket)}
   end
 
-  def handle_event("delete_hub_secret", attrs, socket) do
-    %{hub: hub} = socket.assigns
-
-    on_confirm = fn socket ->
-      {:ok, secret} = Livebook.Secrets.update_secret(%Livebook.Secrets.Secret{}, attrs)
-      _ = Livebook.Hubs.delete_secret(hub, secret)
-
-      socket
-    end
-
-    {:noreply,
-     confirm(socket, on_confirm,
-       title: "Delete hub secret - #{attrs["name"]}",
-       description: "Are you sure you want to delete this hub secret?",
-       confirm_text: "Delete",
-       confirm_icon: "delete-bin-6-line"
-     )}
-  end
-
   defp save(params, changeset_name, socket) do
     case Personal.update_hub(socket.assigns.hub, params) do
       {:ok, hub} ->

lib/livebook_web/live/hub/edit/team_component.ex

@@ -13,7 +13,7 @@ defmodule LivebookWeb.Hub.Edit.TeamComponent do
     changeset = Team.change_hub(assigns.hub)
     show_key? = assigns.params["show-key"] == "true"
     secrets = Livebook.Hubs.get_secrets(assigns.hub)
-    [_local | file_systems] = Hubs.get_file_systems(assigns.hub)
+    file_systems = Hubs.get_file_systems(assigns.hub, hub_only: true)
     secret_name = assigns.params["secret_name"]
     file_system_id = assigns.params["file_system_id"]
     is_default? = is_default?(assigns.hub)

lib/livebook_web/live/hub/edit_live.ex

@@ -9,12 +9,13 @@ defmodule LivebookWeb.Hub.EditLive do
 
   @impl true
   def mount(_params, _session, socket) do
+    Hubs.subscribe([:connection])
+
     {:ok, assign(socket, hub: nil, type: nil, page_title: "Hub - Livebook", params: %{})}
   end
 
   @impl true
   def handle_params(params, _url, socket) do
-    Hubs.subscribe([:connection, :secrets, :file_systems])
     hub = Hubs.fetch_hub!(params["id"])
     type = Provider.type(hub)
 
@@ -85,55 +86,8 @@ defmodule LivebookWeb.Hub.EditLive do
   end
 
   @impl true
-  def handle_info(
-        {:secret_created, %{name: name, hub_id: id}},
-        %{assigns: %{hub: %{id: id}}} = socket
-      ) do
-    {:noreply,
-     socket
-     |> push_navigate(to: ~p"/hub/#{id}")
-     |> put_flash(:success, "Secret #{name} added successfully")}
-  end
-
-  def handle_info(
-        {:secret_updated, %{name: name, hub_id: id}},
-        %{assigns: %{hub: %{id: id}}} = socket
-      ) do
-    {:noreply,
-     socket
-     |> push_navigate(to: ~p"/hub/#{id}")
-     |> put_flash(:success, "Secret #{name} updated successfully")}
-  end
-
-  def handle_info(
-        {:secret_deleted, %{name: name, hub_id: id}},
-        %{assigns: %{hub: %{id: id}}} = socket
-      ) do
-    {:noreply,
-     socket
-     |> push_navigate(to: ~p"/hub/#{id}")
-     |> put_flash(:success, "Secret #{name} deleted successfully")}
-  end
-
-  def handle_info({:file_system_created, _}, socket) do
-    {:noreply,
-     socket
-     |> push_navigate(to: ~p"/hub/#{socket.assigns.hub.id}")
-     |> put_flash(:success, "File storage added successfully")}
-  end
-
-  def handle_info({:file_system_updated, _}, socket) do
-    {:noreply,
-     socket
-     |> push_navigate(to: ~p"/hub/#{socket.assigns.hub.id}")
-     |> put_flash(:success, "File storage updated successfully")}
-  end
-
-  def handle_info({:file_system_deleted, _}, socket) do
-    {:noreply,
-     socket
-     |> push_navigate(to: ~p"/hub/#{socket.assigns.hub.id}")
-     |> put_flash(:success, "File storage deleted successfully")}
+  def handle_info({:redirect, id, flash}, %{assigns: %{hub: %{id: id}}} = socket) do
+    {:noreply, socket |> push_navigate(to: ~p"/hub/#{id}") |> put_flash(:success, flash)}
   end
 
   def handle_info({:hub_connected, id}, %{assigns: %{hub: %{id: id}}} = socket) do

lib/livebook_web/live/hub/file_system_form_component.ex

@@ -5,7 +5,13 @@ defmodule LivebookWeb.Hub.FileSystemFormComponent do
   alias Livebook.FileSystems
 
   @impl true
-  def update(%{file_system: file_system} = assigns, socket) do
+  def update(assigns, socket) do
+    {file_system, assigns} = Map.pop!(assigns, :file_system)
+
+    mode = mode(file_system)
+    button = button(file_system)
+    title = title(file_system)
+
     file_system = file_system || %FileSystem.S3{}
     changeset = FileSystems.change_file_system(file_system)
     socket = assign(socket, assigns)
@@ -14,9 +20,9 @@ defmodule LivebookWeb.Hub.FileSystemFormComponent do
      assign(socket,
        file_system: file_system,
        changeset: changeset,
-       mode: mode(socket),
-       title: title(socket),
-       button: button(socket),
+       mode: mode,
+       title: title,
+       button: button,
        error_message: nil
      )}
   end
@@ -84,9 +90,19 @@ defmodule LivebookWeb.Hub.FileSystemFormComponent do
     with {:ok, file_system} <- FileSystems.update_file_system(socket.assigns.file_system, attrs),
          :ok <- check_file_system_conectivity(file_system),
          :ok <- save_file_system(file_system, socket) do
-      {:noreply, push_patch(socket, to: socket.assigns.return_to)}
+      message =
+        case socket.assigns.mode do
+          :new -> "File storage added successfully"
+          :edit -> "File storage updated successfully"
+        end
+
+      {:noreply,
+       socket
+       |> put_flash(:success, message)
+       |> push_redirect(to: socket.assigns.return_to)}
     else
       {:error, %Ecto.Changeset{} = changeset} -> {:noreply, assign(socket, changeset: changeset)}
+      {:transport_error, message} -> {:noreply, put_flash(socket, :error, message)}
       {:error, message} -> {:noreply, assign(socket, error_message: message)}
     end
   end
@@ -107,12 +123,12 @@ defmodule LivebookWeb.Hub.FileSystemFormComponent do
     end
   end
 
-  defp mode(%{assigns: %{file_system: nil}}), do: :new
+  defp mode(nil), do: :new
   defp mode(_), do: :edit
 
-  defp title(%{assigns: %{file_system: nil}}), do: "Add file storage"
+  defp title(nil), do: "Add file storage"
   defp title(_), do: "Edit file storage"
 
-  defp button(%{assigns: %{file_system: nil}}), do: %{icon: "add-line", label: "Add"}
+  defp button(nil), do: %{icon: "add-line", label: "Add"}
   defp button(_), do: %{icon: "save-line", label: "Save"}
 end

lib/livebook_web/live/hub/file_system_list_component.ex

@@ -80,8 +80,12 @@ defmodule LivebookWeb.Hub.FileSystemListComponent do
       file_system = Enum.find(file_systems, &(&1.id == id))
 
       case Livebook.Hubs.delete_file_system(hub, file_system) do
-        :ok -> socket
-        {:transport_error, reason} -> put_flash(socket, :error, reason)
+        :ok ->
+          send(self(), {:redirect, hub.id, "File storage deleted successfully"})
+          socket
+
+        {:transport_error, reason} ->
+          put_flash(socket, :error, reason)
       end
     end
 

lib/livebook_web/live/hub/secret_list_component.ex

@@ -64,7 +64,7 @@ defmodule LivebookWeb.Hub.SecretListComponent do
                         }
                       )
                     }
-                    phx-target={@target}
+                    phx-target={@myself}
                     role="menuitem"
                   >
                     <.remix_icon icon="delete-bin-line" />
@@ -92,8 +92,12 @@ defmodule LivebookWeb.Hub.SecretListComponent do
       hub = Hubs.fetch_hub!(secret.hub_id)
 
       case Hubs.delete_secret(hub, secret) do
-        :ok -> socket
-        {:transport_error, reason} -> put_flash(socket, :error, reason)
+        :ok ->
+          send(self(), {:redirect, hub.id, "Secret #{secret.name} deleted successfully"})
+          socket
+
+        {:transport_error, reason} ->
+          put_flash(socket, :error, reason)
       end
     end
 

lib/livebook_web/live/session_live/persistence_component.ex

@@ -49,7 +49,6 @@ defmodule LivebookWeb.SessionLive.PersistenceComponent do
     socket =
       socket
       |> assign(assigns)
-      |> assign_new(:hub, fn -> nil end)
       |> assign_new(:attrs, fn -> attrs end)
       |> assign_new(:new_attrs, fn -> attrs end)
       |> assign_new(:draft_file, fn ->