Pull dependencies implicitly when running a report. (#47)

If a user runs a report with a `allow_remote=True` in their search
options, specifying a dependency will look for packets in remote
locations, and pull files from those locations as necessary. If
`pull_metadata` is set as well, the metadata will be synchronized from
all (enabled) locations.

By default, only the requested files are copied, and not whole packets.
However, if the `require_complete_tree` option is enabled on the root
then the entire packet, and all of its transitive dependencies, are
pulled as well.

src/orderly/core.py

@@ -8,7 +8,7 @@
 
 from orderly.current import get_active_context
 from outpack import util
-from outpack.helpers import copy_files
+from outpack.copy_files import copy_files
 from outpack.search import search_unique
 
 
@@ -233,16 +233,16 @@ def dependency(name, query, files):
         raise Exception(msg)
 
     if ctx.is_active:
-        # TODO: search options here from need to come through from
-        # orderly_run via the context, it's not passed through from
-        # run yet, or present in the context, because search is barely
-        # supported.
-        result = ctx.packet.use_dependency(query, files)
+        result = ctx.packet.use_dependency(query, files, ctx.search_options)
     else:
-        # TODO: get options from the interactive search options, once
-        # it does anything.
-        id = search_unique(query, root=ctx.root)
-        result = copy_files(id, files, ctx.path, root=ctx.root)
+        id = search_unique(query, root=ctx.root, options=ctx.search_options)
+        result = copy_files(
+            id,
+            files,
+            ctx.path,
+            root=ctx.root,
+            options=ctx.search_options,
+        )
 
     # TODO: print about this, once we decide what that looks like generally
     return result

src/orderly/current.py

@@ -5,6 +5,7 @@
 
 from outpack.packet import Packet
 from outpack.root import OutpackRoot, root_open
+from outpack.search_options import SearchOptions
 
 
 class OrderlyCustomMetadata:
@@ -35,9 +36,11 @@ class OrderlyContext:
     id: Optional[str]
     # Special orderly custom metadata
     orderly: OrderlyCustomMetadata
+    # Options used when searching for dependencies
+    search_options: Optional[SearchOptions]
 
     @staticmethod
-    def from_packet(packet, path_src):
+    def from_packet(packet, path_src, search_options=None):
         return OrderlyContext(
             is_active=True,
             packet=packet,
@@ -48,6 +51,7 @@ def from_packet(packet, path_src):
             name=packet.name,
             id=packet.id,
             orderly=OrderlyCustomMetadata(),
+            search_options=search_options,
         )
 
     @staticmethod
@@ -63,15 +67,17 @@ def interactive():
             name=path.name,
             id=None,
             orderly=OrderlyCustomMetadata(),
+            # TODO: expose a way of configuring this
+            search_options=None,
         )
 
 
 class ActiveOrderlyContext:
     _context = None
     _our_context = None
 
-    def __init__(self, packet, path_src):
-        self._our_context = OrderlyContext.from_packet(packet, path_src)
+    def __init__(self, *args, **kwargs):
+        self._our_context = OrderlyContext.from_packet(*args, **kwargs)
 
     def __enter__(self):
         ActiveOrderlyContext._context = self._our_context

src/orderly/run.py

@@ -11,7 +11,9 @@
 from outpack.util import all_normal_files, run_script
 
 
-def orderly_run(name, *, parameters=None, root=None, locate=True):
+def orderly_run(
+    name, *, parameters=None, search_options=None, root=None, locate=True
+):
     root = root_open(root, locate=locate)
 
     path_src, entrypoint = _validate_src_directory(name, root)
@@ -29,7 +31,7 @@ def orderly_run(name, *, parameters=None, root=None, locate=True):
         root, path_dest, name, id=packet_id, locate=False, parameters=envir
     )
     try:
-        with ActiveOrderlyContext(packet, path_src) as orderly:
+        with ActiveOrderlyContext(packet, path_src, search_options) as orderly:
             packet.mark_file_immutable(entrypoint)
             run_script(path_dest, entrypoint, envir)
     except Exception as error:

src/outpack/copy_files.py

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict
+
+from outpack.location_pull import (
+    location_build_pull_plan,
+    location_pull_files,
+)
+from outpack.metadata import PacketFile
+from outpack.root import OutpackRoot
+from outpack.search_options import SearchOptions
+
+
+@dataclass
+class Plan:
+    id: str
+    name: str
+    files: Dict[str, PacketFile]
+
+
+def copy_files(
+    id: str,
+    files: Dict[str, str],
+    dest: Path,
+    options: SearchOptions,
+    root: OutpackRoot,
+) -> Plan:
+    plan = _plan_copy_files(root, id, files)
+
+    try:
+        for here, there in plan.files.items():
+            root.export_file(id, there.path, here, dest)
+
+    except FileNotFoundError as e:
+        if not options.allow_remote:
+            msg = f"File `{e.filename}` from packet {id} is not available locally."
+            raise Exception(msg) from e
+        else:
+            copy_files_from_remote(id, plan.files, dest, options, root)
+
+    return plan
+
+
+def copy_files_from_remote(
+    id: str,
+    files: Dict[str, PacketFile],
+    dest: Path,
+    options: SearchOptions,
+    root: OutpackRoot,
+):
+    plan = location_build_pull_plan(
+        [id],
+        options.location,
+        recursive=False,
+        files={id: [f.hash for f in files.values()]},
+        root=root,
+    )
+
+    with location_pull_files(plan.files, root) as store:
+        for here, there in files.items():
+            store.get(there.hash, dest / here, overwrite=True)
+
+
+def _validate_files(files):
+    if isinstance(files, str):
+        files = {files: files}
+    if isinstance(files, list):
+        files = {x: x for x in files}
+    return files
+
+
+def _plan_copy_files(root, id, files):
+    meta = root.index.metadata(id)
+    files = _validate_files(files)
+    known = {f.path: f for f in meta.files}
+
+    plan = {}
+    for here, there in files.items():
+        # TODO: check absolute paths
+        if here.endswith("/") or there.endswith("/"):
+            msg = "Directories not yet supported for export"
+            raise Exception(msg)
+
+        f = known.get(there, None)
+        if f is None:
+            msg = f"Packet '{id}' does not contain the requested path '{there}'"
+            raise Exception(msg)
+        plan[here] = f
+    return Plan(id, meta.name, plan)

src/outpack/filestore.py

@@ -3,6 +3,7 @@
 import shutil
 import stat
 from contextlib import contextmanager
+from errno import ENOENT
 from pathlib import Path
 
 from outpack.hash import Hash, hash_parse, hash_validate_file
@@ -22,7 +23,7 @@ def get(self, hash, dst, *, overwrite=False):
         src = self.filename(hash)
         if not os.path.exists(src):
             msg = f"Hash '{hash}' not found in store"
-            raise Exception(msg)
+            raise FileNotFoundError(ENOENT, msg)
         os.makedirs(os.path.dirname(dst), exist_ok=True)
         if not overwrite and os.path.exists(dst):
             msg = f"Failed to copy '{src}' to '{dst}', file already exists"

src/outpack/location.py

@@ -1,9 +1,10 @@
 import collections
 import shutil
+from pathlib import PurePath
 
 from outpack.config import Location, update_config
 from outpack.location_path import OutpackLocationPath
-from outpack.root import root_open
+from outpack.root import OutpackRoot, root_open
 from outpack.static import (
     LOCATION_LOCAL,
     LOCATION_ORPHAN,
@@ -38,6 +39,15 @@ def outpack_location_add(name, type, args, root=None, *, locate=True):
     update_config(config, root.path)
 
 
+def outpack_location_add_path(name, path, root=None, *, locate=True):
+    if isinstance(path, OutpackRoot):
+        path = str(path.path)
+    elif isinstance(path, PurePath):
+        path = str(path)
+
+    outpack_location_add(name, "path", {"path": path}, root=root, locate=locate)
+
+
 def outpack_location_remove(name, root=None, *, locate=True):
     root = root_open(root, locate=locate)