finished coding move and copy

sirepo/const.py

@@ -17,6 +17,9 @@
 
 JSON_SUFFIX = ".json"
 
+#: where template resources and template non-sim user files live
+LIB_DIR = "lib"
+
 # matches requirements for uid and isn't actually put in the db
 MOCK_UID = "someuser"
 

sirepo/sim_data/__init__.py

@@ -52,8 +52,7 @@
     "computeJobSerial",
 )
 
-# POSIT: same as simulation_db._LIB_DIR
-_LIB_DIR = "lib"
+_LIB_DIR = sirepo.const.LIB_DIR
 
 _TEMPLATE_RESOURCE_DIR = "template"
 
@@ -441,7 +440,7 @@ def lib_file_resource_path(cls, path):
         return sirepo.resource.file_path(
             _TEMPLATE_RESOURCE_DIR,
             cls.sim_type(),
-            cls.LIB_DIR,
+            _LIB_DIR,
             path,
         )
 
@@ -456,7 +455,7 @@ def lib_file_read_binary(cls, basename, qcall=None):
             bytes: contents of file
         """
         if cls._is_agent_side():
-            return cls.sim_db_client().get(cls.LIB_DIR, basename)
+            return cls.sim_db_client().get(_LIB_DIR, basename)
         return cls._lib_file_abspath(basename, qcall=qcall).read_binary()
 
     @classmethod
@@ -482,7 +481,7 @@ def lib_file_size(cls, basename, qcall=None):
             int: size in bytes
         """
         if cls._is_agent_side():
-            return cls.sim_db_client().size(cls.LIB_DIR, basename)
+            return cls.sim_db_client().size(_LIB_DIR, basename)
         return cls._lib_file_abspath(basename, qcall=qcall).size()
 
     @classmethod
@@ -502,7 +501,7 @@ def _target():
             )
 
         if cls._is_agent_side():
-            cls.sim_db_client().write(cls.LIB_DIR, basename, path_or_content)
+            cls.sim_db_client().write(_LIB_DIR, basename, path_or_content)
             return
         if isinstance(path_or_content, pkconst.PY_PATH_LOCAL_TYPE):
             path_or_content.write_binary(_target())
@@ -718,6 +717,19 @@ def schema(cls):
 
     @classmethod
     def sim_db_client(cls):
+        """Low-level for sim_db_file ops for job agents
+
+        Used to manipulate sim db files in job agent. Care should be
+        taken to avoid inefficiencies as these are remote requests.
+        Typically, these are done in job_cmd, not job_agent, because
+        operations are synchronous.
+
+        Returns:
+            _SimDbClient: interface to `sirepo.sim_db_file`
+        """
+        # This assertion is sufficient even though memoized, because
+        # it is executed once per process.
+        cls._assert_agent_side()
         return cls._memoize(_SimDbClient(cls))
 
     @classmethod
@@ -763,6 +775,11 @@ def watchpoint_id(cls, report):
             raise RuntimeError("invalid watchpoint report name: ", report)
         return int(m.group(1))
 
+    @classmethod
+    def _assert_agent_side(cls):
+        if not cls._is_agent_side():
+            raise AssertionError(f"method={_caller()} only in job_agent")
+
     @classmethod
     def _assert_server_side(cls):
         if cls._is_agent_side():
@@ -859,20 +876,18 @@ def _lib_file_abspath_or_exists(
         """
         if cls._is_agent_side():
             if exists_only:
-                if cls.sim_db_client().exists(cls.LIB_DIR, basename):
+                if cls.sim_db_client().exists(_LIB_DIR, basename):
                     return True
             else:
                 try:
                     return cls.sim_db_client().get_and_save(
-                        cls.LIB_DIR, basename, pkio.py_path()
+                        _LIB_DIR, basename, pkio.py_path()
                     )
                 except Exception as e:
                     if not pkio.exception_is_not_found(e):
                         raise
                     # try to find below
         elif not _cfg.lib_file_resource_only:
-            from sirepo import simulation_db
-
             # Command line utility or server
             f = cls._lib_file_abspath(basename, qcall)
             if f.check(file=True):
@@ -901,14 +916,12 @@ def _lib_file_list(cls, pat, want_user_lib_dir=True, qcall=None):
                 for f in sirepo.resource.glob_paths(
                     _TEMPLATE_RESOURCE_DIR,
                     cls.sim_type(),
-                    cls.LIB_DIR,
+                    _LIB_DIR,
                     pat,
                 )
             )
         )
         if want_user_lib_dir:
-            from sirepo import simulation_db
-
             # lib_dir overwrites resource_dir
             res.update(
                 (f.basename, f)
@@ -1037,7 +1050,7 @@ def size(self, lib_sid_uri, basename=None, sim_type=None):
     def uri(self, lib_sid_uri, basename=None, sim_type=None):
         """Create a `_SimDbUri`
 
-        ``lib_sid_uri`` may be `cls.LIB_DIR`, a simulation id, or a
+        ``lib_sid_uri`` may be `sirepo.const.LIB_DIR`, a simulation id, or a
         `_SimDbUri`.  In the latter case, the uri must match ``sim_type``
         (if supplied), and ``basename`` must be None.
 
@@ -1079,8 +1092,6 @@ def _post(self, method, lib_sid_uri, basename, args=None, sim_type=None):
     def _request(
         self, method, lib_sid_uri, basename, data=None, json=None, sim_type=None
     ):
-        from sirepo import simulation_db
-
         u = self.uri(lib_sid_uri, basename, sim_type)
         r = sirepo.agent_supervisor_api.request(
             method,
@@ -1097,8 +1108,6 @@ def _request(
 
 class _SimDbUri(str):
     def __new__(cls, sim_type, slu, basename):
-        from sirepo import simulation_db
-
         if isinstance(slu, _SimDbFileUri):
             if basename is not None:
                 raise AssertionError(
@@ -1108,7 +1117,7 @@ def __new__(cls, sim_type, slu, basename):
                 raise AssertionError(f"sim_type={sim_type} disagrees with uri={slu}")
             return slu
         self = super().__new__(
-            simulation_db.sim_db_file_uri(sim_type, sid_or_lib, basename)
+            sim_db_file.uri_from_parts(sim_type, sid_or_lib, basename)
         )
         self.__stype = sim_type
         return self

sirepo/sim_db_file.py

@@ -16,6 +16,27 @@
 _URI_RE = re.compile(f"^{sirepo.job.SIM_DB_FILE_URI}/(.+)")
 
 
+def uri_from_parts(sim_type, sid_or_lib, basename):
+    """Generate relative URI for path
+
+    Args:
+        sim_type (str): which code
+        sid_or_lib (str): simulation id. if None or `_LIB_DIR`, is library file
+        basename (str): file name with extension
+
+    Returns:
+        str: uri to be passed to sim_db_file functions
+    """
+
+    return "/".join(
+        [
+            sirepo.template.assert_sim_type(sim_type),
+            _sid_or_lib(sid_or_lib),
+            assert_sim_db_basename(basename),
+        ]
+    )
+
+
 class FileReq(sirepo.agent_supervisor_api.ReqBase):
     _TOKEN_TO_UID = PKDict()
     _UID_TO_TOKEN = PKDict()
@@ -27,15 +48,23 @@ def get(self, unused_arg):
         self.write(pkio.read_binary(p))
 
     async def post(self, unused_arg):
+        def _result(value):
+            if value is None:
+                value = PKDict()
+            elif not isinstance(value, PKDict):
+                value = PKDict(result=value)
+            return value.pksetdefault(state="ok")
+
         try:
-            r = pkjson.load_any(self.request.body)
-            a = r.get("args")
-            # note that args may be empty, since uri has path
+            a = pkjson.load_any(self.request.body).get("args")
+            # note that args may be empty (but must be PKDict), since uri has path
             if not isinstance(a, PKDict):
                 raise AssertionError(f"invalid post args={a}")
             self.__path = self.__authenticate_and_path()
             self.write(
-                getattr(self, "_sr_post_" + r.get("method", "missing_method"))(a)
+                _result(
+                    getattr(self, "_sr_post_" + r.get("method", "missing_method"))(a),
+                ),
             )
         except Exception as e:
             if pkio.exception_is_not_found(e):
@@ -55,10 +84,6 @@ async def post(self, unused_arg):
     def put(self, unused_arg):
         self.__authenticate_and_path().write_binary(self.request.body)
 
-    def __authenticate_and_path(self):
-        self.__uid = self._rs_authenticate()
-        return self.__uri_to_path(self.request.path)
-
     def _sr_post_delete_glob(self, args):
         t = []
         for f in pkio.sorted_glob(f"{self.__authenticate_and_path()}*"):
@@ -70,28 +95,48 @@ def _sr_post_delete_glob(self, args):
             pkio.unchecked_remove(f)
 
     def _sr_post_copy(self, args):
-        self._path.copy(self.__uri_to_path(self.args.dst_uri))
-        return PKDict(state="ok")
+        self._path.copy(self.__uri_to_path_simple(self.__uri_arg_to_path(args.dst_uri)))
 
     def _sr_post_exists(self, args):
-        return PKDict(state="ok", result=self._path.check(file=True))
+        return self._path.check(file=True)
 
     def _sr_post_move(self, args):
-        self._path.move(self.__uri_to_path(self.args.dst_uri))
-        return PKDict(state="ok")
+        self._path.move(self.__uri_to_path_simple(self.__uri_arg_to_path(args.dst_uri)))
 
     def _sr_post_size(self, args):
-        return PKDict(state="ok", result=self._path.size())
+        return self._path.size()
 
     def _sr_post_missing_method(self, args):
         raise AssertionError("missing method path={} args={}", self._path, args)
 
+    def __authenticate_and_path(self):
+        self.__uid = self._rs_authenticate()
+        return self.__uri_to_path(self.request.path)
+
+    def __uri_arg_to_path(self, uri):
+        p = uri.split("/")
+        if len(p) != 3:
+            raise AssertionError(f"uri={p} must be 3 parts")
+        return self.__uri_to_path_simple(*p)
+
     def __uri_to_path(self, uri):
         m = _URI_RE.search(uri)
         if not m:
             pkdlog("uri={} missing {sirepo.job.SIM_DB_FILE_URI} prefix", uri)
             raise sirepo.tornado.error_forbidden()
-        return sirepo.simulation_db.sim_db_file_uri_to_path(
-            uri=m.group(1),
-            expect_uid=self.__uid,
+        p = m.group(1).split("/")
+        assert len(p) == 4, f"uri={p} must be 4 parts"
+        assert p[0] == self.__uid, f"uid={p[0]} is not expect_uid={self.__uid}"
+        return self.__uri_to_path_simple(*p[1:])
+
+    def __uri_to_path_simple(self, stype, sid_or_lib, basename):
+        sirepo.template.assert_sim_type(stype),
+        _sid_or_lib(sid_or_lib),
+        sirepo.simulation_db.assert_sim_db_basename(basename),
+        return simulation_db.user_path_root().join(
+            self.__uid, stype, sid_or_lib, basename
         )
+
+
+def _sid_or_lib(value):
+    return _LIB_DIR if value is None or value == _LIB_DIR else assert_sid(value)

sirepo/simulation_db.py

@@ -59,7 +59,7 @@
 _ID_RE = re.compile("^{}$".format(_ID_PARTIAL_RE_STR))
 
 #: where users live under db_dir
-_LIB_DIR = "lib"
+_LIB_DIR = sirepo.const.LIB_DIR
 
 #: lib relative to sim_dir
 _REL_LIB_DIR = "../" + _LIB_DIR
@@ -696,53 +696,6 @@ def sim_data_file(sim_type, sim_id, qcall=None):
     return simulation_dir(sim_type, sim_id, qcall=qcall).join(SIMULATION_DATA_FILE)
 
 
-def sim_db_file_uri(sim_type, sid_or_lib, basename):
-    """Generate relative URI for path
-
-    Args:
-        sim_type (str): which code
-        sid_or_lib (str): simulation id. if None or `_LIB_DIR`, is library file
-        basename (str): file name with extension
-
-    Returns:
-        str: uri to be passed to sim_db_file functions
-    """
-
-    def _sid_or_lib():
-        return (
-            _LIB_DIR
-            if sid_or_lib is None or sid_or_lib == _LIB_DIR
-            else assert_sid(sid_or_lib)
-        )
-
-    return "/".join(
-        [
-            sirepo.template.assert_sim_type(sim_type),
-            _sid_or_lib(),
-            assert_sim_db_basename(basename),
-        ]
-    )
-
-
-def sim_db_file_uri_to_path(uri, expect_uid):
-    """Generate absolute path for `uri`
-
-    Args:
-        uri (str): three component relative uri, e.g. srw/sid123/sirepo-data.json
-        except_uid (str): which user should be asserted
-    Returns:
-        py.path: absolute path to file (which may not exist)
-    """
-    p = uri.split("/")
-    assert len(p) == 4, f"uri={p} has too many parts"
-    assert p[0] == expect_uid, f"uid={p[0]} is not expect_uid={expect_uid}"
-    sirepo.template.assert_sim_type(p[1]),
-    if p[2] != _LIB_DIR:
-        assert_sid(p[2]),
-    assert_sim_db_basename(p[3]),
-    return user_path_root().join(*p)
-
-
 def sim_from_path(path):
     return _sim_from_path(path)