deploy: a789451

_modules/biomero/slurm_client.html

@@ -336,6 +336,7 @@ <h1>Source code for biomero.slurm_client</h1><div class="highlight"><pre>
     <span class="n">_DEFAULT_SLURM_GIT_SCRIPT_PATH</span> <span class="o">=</span> <span class="s2">&quot;slurm-scripts&quot;</span>
     <span class="n">_OUT_SEP</span> <span class="o">=</span> <span class="s2">&quot;--split--&quot;</span>
     <span class="n">_VERSION_CMD</span> <span class="o">=</span> <span class="s2">&quot;ls -h </span><span class="se">\&quot;</span><span class="si">{slurm_images_path}</span><span class="s2">/</span><span class="si">{image_path}</span><span class="se">\&quot;</span><span class="s2"> | grep -oP &#39;(?&lt;=\-|\_)(v.+|latest)(?=.simg|.sif)&#39;&quot;</span>
+    <span class="n">_CONVERTER_VERSION_CMD</span> <span class="o">=</span> <span class="s2">&quot;ls -h </span><span class="se">\&quot;</span><span class="si">{converter_path}</span><span class="se">\&quot;</span><span class="s2"> | grep -oP &#39;(convert_.+)(?=.simg|.sif)&#39; | awk &#39;{n=split($0, a, </span><span class="se">\&quot;</span><span class="s2">_</span><span class="se">\&quot;</span><span class="s2">); last=a[n]; sub(</span><span class="se">\&quot;</span><span class="s2">_</span><span class="se">\&quot;</span><span class="s2">last</span><span class="se">\&quot;</span><span class="s2">$</span><span class="se">\&quot;</span><span class="s2">, </span><span class="se">\&quot;\&quot;</span><span class="s2">, $0); print $0, last}&#39;&quot;</span>
     <span class="c1"># Note, grep returns exitcode 1 if no match is found!</span>
     <span class="c1"># This will translate into a UnexpectedExit error, so mute that if you</span>
     <span class="c1"># don&#39;t care about empty.</span>
@@ -588,6 +589,19 @@ <h1>Source code for biomero.slurm_client</h1><div class="highlight"><pre>
                 <span class="c1"># cmd = &quot;singularity cache clean -f&quot;</span>
                 <span class="c1"># r = self.run_commands([cmd])</span>
 
+<div class="viewcode-block" id="SlurmClient.list_available_converter_versions"><a class="viewcode-back" href="../../biomero.html#biomero.slurm_client.SlurmClient.list_available_converter_versions">[docs]</a>    <span class="k">def</span> <span class="nf">list_available_converter_versions</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">:</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Note, assumes you use versioned converters.</span>
+<span class="sd">        Will return a dict with a version of each converter on your Slurm.</span>
+<span class="sd">        However, doesn&#39;t work properly with unversioned sif.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">cmd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_CONVERTER_VERSION_CMD</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">converter_path</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">slurm_converters_path</span><span class="p">),</span>
+        <span class="n">r</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_commands</span><span class="p">([</span><span class="n">cmd</span><span class="p">])</span>
+        <span class="c1"># split lines further into a k,v dict</span>
+        <span class="n">result_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">line</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]:</span> <span class="n">line</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">r</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)}</span>
+        <span class="k">return</span> <span class="n">result_dict</span></div>
+
 <div class="viewcode-block" id="SlurmClient.setup_converters"><a class="viewcode-back" href="../../biomero.html#biomero.slurm_client.SlurmClient.setup_converters">[docs]</a>    <span class="k">def</span> <span class="nf">setup_converters</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Sets up converters for Slurm operations.</span>
@@ -614,13 +628,18 @@ <h1>Source code for biomero.slurm_client</h1><div class="highlight"><pre>
                     <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">__version__</span>
                     <span class="c1"># use default container version of our package   </span>
                     <span class="n">version</span> <span class="o">=</span> <span class="n">__version__</span>
+                <span class="k">if</span> <span class="n">version</span><span class="p">:</span>
+                    <span class="n">chosen_converter</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;convert_</span><span class="si">{path}</span><span class="s2">_</span><span class="si">{version}</span><span class="s2">.sif&quot;</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">chosen_converter</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;convert_</span><span class="si">{path}</span><span class="s2">.sif&quot;</span>
                 <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">cd</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">slurm_converters_path</span><span class="p">):</span>
-                    <span class="n">pull_template</span> <span class="o">=</span> <span class="s2">&quot;echo &#39;starting $path $version&#39; &gt;&gt; sing.log</span><span class="se">\n</span><span class="s2">nohup sh -c </span><span class="se">\&quot;</span><span class="s2">singularity pull --force --disable-cache docker://$image:$version; echo &#39;finished $path $version&#39;</span><span class="se">\&quot;</span><span class="s2"> &gt;&gt; sing.log 2&gt;&amp;1 &amp; disown&quot;</span>
+                    <span class="n">pull_template</span> <span class="o">=</span> <span class="s2">&quot;echo &#39;starting $path $version&#39; &gt;&gt; sing.log</span><span class="se">\n</span><span class="s2">nohup sh -c </span><span class="se">\&quot;</span><span class="s2">singularity pull --force --disable-cache $conv_name docker://$image:$version; echo &#39;finished $path $version&#39;</span><span class="se">\&quot;</span><span class="s2"> &gt;&gt; sing.log 2&gt;&amp;1 &amp; disown&quot;</span>
                     <span class="n">t</span> <span class="o">=</span> <span class="n">Template</span><span class="p">(</span><span class="n">pull_template</span><span class="p">)</span>
                     <span class="n">substitutes</span> <span class="o">=</span> <span class="p">{}</span>
                     <span class="n">substitutes</span><span class="p">[</span><span class="s1">&#39;path&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">path</span>
                     <span class="n">substitutes</span><span class="p">[</span><span class="s1">&#39;image&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">image</span>
                     <span class="n">substitutes</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">version</span>
+                    <span class="n">substitutes</span><span class="p">[</span><span class="s1">&#39;conv_name&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">chosen_converter</span>
                     <span class="n">cmd</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">safe_substitute</span><span class="p">(</span><span class="n">substitutes</span><span class="p">)</span>
                     <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;substituted: </span><span class="si">{cmd}</span><span class="s2">&quot;</span><span class="p">)</span>
                     <span class="n">pull_commands</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">cmd</span><span class="p">)</span>
@@ -1861,7 +1880,14 @@ <h1>Source code for biomero.slurm_client</h1><div class="highlight"><pre>
             <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
                 <span class="sa">f</span><span class="s2">&quot;Conversion from </span><span class="si">{source_format}</span><span class="s2"> to </span><span class="si">{target_format}</span><span class="s2"> is not supported by default!&quot;</span><span class="p">)</span>
 
-        <span class="n">chosen_converter</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;convert_</span><span class="si">{source_format}</span><span class="s2">_to_</span><span class="si">{target_format}</span><span class="s2">.sif&quot;</span>
+        <span class="n">image</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">converter_images</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{source_format}</span><span class="s2">_to_</span><span class="si">{target_format}</span><span class="s2">&quot;</span><span class="p">]</span>
+        <span class="n">version</span><span class="p">,</span> <span class="n">image</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parse_docker_image_version</span><span class="p">(</span><span class="n">image</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">version</span><span class="p">:</span>
+            <span class="n">chosen_converter</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;convert_</span><span class="si">{source_format}</span><span class="s2">_to_</span><span class="si">{target_format}</span><span class="s2">_</span><span class="si">{version}</span><span class="s2">.sif&quot;</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">chosen_converter</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;convert_</span><span class="si">{source_format}</span><span class="s2">_to_</span><span class="si">{target_format}</span><span class="s2">.sif&quot;</span>
+
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Converting with </span><span class="si">{chosen_converter}</span><span class="s2">&quot;</span><span class="p">)</span>
         <span class="n">sbatch_env</span> <span class="o">=</span> <span class="p">{</span>
             <span class="s2">&quot;DATA_PATH&quot;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="se">\&quot;</span><span class="si">{data_path}</span><span class="se">\&quot;</span><span class="s2">&quot;</span><span class="p">,</span>
             <span class="s2">&quot;CONVERSION_PATH&quot;</span><span class="p">:</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="se">\&quot;</span><span class="si">{self.slurm_converters_path}</span><span class="se">\&quot;</span><span class="s2">&quot;</span><span class="p">,</span>

biomero.html

@@ -908,6 +908,14 @@ <h1>biomero package<a class="headerlink" href="#biomero-package" title="Permalin
 </dl>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="biomero.slurm_client.SlurmClient.list_available_converter_versions">
+<span class="sig-name descname"><span class="pre">list_available_converter_versions</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Dict" title="(in Python v3.12)"><span class="pre">Dict</span></a></span></span><a class="reference internal" href="_modules/biomero/slurm_client.html#SlurmClient.list_available_converter_versions"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#biomero.slurm_client.SlurmClient.list_available_converter_versions" title="Permalink to this definition"></a></dt>
+<dd><p>Note, assumes you use versioned converters.
+Will return a dict with a version of each converter on your Slurm.
+However, doesn’t work properly with unversioned sif.</p>
+</dd></dl>
+
 <dl class="py method">
 <dt class="sig sig-object py" id="biomero.slurm_client.SlurmClient.list_completed_jobs">
 <span class="sig-name descname"><span class="pre">list_completed_jobs</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">env</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Optional" title="(in Python v3.12)"><span class="pre">Optional</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Dict" title="(in Python v3.12)"><span class="pre">Dict</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.12)"><span class="pre">str</span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.12)"><span class="pre">str</span></a><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.12)"><span class="pre">List</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.12)"><span class="pre">str</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="reference internal" href="_modules/biomero/slurm_client.html#SlurmClient.list_completed_jobs"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#biomero.slurm_client.SlurmClient.list_completed_jobs" title="Permalink to this definition"></a></dt>

genindex.html

@@ -245,10 +245,12 @@ <h2 id="L">L</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="biomero.html#biomero.slurm_client.SlurmClient.list_active_jobs">list_active_jobs() (biomero.slurm_client.SlurmClient method)</a>
+</li>
+      <li><a href="biomero.html#biomero.slurm_client.SlurmClient.list_all_jobs">list_all_jobs() (biomero.slurm_client.SlurmClient method)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="biomero.html#biomero.slurm_client.SlurmClient.list_all_jobs">list_all_jobs() (biomero.slurm_client.SlurmClient method)</a>
+      <li><a href="biomero.html#biomero.slurm_client.SlurmClient.list_available_converter_versions">list_available_converter_versions() (biomero.slurm_client.SlurmClient method)</a>
 </li>
       <li><a href="biomero.html#biomero.slurm_client.SlurmClient.list_completed_jobs">list_completed_jobs() (biomero.slurm_client.SlurmClient method)</a>
 </li>