Fix pandas version in runtime container (#270)

- Add pandas 1.3 to conda recipe. This will allow pandas version in conda package to match pandas from runtime pip dependencies. When they don't match pypi package was being installed.
- Update Dockerfile so that runtime container build conda installs morpheus before installing runtime pip dependencies. pandas from conda-forge will first be installed. Since pandas from `requirements.txt` will also resolve to 1.3, pandas from conda-forge will now be kept.
- Update log parsing example to explicitly convert cupy arrays to numpy in postprocessing stage. Implicit conversion causes error in pandas 1.4.3. See #267.

Fixes #267

Authors:
  - Eli Fajardo (https://github.com/efajardo-nv)
  - https://github.com/apps/dependabot
  - https://github.com/raykallen
  - Michael Demoret (https://github.com/mdemoret-nv)
  - gpuCI (https://github.com/GPUtester)
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - David Gardner (https://github.com/dagardner-nv)
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #270

ci/Jenkinsfile

@@ -44,15 +44,13 @@ pipeline {
                         timeout(time: 4, unit: 'HOURS')
                     }
                     environment {
-                        PARALLEL_LEVEL = '10'
                         USE_SCCACHE = '1'
                         HOME = "${WORKSPACE}"
                     }
                     agent {
                         docker {
-                            image 'gpuci/rapidsai:22.04-cuda11.5-devel-centos7-py3.8'
-                            label 'driver-495'
-                            args '--runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"'
+                            image 'gpuci/rapidsai-driver:22.04-cuda11.5-devel-centos7-py3.8'
+                            label 'cpu4'
                         }
                     }
                     steps {
@@ -90,6 +88,7 @@ pipeline {
                     }
                     environment {
                         HOME = "${WORKSPACE}"
+                        PARALLEL_LEVEL = '10'
                     }
                     steps {
                         cleanWs(
@@ -114,9 +113,8 @@ pipeline {
                     }
                     agent {
                         docker {
-                            image 'gpuci/rapidsai:22.04-cuda11.5-devel-centos7-py3.8'
-                            label 'driver-495'
-                            args '--runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"'
+                            image 'gpuci/rapidsai-driver:22.04-cuda11.5-devel-centos7-py3.8'
+                            label 'cpu'
                         }
                     }
                     environment {

ci/conda/recipes/morpheus/meta.yaml

@@ -77,6 +77,7 @@ outputs:
         - grpcio # Version determined from cudf
         - mlflow >=1.23
         - networkx
+        - pandas 1.3
         - python
         - tqdm
         - typing_utils

ci/scripts/jenkins/build.sh

@@ -23,28 +23,20 @@ rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/
 conda config --add pkgs_dirs /opt/conda/pkgs
 conda config --env --add channels conda-forge
 conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS:-"https://conda.anaconda.org"}
-mamba install -q -y -n base -c conda-forge "boa >=0.10"
-mamba create -q -y -n morpheus python=${PYTHON_VER}
+mamba env create -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml
 conda activate morpheus
 
 gpuci_logger "Installing CI dependencies"
 mamba env update -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_ci.yml
+conda deactivate && conda activate morpheus
+
+gpuci_logger "Final Conda Environment"
+show_conda_info
 
 gpuci_logger "Check versions"
 python3 --version
 gcc --version
 g++ --version
-
-show_conda_info
-
-gpuci_logger "Installing dependencies"
-mamba env update -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml
-conda deactivate && conda activate morpheus
-
-gpuci_logger "Final Conda Environment"
-conda list
-
-gpuci_logger "Check cmake & ninja"
 cmake --version
 ninja --version
 

ci/scripts/jenkins/common.sh

@@ -17,9 +17,16 @@
 gpuci_logger "Env Setup"
 source /opt/conda/etc/profile.d/conda.sh
 export MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
-gpuci_logger "Procs: $(nproc)"
-gpuci_logger "Memory"
 
+# For non-gpu hosts nproc will correctly report the number of cores we are able to use
+# On a GPU host however nproc will report the total number of cores and PARALLEL_LEVEL
+# will be defined specifying the subset we are allowed to use.
+NUM_CORES=$(nproc)
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-${NUM_CORES}}
+gpuci_logger "Procs: ${NUM_CORES}"
+/usr/bin/lscpu
+
+gpuci_logger "Memory"
 /usr/bin/free -g
 
 gpuci_logger "User Info"

ci/scripts/jenkins/docs.sh

@@ -17,7 +17,6 @@
 set -e
 
 source ${WORKSPACE}/ci/scripts/jenkins/common.sh
-/usr/bin/nvidia-smi
 
 restore_conda_env
 pip install ${MORPHEUS_ROOT}/build/wheel
@@ -27,7 +26,7 @@ gpuci_logger "Installing Documentation dependencies"
 pip install -r requirement.txt
 
 gpuci_logger "Building docs"
-make html
+make -j ${PARALLEL_LEVEL} html
 
 gpuci_logger "Tarring the docs"
 tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/html

docker/Dockerfile

@@ -138,10 +138,10 @@ COPY docker/conda/environments/cuda${CUDA_VER}_runtime.yml ./docker/conda/enviro
 RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \
     --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
     source activate morpheus &&\
-    # Install runtime dependencies that are pip-only
-    /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_VER}_runtime.yml &&\
     # Install morpheus
     CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus -c local -c rapidsai -c nvidia -c nvidia/label/dev -c conda-forge morpheus &&\
+    # Install runtime dependencies that are pip-only
+    /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_VER}_runtime.yml &&\
     # Clean and activate
     conda clean -afy
 

examples/log_parsing/postprocessing.py

@@ -58,7 +58,7 @@ def accepted_types(self) -> typing.Tuple:
 
     def _postprocess(self, x: MultiPostprocLogParsingMessage):
 
-        infer_pdf = pd.DataFrame(x.seq_ids).astype(int)
+        infer_pdf = pd.DataFrame(x.seq_ids.get()).astype(int)
         infer_pdf.columns = ["doc", "start", "stop"]
         infer_pdf["confidences"] = x.confidences.tolist()
         infer_pdf["labels"] = x.labels.tolist()

models/training-tuning-scripts/fraud-detection-models/requirements.txt

@@ -3,7 +3,7 @@ cuml==0+untagged.1.g835a9ae
 dateparser==1.1.0
 matplotlib==3.4.3
 networkx==2.6.3
-numpy==1.19.5
+numpy==1.22.0
 pandas==1.3.3
 scikit_learn==1.0.2
 stellargraph==1.2.1

models/validation-inference-scripts/fraud-detection-models/requirements.txt

@@ -3,7 +3,7 @@ cuml==0+untagged.1.g835a9ae
 dateparser==1.1.0
 matplotlib==3.4.3
 networkx==2.6.3
-numpy==1.19.5
+numpy==1.22.0
 pandas==1.3.3
 scikit_learn==1.0.2
 stellargraph==1.2.1

models/validation-inference-scripts/hammah-models/requirements.txt

@@ -3,7 +3,7 @@ cupy==9.1.0
 dfencoder==0.0.35
 dill==0.3.4
 matplotlib==3.4.2
-numpy==1.20.3
+numpy==1.22.0
 pandas==1.0.1
 scikit_learn==1.0.2
 torch==1.7.1

models/validation-inference-scripts/phishing-models/requirements.txt

@@ -1,7 +1,7 @@
 clx==0.0.1
 cudf==0+untagged.1.g788bddd
 cupy==9.1.0
-numpy==1.20.3
+numpy==1.22.0
 onnxruntime==1.10.0
 s3fs==2021.5.0
 scipy==1.6.0