Update morpheus dev container to support flag for building with debug python build + source files.

CONTRIBUTING.md

@@ -90,6 +90,16 @@ This workflow utilizes a docker container to set up most dependencies ensuring a
       DOCKER_IMAGE_TAG=my_tag ./docker/build_container_dev.sh
       ```
       Would build the container `morpheus:my_tag`.
+   1. To build the container with a debugging version of cpython installed, modify your build command as:
+   ```shell
+   DOCKER_EXTRA_ARGS="--build-arg MORPHEUS_WITH_PYDEBUG=true" docker/build_container_dev.sh
+   ```
+   1. Note: When debugging python code, you just need to add `/usr/local/src/python-dbg` to your debugger's source path.
+   1. Note: Now when running the container, conda should list your python version as `pyxxx_dbg_morpheus`.
+   ```shell
+    (morpheus) user@host:/workspace# conda list | grep python
+    python                    3.8.13          py3.8.13_dbg_morpheus    local
+   ```
    1. Note: This does not build any Morpheus or Neo code and defers building the code until the entire repo can be mounted into a running container. This allows for faster incremental builds during development.
 2. Set up `ssh-agent` to allow container to pull from private repos
    ```bash

ci/conda/recipes/python-dbg/build.sh

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+./configure --with-pydebug --enable-shared --enable-ipv6 --prefix=$PREFIX
+make -j
+make install
+
+
+cd $PREFIX/bin
+ln -s python${PYTHON_VER} python
+ln -s pydoc${PYTHON_VER} pydoc

ci/conda/recipes/python-dbg/conda_build_config.yaml

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+c_compiler_version:
+  - 9.4
+
+cxx_compiler_version:
+  - 9.4

ci/conda/recipes/python-dbg/meta.yaml

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% set version = environ.get('MORPHEUS_PYTHON_VER') %}
+
+package:
+  name: python
+  version: {{ version }}
+
+source:
+  fn: Python-{{ version }}.tar.xz
+  url: https://www.python.org/ftp/python/{{ version }}/Python-{{ version }}.tar.xz
+  # md5: c4b7100dcaace9d33ab1fda9a3a038d6
+  # If you want to build from the github source. This is quite a bit slower than pulling the tarball.
+  #git_url: https://github.com/python/cpython.git
+  #git_rev: 3.8
+
+build:
+  include_recipe: False
+  no_link: bin/python
+  number: 0
+  string: py{{ version }}_dbg_morpheus
+
+requirements:
+  build:
+    - bzip2       [unix]
+    - libffi      [unix] # Required for _ctypes, pip will complain that setuptools doesn't exist if _ctypes is missing.
+    - openssl >=1.1.1n,<2.0   # Required for TLS/SSL capabilities in pip.
+    - readline    [unix]
+    - sqlite      [unix]
+    - tk          [unix]
+    - xz          [unix]
+    - zlib        [unix]
+  run:
+    - bzip2       [unix]
+    - libffi      [unix]
+    - openssl >=1.1.1n,<2.0   # Required for TLS/SSL capabilities in pip.
+    - readline    [unix]
+    - sqlite      [unix]
+    - tk          [unix]
+    - xz          [unix]
+    - zlib        [unix]
+
+test:
+  commands:
+    - python -V               [unix]
+    - python3 -V              [unix]
+    - 2to3 -h
+    - pydoc -h
+    - python3-config --help   [unix]
+
+about:
+  home: http://www.python.org/
+  license: PSF
+  summary: general purpose programming language

docker/Dockerfile

@@ -178,10 +178,31 @@ COPY ["*.md", "LICENSE", "./"]
 # Use morpheus by default
 CMD [ "morpheus" ]
 
+# ============ Stage: development_deps ===========
+# Configure and build cpython with debugging symbols
+FROM conda_env_dev as development_deps
+
+# Capture python version
+ARG PYTHON_VER
+COPY ci/conda/recipes/python-dbg/ ./ci/conda/recipes/python-dbg
+
+# Flag indicating if we're going to build cpython
+ARG MORPHEUS_WITH_PYDEBUG=0
+
+RUN if [[ "${MORPHEUS_WITH_PYDEBUG}" =~ ^(TRUE|True|true|1)$ ]]; then \
+      ## Figure out which python version is in the existing environment -- we will build its debug equivalent \
+      source activate morpheus \
+      && \
+        MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2) \
+        CONDA_BLD_PATH=/opt/conda/conda-bld \
+        CONDA_ARGS="--no-test" \
+      conda mambabuild -c conda-forge -c system ./ci/conda/recipes/python-dbg \
+      ; \
+    fi
+
 # ============ Stage: development ============
 # Install and configure development only packages
-FROM conda_env_dev as development
-
+FROM development_deps as development
 # Copy the source
 # COPY . ./
 
@@ -192,3 +213,29 @@ RUN npm install -g camouflage-server
 # greater. See https://marc.info/?l=git&m=164989570902912&w=2. Only enable for
 # development
 RUN git config --global --add safe.directory "*"
+
+# Temporary until #68 goes in
+ARG MORPHEUS_USER="root"
+RUN --mount=type=bind,from=development_deps,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \
+    # swap out python binary if it was built \
+    if [[ "${MORPHEUS_WITH_PYDEBUG}" =~ ^(TRUE|True|true|1)$ ]]; then \
+      source activate morpheus \
+      && export source_file=$( ls /opt/conda/conda-bld/src_cache/Python-${PYTHON_VER}*.tar.xz ) \
+      && export install_file=$( ls /opt/conda/conda-bld/linux-64/python-${PYTHON_VER}*.tar.bz2 ) \
+      && mamba install --use-local ${install_file} \
+      && export debug_py_ver=$(python --version | cut -d ' ' -f2) \
+      ## Copy python includes from python3.xd to python3.x so CMake can find them \
+      && cp -R /opt/conda/envs/morpheus/include/python${PYTHON_VER}d/* \
+        /opt/conda/envs/morpheus/include/python${PYTHON_VER} \
+      # Extract cpython source to /workspace \
+      && for src in Include Misc Modules Objects Python; do  \
+          tar --strip-components=1 --extract --wildcards --file=${source_file} Python-${debug_py_ver}/${src}/* \
+          && mkdir -p /usr/local/src/python-dbg/${src} \
+          && cp -R ./${src} /usr/local/src/python-dbg/${src} \
+          ; \
+         done \
+      # Move cpython gdb helper macros to ${HOME}/.gdbinit \
+      # See: https://github.com/python/cpython/blob/main/Misc/gdbinit
+      && cp Misc/gdbinit $(eval echo "~${MORPHEUS_USER}")/.gdbinit \
+      ; \
+    fi