Modernize OpenVINO-based Nuclio functions and allow them to run on Ku…

…bernetes (#6129)

Currently, OpenVINO-based functions assume that a local directory will
be mounted into the container. In Kubernetes, that isn't possible, so
implement an alternate approach: create a separate base image and
inherit the function image from it.

In addition, implement some modernizations:

* Upgrade the version of OpenVINO to the latest (2022.3). Make the
necessary updates to the code. Note that 2022.1 introduced an entirely
new inference API, but I haven't switched to it yet to minimize changes.

* Use the runtime version of the Docker image as the base instead of the
dev version. This significantly reduces the size of the final image (by
~3GB).

* Replace the `faster_rcnn_inception_v2_coco` model with
`faster_rcnn_inception_resnet_v2_atrous_coco`, as the former has been
  removed from OMZ.

* Ditto with `person-reidentification-retail-0300` -> `0277`.

* The IRs used in the DEXTR function are not supported by OpenVINO
anymore (format too old), so rewrite the build process to create them
from the original code/weights instead.

CHANGELOG.md

@@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - Running SAM masks decoder on frontend (<https://github.com/opencv/cvat/pull/6019>)
+- The `person-reidentification-retail-0300` and
+  `faster_rcnn_inception_v2_coco` Nuclio functions were replaced with
+  `person-reidentification-retail-0277` and
+  `faster_rcnn_inception_resnet_v2_atrous_coco`, respectively
+  (<https://github.com/opencv/cvat/pull/6129>).
+- OpenVINO-based Nuclio functions now use the OpenVINO 2022.3 runtime
+  (<https://github.com/opencv/cvat/pull/6129>).
 
 ### Deprecated
 - TDB
@@ -24,6 +31,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The issue azure.core.exceptions.ResourceExistsError: The specified blob already exists (<https://github.com/opencv/cvat/pull/6082>)
 - Image scaling when moving between images with different resolution (<https://github.com/opencv/cvat/pull/6081>)
 - Invalid completed job count reporting (<https://github.com/opencv/cvat/issues/6098>)
+- OpenVINO-based Nuclio functions can now be deployed to Kubernetes
+  (<https://github.com/opencv/cvat/pull/6129>).
 
 ### Security
 - TDB

serverless/deploy_cpu.sh

@@ -1,22 +1,30 @@
 #!/bin/bash
 # Sample commands to deploy nuclio functions on CPU
 
+set -eu
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 FUNCTIONS_DIR=${1:-$SCRIPT_DIR}
 
-nuctl create project cvat
+export DOCKER_BUILDKIT=1
+
+docker build -t cvat.openvino.base "$SCRIPT_DIR/openvino/base"
+
+nuctl create project cvat --platform local
 
 shopt -s globstar
 
 for func_config in "$FUNCTIONS_DIR"/**/function.yaml
 do
-    func_root=$(dirname "$func_config")
-    echo Deploying $(dirname "$func_root") function...
-    nuctl deploy --project-name cvat --path "$func_root" \
-        --volume "$SCRIPT_DIR/common:/opt/nuclio/common" \
-        --platform local
-done
+    func_root="$(dirname "$func_config")"
+    func_rel_path="$(realpath --relative-to="$SCRIPT_DIR" "$(dirname "$func_root")")"
 
-nuctl get function
+    if [ -f "$func_root/Dockerfile" ]; then
+        docker build -t "cvat.${func_rel_path//\//.}.base" "$func_root"
+    fi
 
+    echo "Deploying $func_rel_path function..."
+    nuctl deploy --project-name cvat --path "$func_root" --platform local
+done
 
+nuctl get function --platform local

serverless/deploy_gpu.sh

@@ -1,20 +1,23 @@
 #!/bin/bash
 # Sample commands to deploy nuclio functions on GPU
 
+set -eu
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 FUNCTIONS_DIR=${1:-$SCRIPT_DIR}
 
-nuctl create project cvat
+nuctl create project cvat --platform local
 
 shopt -s globstar
 
 for func_config in "$FUNCTIONS_DIR"/**/function-gpu.yaml
 do
-    func_root=$(dirname "$func_config")
-    echo "Deploying $(dirname "$func_root") function..."
+    func_root="$(dirname "$func_config")"
+    func_rel_path="$(realpath --relative-to="$SCRIPT_DIR" "$(dirname "$func_root")")"
+
+    echo "Deploying $func_rel_path function..."
     nuctl deploy --project-name cvat --path "$func_root" \
-        --volume "$SCRIPT_DIR/common:/opt/nuclio/common" \
         --file "$func_config" --platform local
 done
 
-nuctl get function
+nuctl get function --platform local

serverless/openvino/base/Dockerfile

@@ -0,0 +1,15 @@
+FROM openvino/ubuntu20_runtime:2022.3.0
+
+USER root
+
+RUN apt-get update \
+    && apt-get -y --no-install-recommends install python-is-python3 \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir opencv-python-headless pillow pyyaml
+
+COPY model_loader.py shared.py /opt/nuclio/common/openvino/
+
+ENV PYTHONPATH=/opt/nuclio/common/openvino:$PYTHONPATH
+
+USER openvino

serverless/common/openvino/model_loader.py → serverless/openvino/base/model_loader.py

@@ -14,24 +14,22 @@ def __init__(self, model, weights):
 
         # Initialize input blobs
         self._input_info_name = None
-        for blob_name in network.inputs:
-            if len(network.inputs[blob_name].shape) == 4:
+        for blob_name in network.input_info:
+            if len(network.input_info[blob_name].tensor_desc.dims) == 4:
                 self._input_blob_name = blob_name
-            elif len(network.inputs[blob_name].shape) == 2:
+                self._input_layout = network.input_info[blob_name].tensor_desc.dims
+            elif len(network.input_info[blob_name].tensor_desc.dims) == 2:
                 self._input_info_name = blob_name
             else:
                 raise RuntimeError(
                     "Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported"
-                    .format(len(network.inputs[blob_name].shape), blob_name))
+                    .format(len(network.input_info[blob_name].tensor_desc.dims), blob_name))
 
         # Initialize output blob
         self._output_blob_name = next(iter(network.outputs))
 
         # Load network
         self._net = ie_core.load_network(network, "CPU", num_requests=2)
-        input_type = network.inputs[self._input_blob_name]
-        self._input_layout = input_type if isinstance(input_type, list) else input_type.shape
-
 
     def _prepare_inputs(self, image, preprocessing):
         image = np.array(image)
@@ -67,5 +65,5 @@ def input_size(self):
         return self._input_layout[2:]
 
     @property
-    def layers(self):
-        return self._network.layers
+    def network(self):
+        return self._network

serverless/openvino/dextr/nuclio/Dockerfile

@@ -0,0 +1,32 @@
+FROM openvino/ubuntu20_dev:2022.3.0 AS build
+
+USER root
+
+RUN apt-get update \
+    && apt-get -y --no-install-recommends install patch \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /root
+
+ARG DEXTR_COMMIT=352ccc76067156ebcf7267b07e0a5e43d32e83d5
+
+# TODO: use `ADD --checksum` when that feature becomes stable
+ADD https://data.vision.ee.ethz.ch/csergi/share/DEXTR/dextr_pascal-sbd.pth ./
+
+ADD https://github.com/scaelles/DEXTR-PyTorch/archive/$DEXTR_COMMIT.zip dextr.zip
+
+RUN python3 -m zipfile -e dextr.zip .
+
+WORKDIR /root/DEXTR-PyTorch-$DEXTR_COMMIT
+
+ADD export.py adaptive-pool.patch .
+
+RUN patch -p1 -i adaptive-pool.patch
+
+RUN python3 export.py /root/dextr_pascal-sbd.pth /root/dextr.onnx
+
+RUN mo --input_model=/root/dextr.onnx --model_name=dextr --output_dir=/root
+
+FROM cvat.openvino.base
+
+COPY --from=build --chown=root:root /root/dextr.xml /root/dextr.bin /opt/nuclio/

serverless/openvino/dextr/nuclio/adaptive-pool.patch

@@ -0,0 +1,27 @@
+This is a hack to work around the the lack of support for AdaptiveAvgPool2d
+in PyTorch's ONNX exporter (<https://github.com/pytorch/pytorch/issues/42653>).
+
+It might become unnecessary in the future, since OpenVINO 2023 is to add support
+for AdaptiveAvgPool2d exported with operator_export_type=ONNX_ATEN_FALLBACK
+(<https://github.com/openvinotoolkit/openvino/pull/14682>).
+
+diff --git a/networks/deeplab_resnet.py b/networks/deeplab_resnet.py
+index ecfa084..e8ff297 100644
+--- a/networks/deeplab_resnet.py
++++ b/networks/deeplab_resnet.py
+@@ -99,7 +99,14 @@ class PSPModule(nn.Module):
+         self.final = nn.Conv2d(out_features, n_classes, kernel_size=1)
+
+     def _make_stage_1(self, in_features, size):
+-        prior = nn.AdaptiveAvgPool2d(output_size=(size, size))
++        kernel_size, stride = {
++            1: (64, 64),
++            2: (32, 32),
++            3: (22, 21),
++            6: (11, 9),
++        }[size]
++
++        prior = nn.AvgPool2d(kernel_size=kernel_size, stride=stride)
+         conv = nn.Conv2d(in_features, in_features//4, kernel_size=1, bias=False)
+         bn = nn.BatchNorm2d(in_features//4, affine=affine_par)
+         relu = nn.ReLU(inplace=True)

serverless/openvino/dextr/nuclio/export.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import sys
+
+import torch
+import torch.nn
+import torch.onnx
+
+import networks.deeplab_resnet as resnet
+
+net = resnet.resnet101(1, nInputChannels=4, classifier='psp')
+
+state_dict_checkpoint = torch.load(sys.argv[1], map_location=torch.device('cpu'))
+
+net.load_state_dict(state_dict_checkpoint)
+
+full_net = torch.nn.Sequential(
+    net,
+    torch.nn.Upsample((512, 512), mode='bilinear', align_corners=True),
+    torch.nn.Sigmoid(),
+)
+full_net.eval()
+
+input_tensor = torch.randn((1, 4, 512, 512))
+
+torch.onnx.export(full_net, input_tensor, sys.argv[2])

serverless/openvino/dextr/nuclio/function.yaml

@@ -13,43 +13,13 @@ metadata:
 
 spec:
   description: Deep Extreme Cut
-  runtime: 'python:3.6'
+  runtime: 'python:3.8'
   handler: main:handler
   eventTimeout: 30s
-  env:
-    - name: NUCLIO_PYTHON_EXE_PATH
-      value: /opt/nuclio/common/openvino/python3
-
-  volumes:
-    - volume:
-        name: openvino-common
-        configMap:
-          name: "cvat-nuclio-openvino-common"
-          defaultMode: 0750
-      volumeMount:
-        name: openvino-common
-        mountPath: /opt/nuclio/common/openvino
 
   build:
-    image: cvat/openvino.dextr
-    baseImage: openvino/ubuntu18_runtime:2020.2
-
-    directives:
-      preCopy:
-        - kind: USER
-          value: root
-        - kind: WORKDIR
-          value: /opt/nuclio
-        - kind: RUN
-          value: ln -s /usr/bin/pip3 /usr/bin/pip
-
-      postCopy:
-        - kind: RUN
-          value: curl -O https://download.01.org/openvinotoolkit/models_contrib/cvat/dextr_model_v1.zip
-        - kind: RUN
-          value: unzip dextr_model_v1.zip
-        - kind: RUN
-          value: pip3 install -U pip && pip3 install wheel Pillow
+    image: cvat.openvino.dextr
+    baseImage: cvat.openvino.dextr.base
 
   triggers:
     myHttpTrigger:

serverless/openvino/omz/intel/face-detection-0205/nuclio/Dockerfile

@@ -0,0 +1,12 @@
+FROM openvino/ubuntu20_dev:2022.3.0 AS build
+
+USER root
+
+RUN omz_downloader \
+    --name face-detection-0205,emotions-recognition-retail-0003,age-gender-recognition-retail-0013 \
+    --precisions FP32 \
+    -o /opt/nuclio/open_model_zoo
+
+FROM cvat.openvino.base
+
+COPY --from=build --chown=root:root /opt/nuclio/open_model_zoo /opt/nuclio/open_model_zoo

serverless/openvino/omz/intel/face-detection-0205/nuclio/function.yaml

@@ -1,5 +1,5 @@
 metadata:
-  name: openvino-omz-face-detection-0205
+  name: openvino-omz-intel-face-detection-0205
   namespace: cvat
   annotations:
     name: Attributed face detection
@@ -28,47 +28,13 @@ metadata:
 
 spec:
   description: Detection network finding faces and defining age, gender and emotion attributes
-  runtime: 'python:3.6'
+  runtime: 'python:3.8'
   handler: main:handler
   eventTimeout: 30000s
-  env:
-    - name: NUCLIO_PYTHON_EXE_PATH
-      value: /opt/nuclio/common/openvino/python3
-
-  volumes:
-    - volume:
-        name: openvino-common
-        configMap:
-          name: "cvat-nuclio-openvino-common"
-          defaultMode: 0750
-      volumeMount:
-        name: openvino-common
-        mountPath: /opt/nuclio/common/openvino
 
   build:
     image: cvat.openvino.omz.intel.face-detection-0205
-    baseImage: openvino/ubuntu18_dev:2021.1
-
-    directives:
-      preCopy:
-        - kind: USER
-          value: root
-        - kind: WORKDIR
-          value: /opt/nuclio
-        - kind: RUN
-          value: ln -s /usr/bin/pip3 /usr/bin/pip
-        - kind: RUN
-          value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name face-detection-0205 -o /opt/nuclio/open_model_zoo
-        - kind: RUN
-          value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name emotions-recognition-retail-0003 -o /opt/nuclio/open_model_zoo
-        - kind: RUN
-          value: /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/downloader.py --name age-gender-recognition-retail-0013 -o /opt/nuclio/open_model_zoo
-
-      postCopy:
-        - kind: RUN
-          value: apt update && DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y python3-skimage
-        - kind: RUN
-          value: pip3 install "numpy<1.16.0" # workaround for skimage
+    baseImage: cvat.openvino.omz.intel.face-detection-0205.base
 
   triggers:
     myHttpTrigger:

serverless/openvino/omz/intel/person-reidentification-retail-0277/nuclio/Dockerfile

@@ -0,0 +1,18 @@
+FROM openvino/ubuntu20_dev:2022.3.0 AS build
+
+USER root
+
+RUN omz_downloader \
+    --name person-reidentification-retail-0277 \
+    --precisions FP32 \
+    -o /opt/nuclio/open_model_zoo
+
+FROM cvat.openvino.base
+
+USER root
+
+RUN pip install --no-cache-dir scipy
+
+COPY --from=build /opt/nuclio/open_model_zoo /opt/nuclio/open_model_zoo
+
+USER openvino