[µTVM] Modify reference VMs to support new µTVM demo (apache#7001)

apps/microtvm/reference-vm/base-box-tool.py

@@ -232,7 +232,8 @@ def do_build_release_test_vm(release_test_dir, user_box_dir, base_box_dir, provi
             box_package = os.path.join(
                 base_box_dir, f"output-packer-{provider_name}", "package.box"
             )
-            f.write(f'{m.group(1)} = "{os.path.relpath(box_package, release_test_dir)}"\n')
+            box_relpath = os.path.relpath(box_package, release_test_dir)
+            f.write(f'{m.group(1)} = "{box_relpath}"\n')
             found_box_line = True
 
     if not found_box_line:
@@ -242,6 +243,10 @@ def do_build_release_test_vm(release_test_dir, user_box_dir, base_box_dir, provi
         )
         return False
 
+    # Delete the old box registered with Vagrant, which may lead to a falsely-passing release test.
+    remove_args = ["vagrant", "box", "remove", box_relpath]
+    return_code = subprocess.call(remove_args, cwd=release_test_dir)
+    assert return_code in (0, 1), f'{" ".join(remove_args)} returned exit code {return_code}'
     subprocess.check_call(["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir)
 
     return True
@@ -281,7 +286,7 @@ def test_command(args):
         test_config["vid_hex"] = test_config["vid_hex"].lower()
         test_config["pid_hex"] = test_config["pid_hex"].lower()
 
-    providers = args.provider.split(",")
+    providers = args.provider
     provider_passed = {p: False for p in providers}
 
     release_test_dir = os.path.join(THIS_DIR, "release-test")
@@ -313,19 +318,28 @@ def test_command(args):
 
 
 def release_command(args):
-    #  subprocess.check_call(["vagrant", "cloud", "version", "create", f"tlcpack/microtvm-{args.platform}", args.version])
-    if not args.version:
-        sys.exit(f"--version must be specified")
+    subprocess.check_call(
+        [
+            "vagrant",
+            "cloud",
+            "version",
+            "create",
+            f"tlcpack/microtvm-{args.platform}",
+            args.release_version,
+        ]
+    )
+    if not args.release_version:
+        sys.exit(f"--release-version must be specified")
 
-    for provider_name in args.provider.split(","):
+    for provider_name in args.provider:
         subprocess.check_call(
             [
                 "vagrant",
                 "cloud",
                 "publish",
                 "-f",
                 f"tlcpack/microtvm-{args.platform}",
-                args.version,
+                args.release_version,
                 provider_name,
                 os.path.join(
                     THIS_DIR,
@@ -361,6 +375,8 @@ def parse_args():
     parser.add_argument(
         "--provider",
         choices=ALL_PROVIDERS,
+        action="append",
+        default=[],
         help="Name of the provider or providers to act on; if not specified, act on all",
     )
     parser.add_argument(
@@ -392,6 +408,9 @@ def main():
     if os.path.sep in args.platform or not os.path.isdir(os.path.join(THIS_DIR, args.platform)):
         sys.exit(f"<platform> must be a sub-direcotry of {THIS_DIR}; got {args.platform}")
 
+    if not args.provider:
+        args.provider = list(ALL_PROVIDERS)
+
     todo = []
     for phase in args.command.split(","):
         if phase not in ALL_COMMANDS:

apps/microtvm/reference-vm/zephyr/Vagrantfile

@@ -20,6 +20,10 @@ Vagrant.configure("2") do |config|
 
   tvm_home = "../../../.."
   dirs_to_mount = [Pathname.new(Pathname.new(tvm_home).expand_path())]
+  if ENV.has_key?("TVM_PROJECT_DIR") then
+    dirs_to_mount.append(ENV["TVM_PROJECT_DIR"])
+    puts "NOTE: also configuring project dir: %s" % [dirs_to_mount[-1]]
+  end
 
   git_file = Pathname.new(tvm_home + "/.git")
   if git_file.ftype() == "file" then

apps/microtvm/reference-vm/zephyr/base-box/setup.sh

@@ -27,10 +27,13 @@ wget --no-verbose https://apt.kitware.com/keys/kitware-archive-latest.asc
 sudo apt-key add kitware-archive-latest.asc
 sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
 sudo apt update
-sudo apt install -y --no-install-recommends git cmake ninja-build gperf \
-  ccache dfu-util device-tree-compiler wget \
-  python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \
-  make gcc gcc-multilib g++-multilib libsdl2-dev
+# NOTE: latest cmake cannot be installed due to
+# https://github.com/zephyrproject-rtos/zephyr/issues/30232
+sudo apt install -y --no-install-recommends git \
+     cmake=3.18.4-0kitware1 cmake-data=3.18.4-0kitware1 \
+     ninja-build gperf ccache dfu-util device-tree-compiler wget \
+     python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \
+     make gcc gcc-multilib g++-multilib libsdl2-dev
 
 # Avahi, so that ssh microtvm works.
 # apt install -y avahi-daemon

apps/microtvm/reference-vm/zephyr/base-box/test-config.json

@@ -0,0 +1,4 @@
+{"vid_hex": "0483",
+ "pid_hex": "374b",
+ "test_cmd": ["pytest", "tests/micro/qemu/test_zephyr.py", "--microtvm-platforms=stm32f746xx"]
+}

apps/microtvm/reference-vm/zephyr/pyproject.toml

@@ -92,8 +92,9 @@ onnx = {version = "1.6.0", optional = true}
 onnxruntime = {version = "1.0.0", optional = true}
 
 # Pytorch (also used by ONNX)
-torch = {version = "1.4.0", optional = true}
-torchvision = {version = "0.5.0", optional = true}
+# NOTE: cannot download this right now due to https://github.com/python-poetry/poetry/issues/2247
+# torch = {url = "https://download.pytorch.org/whl/cu101/torch-1.4.0-cp36-cp36m-manylinux1_x86_64.whl", optional = true}
+# torchvision = {version = "0.5.0", optional = true}
 # NOTE: torch depends on a number of other packages, but unhelpfully, does not expose that in the
 # wheel!!!
 future = {version = "*", optional = true}

apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh

@@ -28,6 +28,7 @@ fi
 cp cmake/config.cmake "${BUILD_DIR}"
 cd "${BUILD_DIR}"
 sed -i 's/USE_MICRO OFF/USE_MICRO ON/' config.cmake
+sed -i 's/USE_GRAPH_RUNTIME_DEBUG OFF/USE_GRAPH_RUNTIME_DEBUG ON/' config.cmake
 sed -i 's/USE_LLVM OFF/USE_LLVM ON/' config.cmake
 cmake ..
 make -j4

apps/microtvm/reference-vm/zephyr/setup.sh

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -ex
 
 # TVM
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
@@ -26,13 +26,16 @@ apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 cd apps/microtvm/reference-vm/zephyr
 
+poetry env use 3.6
+# NOTE: due to https://github.com/python-poetry/poetry/issues/2247, download torch here.
+poetry run pip3 install torch==1.4.0 torchvision==0.5.0
+
 echo "------------------------------[ TVM Message ]------------------------------"
 echo "WARNING: running 'poetry lock', which could take several minutes (depending"
 echo "on your network connection and the state of PyPI) as dependencies are"
 echo "downloaded and cached for future use."
 echo "------------------------------[ TVM Message ]------------------------------"
-
-poetry lock
+poetry lock -vvv
 poetry install
 poetry run pip3 install -r ~/zephyr/zephyr/scripts/requirements.txt