ci(docker): add tuna source (#18)

Co-authored-by: MegEngine <megengine@megvii.com>

Dockerfile

@@ -11,18 +11,20 @@ RUN apt update \
     && apt install -y git \
     && apt install -y build-essential
 
+ENV CARGO_HOME /cargo
+ENV RUSTUP_HOME /rustup
 RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf -o run.sh \
-    && chmod a+x run.sh \
-    && ./run.sh -y \
-    && export PATH=$HOME/.cargo/bin:${PATH} \
-    && cargo --version
+	&& chmod a+x run.sh \
+	&& ./run.sh -y && rm run.sh
+ENV PATH $PATH:/cargo/bin
+RUN chmod -R 777 /cargo /rustup
+COPY ci/cargo-config /cargo/config
 
 RUN mkdir -p $HOME/megflow-runspace
 WORKDIR $HOME/megflow-runspace
 COPY . $HOME/megflow-runspace/
 
-RUN PATH=$HOME/.cargo/bin:${PATH} \
-    && cargo build \
+RUN cargo build \
     && cd flow-python \
     && python3 setup.py install --user \
     && cd examples \

README.md

@@ -2,7 +2,7 @@
   <img width="60%"  src="logo.png">
 </div>
 
-## MegFlow
+## MegFlow | [Documentation](https://megflow.readthedocs.io/zh_CN/latest/#)
 [![GitHub license](https://img.shields.io/badge/license-apache--2--Clause-brightgreen.svg)](./LICENSE)
 [![ubuntu](https://img.shields.io/github/workflow/status/MegEngine/MegFlow/ubuntu-x86-cpu?label=ubuntu)](https://github.com/MegEngine/MegFlow/actions/workflows/ubuntu-x86-cpu.yml?query=workflow%3A)
 [![macos](https://img.shields.io/github/workflow/status/MegEngine/MegFlow/ubuntu-x86-cpu?label=macos)](https://github.com/MegEngine/MegFlow/actions/workflows/macos-x86-cpu.yml?query=workflow%3A)
@@ -57,7 +57,6 @@ MegFlow 提供快速视觉应用落地流程，最快 15 分钟搭建起视频
 - Python 插件内置有栈协程，不依赖 asyncio
 - 基础测试工具，支持插件沙盒，用于单测插件
 
-
 ## Coming Soon
 - 进程级别的节点、子图支持
 - 插件自动化测试部署

doc_link_checker.py → ci/doc_link_checker.py

@@ -1,5 +1,8 @@
+# /bin/python3
+
 import os
 import re
+import sys
 
 pattern = re.compile(r'\[.*?\]\(.*?\)')
 def analyze_doc(home, path):
@@ -13,19 +16,20 @@ def analyze_doc(home, path):
                     start = item.find('(')
                     end = item.find(')')
                     ref = item[start+1: end]
-                    if ref.startswith('http'):
+                    if ref.startswith('http') or ref.startswith('#'):
                         continue
                     fullpath = os.path.join(home, ref)
                     if not os.path.exists(fullpath):
                         problem_list.append(ref)
                         # print(f' {fullpath}  in  {path} not exist!')
             else:
-                continue     
+                continue
     if len(problem_list) > 0:
         print(f'{path}:')
         for item in problem_list:
             print(f'\t {item}')
         print('\n')
+        sys.exit(1)
 
 def traverse(_dir):
     for home, dirs, files in os.walk(_dir):

ci/run_pylint_check.sh

@@ -0,0 +1,6 @@
+python -m pip install pylint==2.5.2
+CHECK_DIR="flow-python/examples/simple_classification flow-python/examples/cat_finder flow-python/examples/electric_bicycle"
+pylint $CHECK_DIR || pylint_ret=$?
+if [ "$pylint_ret" ]; then
+    exit $pylint_ret
+fi

docs/Makefile

@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+

docs/conf.py

@@ -0,0 +1,70 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+#sys.path.insert(0, os.path.abspath('.'))
+
+import sphinx_rtd_theme
+from recommonmark.parser import CommonMarkParser
+from recommonmark.transform import AutoStructify
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'MegFlow'
+copyright = '2021, Megvii'
+author = 'tpoisonooo'
+
+# The full version, including alpha/beta/rc tags
+release = ''
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['recommonmark', 'sphinx_markdown_tables']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'zh'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = ['.rst', '.md']

docs/how-to-add-my-service/01-single-classification-model.zh.md

@@ -4,9 +4,9 @@
 
 ## 准备分类模型
 
-[MegEngine models]() 有现成的 imagenet 预训模型。这里把模型 dump 成 .mge。
+[MegEngine models](https://github.com/MegEngine/models) 有现成的 imagenet 预训模型。这里把模型 dump 成 .mge。
 
-新增 [dump.py](../../flow-python/examples/simple_classification/dump.py)，按 [1, 3, 224, 224] 尺寸 trace 模型，打开推理优化选项，保存为 `model.mge`。
+新增 [dump.py](https://github.com/MegEngine/Models/blob/master/official/vision/classification/dump.py)，按 [1, 3, 224, 224] 尺寸 trace 模型，打开推理优化选项，保存为 `model.mge`。
 
 ```bash
 $ git clone https://github.com/MegEngine/models
@@ -45,6 +45,8 @@ $ cat dump.py
 $ cat flow-python/examples/simple_classification/lite.py
 ...
     def inference(self, mat):
+        img = self.preprocess(mat, input_size=(224,224), scale_im = False, mean=[103.530, 116.280, 123.675], std=[57.375, 57.120, 58.395])
+
         # 设置输入
         inp_data =self.net.get_io_tensor("data")
         inp_data.set_data_by_share(img)
@@ -108,14 +110,7 @@ $ cat flow-python/examples/simple_classification/classify.py
 @register(inputs=['inp'], outputs=['out'])
 class Classify:
     def __init__(self, name, args):
-        logger.info("loading Resnet18 Classification...")
-        self.name = name
-
-        # load model and warmup
-        self._model = PredictorLite(path=args['path'], device=args['device'], device_id=args['device_id'])
-        warmup_data = np.zeros((224, 224, 3), dtype=np.uint8)
-        self._model.inference(warmup_data)
-        logger.info("Resnet18  loaded.")
+        ...
 
     def exec(self):
         envelope = self.inp.recv()

docs/how-to-build-and-run/build-from-source.zh.md

@@ -4,7 +4,7 @@
 
 ### 安装 Rust
 ```bash
-$ sudo apt install yasm git build-essential ffmpeg curl
+$ sudo apt install curl
 $ curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | sh
 ```
 
@@ -14,6 +14,11 @@ $ cargo --version
 cargo 1.53.0 (4369396ce 2021-04-27)
 ```
 
+如果不成功，提示`Command 'cargo' not found`，可以按照提示加载一下环境变量(重新连接或打开终端也可以)：
+```
+source $HOME/.cargo/env
+```
+
 > `cargo` 是 Rust 的包管理器兼编译辅助工具。类似 Java maven/go pkg/C++ CMake 的角色，更易使用。
 
 ### 安装 python3.x （推荐 conda）
@@ -44,7 +49,7 @@ $ conda activate py38
 MegFlow 需要编译 ffmpeg。考虑到 ffmpeg 依赖较多、本身又是常用工具，最简单的办法就是直接装 ffmpeg 把编译依赖装上
 
 ```bash
-$ sudo apt install ffmpeg
+$ sudo apt install yasm git build-essential ffmpeg
 $ ffmpeg 
 ffmpeg version 3.4.8...
 $ sudo apt install clang

docs/index.rst

@@ -0,0 +1,36 @@
+.. MegFlow documentation master file, created by
+   sphinx-quickstart on Wed Mar 24 22:40:10 2021.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+Welcome to MegFlow's documentation!
+===================================
+
+
+*Please select a specific version of the document in the lower left corner of the page.*
+
+.. toctree::
+  :maxdepth: 1 
+  :caption: how to build and run
+  :name: sec-introduction
+
+  how-to-build-and-run/build-with-docker.zh
+  how-to-build-and-run/build-from-source.zh
+  how-to-build-and-run/build-on-win10.zh
+  how-to-build-and-run/generate-rtsp.zh
+
+.. toctree::
+  :maxdepth: 1
+  :caption: how to use 
+  :name: sec-how-to-use
+
+  how-to-add-my-service/01-single-classification-model.zh
+  how-to-add-my-service/appendix-A-graph-definition
+  how-to-add-my-service/appendix-B-python-plugin.zh
+
+.. toctree::
+  :maxdepth: 1
+  :caption: home
+
+  FAQ.zh
+  how-to-contribute.zh
+  how-to-pack-python-whl.zh

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+recommonmark
+sphinx_markdown_tables
+sphinx_rtd_theme

flow-python/examples/cat_finder/det.py

@@ -9,10 +9,11 @@
 #!/usr/bin/env python
 # coding=utf-8
 
-from megflow import register
-from loguru import logger
-from warehouse.detection_yolox import *
 import numpy as np
+from loguru import logger
+from megflow import register
+from warehouse.detection_yolox import PredictorLite
+
 
 @register(inputs=['inp'], outputs=['out'])
 class Detect:
@@ -24,14 +25,19 @@ def __init__(self, name, args):
         self.name = name
 
         # load detect model and warmup
-        self._predictor = PredictorLite(path=args['path'], confthre=args['conf'], nmsthre=args['nms'], test_size=(self._tsize, self._tsize), device=args['device'], device_id=args['device_id'] )
+        self._predictor = PredictorLite(path=args['path'],
+                                        confthre=args['conf'],
+                                        nmsthre=args['nms'],
+                                        test_size=(self._tsize, self._tsize),
+                                        device=args['device'],
+                                        device_id=args['device_id'])
         warmup_data = np.zeros((224, 224, 3), dtype=np.uint8)
         self._predictor.inference(warmup_data)
         logger.info(" YOLOX loaded.")
 
     @staticmethod
     def restrict(val, min, max):
-        assert(min < max)
+        assert min < max
         if val < min:
             val = min
         if val > max:
@@ -72,7 +78,6 @@ def exec(self):
                 # name = 'frame{0:07d}.jpg'.format(envelope.partial_id)
                 # cv2.imwrite(name, x)
 
-            if 1 == self._visualize:
+            if self._visualize == 1:
                 image['data'] = self._predictor.visual(outputs, data)
             self.out.send(envelope)
-