From 6009b26232636c87ca35e8d3716064655417bd32 Mon Sep 17 00:00:00 2001 From: Lanking Date: Thu, 2 Aug 2018 09:55:47 -0700 Subject: [PATCH 01/11] [MXNET-748] linker fixed on Scala issues (#11989) * put force load back as a temporary solution * use project.basedir as relative path for OSX linker --- scala-package/native/osx-x86_64-cpu/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 3f66fe68e041..e1c63104f9ad 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -73,6 +73,8 @@ -Wl,-exported_symbol,_Java_* -Wl,-x ${lddeps} + -force_load ${project.basedir}/../../../lib/libmxnet.a + -force_load ${project.basedir}/../../../3rdparty/tvm/nnvm/lib/libnnvm.a ${ldflags} From 946e9d018452e8b9a8096a90d15c7015311d894f Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Thu, 2 Aug 2018 10:09:43 -0700 Subject: [PATCH 02/11] [MXNET-772] Re-enable test_module.py:test_module_set_params (#11979) --- tests/python/unittest/test_module.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py index 41ea5828ddcd..a2a24762aa4b 100644 --- a/tests/python/unittest/test_module.py +++ b/tests/python/unittest/test_module.py @@ -317,8 +317,9 @@ def create_bucketing_module(key): assert total_bytes_after == total_bytes_before - -@with_seed(11) +# roywei: Getting rid of fixed seed as flakiness could not be reproduced, +# tracked at: https://github.com/apache/incubator-mxnet/issues/11705 +@with_seed() def test_module_set_params(): # data iter data = mx.nd.array([[0.05, .10]]); From 1bd9356b30ecde412663d0020ed769042cf456d6 Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Thu, 2 Aug 2018 10:11:17 -0700 Subject: [PATCH 03/11] [MXNET-771] Fix Flaky Test test_executor.py:test_dot (#11978) * use assert_almost_equal, increase rtol, reduce matrix size * remove seed in test_bind * add seed 0 to test_bind, it is still flaky * add comments for tracking --- tests/python/unittest/test_executor.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index 630cad87496d..3117f6646481 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -18,13 +18,7 @@ import numpy as np import mxnet as mx from common import setup_module, with_seed, teardown - - -def reldiff(a, b): - diff = np.sum(np.abs(a - b)) - norm = np.sum(np.abs(a)) - reldiff = diff / norm - return reldiff +from mxnet.test_utils import assert_almost_equal def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): @@ -64,9 +58,9 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): out1 = uf(lhs_arr.asnumpy(), rhs_arr.asnumpy()) out3 = exec3.outputs[0].asnumpy() out4 = exec4.outputs[0].asnumpy() - assert reldiff(out1, out2) < 1e-6 - assert reldiff(out1, out3) < 1e-6 - assert reldiff(out1, out4) < 1e-6 + assert_almost_equal(out1, out2, rtol=1e-5, atol=1e-5) + assert_almost_equal(out1, out3, rtol=1e-5, atol=1e-5) + assert_almost_equal(out1, out4, rtol=1e-5, atol=1e-5) # test gradient out_grad = mx.nd.array(np.ones(out2.shape)) lhs_grad2, rhs_grad2 = gf(out_grad.asnumpy(), @@ -74,8 +68,8 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None): rhs_arr.asnumpy()) executor.backward([out_grad]) - assert reldiff(lhs_grad.asnumpy(), lhs_grad2) < 1e-6 - assert reldiff(rhs_grad.asnumpy(), rhs_grad2) < 1e-6 + assert_almost_equal(lhs_grad.asnumpy(), lhs_grad2, rtol=1e-5, atol=1e-5) + assert_almost_equal(rhs_grad.asnumpy(), rhs_grad2, rtol=1e-5, atol=1e-5) @with_seed(0) @@ -118,12 +112,14 @@ def check_bind(disable_bulk_exec): check_bind(False) -@with_seed(0) +# @roywei: Removing fixed seed as flakiness in this test is fixed +# tracked at https://github.com/apache/incubator-mxnet/issues/11686 +@with_seed() def test_dot(): nrepeat = 10 maxdim = 4 for repeat in range(nrepeat): - s =tuple(np.random.randint(1, 500, size=3)) + s =tuple(np.random.randint(1, 200, size=3)) check_bind_with_uniform(lambda x, y: np.dot(x, y), lambda g, x, y: (np.dot(g, y.T), np.dot(x.T, g)), 2, @@ -131,7 +127,7 @@ def test_dot(): rshape=(s[1], s[2]), sf = mx.symbol.dot) for repeat in range(nrepeat): - s =tuple(np.random.randint(1, 500, size=1)) + s =tuple(np.random.randint(1, 200, size=1)) check_bind_with_uniform(lambda x, y: np.dot(x, y), lambda g, x, y: (g * y, g * x), 2, From 833de7e78118768c35358db7d2234f950fbd0637 Mon Sep 17 00:00:00 2001 From: jimdunn Date: Thu, 2 Aug 2018 14:38:19 -0700 Subject: [PATCH 04/11] remove mod from arity 2 version of load-checkpoint in clojure-package (#11808) * remove mod from arity 2 version of load-checkpoint * load-checkpoint arity 2 test --- .../src/org/apache/clojure_mxnet/module.clj | 4 +--- .../test/org/apache/clojure_mxnet/module_test.clj | 14 ++++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/contrib/clojure-package/src/org/apache/clojure_mxnet/module.clj b/contrib/clojure-package/src/org/apache/clojure_mxnet/module.clj index 22ab761547e2..ab6d345fe91d 100644 --- a/contrib/clojure-package/src/org/apache/clojure_mxnet/module.clj +++ b/contrib/clojure-package/src/org/apache/clojure_mxnet/module.clj @@ -309,7 +309,6 @@ (defn load-checkpoint "Create a model from previously saved checkpoint. - - mod module - opts map of - prefix Path prefix of saved model files. You should have prefix-symbol.json, prefix-xxxx.params, and optionally prefix-xxxx.states, @@ -341,7 +340,7 @@ (util/->option (when workload-list (util/vec->indexed-seq workload-list))) (util/->option (when fixed-param-names (util/vec->set fixed-param-names))))) ([prefix epoch] - (load-checkpoint mod {:prefix prefix :epoch epoch}))) + (load-checkpoint {:prefix prefix :epoch epoch}))) (defn load-optimizer-states [mod fname] (.mod load fname)) @@ -670,4 +669,3 @@ (fit-params {:allow-missing true}) (fit-params {})) - diff --git a/contrib/clojure-package/test/org/apache/clojure_mxnet/module_test.clj b/contrib/clojure-package/test/org/apache/clojure_mxnet/module_test.clj index f3d4e75e8c97..0f71b5a850cc 100644 --- a/contrib/clojure-package/test/org/apache/clojure_mxnet/module_test.clj +++ b/contrib/clojure-package/test/org/apache/clojure_mxnet/module_test.clj @@ -101,13 +101,20 @@ (m/init-optimizer {:optimizer (optimizer/sgd {:learning-rate 0.1 :momentum 0.9})}) (m/update) (m/save-checkpoint {:prefix "test" :epoch 0 :save-opt-states true})) - (let [mod2 (m/load-checkpoint {:prefix "test" :epoch 0 :load-optimizer-states true})] (-> mod2 (m/bind {:data-shapes [{:name "data" :shape [10 10] :layout "NT"}]}) (m/init-optimizer {:optimizer (optimizer/sgd {:learning-rate 0.1 :momentum 0.9})})) - (is (= (-> mod m/symbol sym/to-json) (-> mod2 m/symbol sym/to-json))) - (is (= (-> mod m/params first) (-> mod2 m/params first)))))) + (is (= (-> mod m/symbol sym/to-json) (-> mod2 m/symbol sym/to-json))) + (is (= (-> mod m/params first) (-> mod2 m/params first)))) + ;; arity 2 version of above. `load-optimizer-states` is `false` here by default, + ;; but optimizers states aren't checked here so it's not relevant to the test outcome. + (let [mod3 (m/load-checkpoint "test" 0)] + (-> mod3 + (m/bind {:data-shapes [{:name "data" :shape [10 10] :layout "NT"}]}) + (m/init-optimizer {:optimizer (optimizer/sgd {:learning-rate 0.1 :momentum 0.9})})) + (is (= (-> mod m/symbol sym/to-json) (-> mod3 m/symbol sym/to-json))) + (is (= (-> mod m/params first) (-> mod3 m/params first)))))) (deftest test-module-save-load-multi-device (let [s (sym/variable "data") @@ -321,4 +328,3 @@ (comment (m/data-shapes x)) - From bcfab3a523baf8c4cbcf36514fe0fd2a580cbc30 Mon Sep 17 00:00:00 2001 From: Pedro Larroy <928489+larroy@users.noreply.github.com> Date: Fri, 3 Aug 2018 00:37:12 +0200 Subject: [PATCH 05/11] Add unit test stage for mxnet cpu in debug mode (#11974) --- Jenkinsfile | 27 +++++++++++++++++++++++++++ ci/docker/runtime_functions.sh | 19 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 6d21f496426e..6d9c60732d6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,6 +26,8 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdpart mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so' +// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. +mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests' mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0' mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' // timeout in minutes @@ -233,6 +235,17 @@ try { } } }, + 'CPU: Openblas, debug': { + node('mxnetlinux-cpu') { + ws('workspace/build-cpu-openblas') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_debug', false) + pack_lib('cpu_debug', mx_cmake_lib_debug) + } + } + } + }, 'CPU: Clang 3.9': { node('mxnetlinux-cpu') { ws('workspace/build-cpu-clang39') { @@ -574,6 +587,20 @@ try { } } }, + 'Python3: CPU debug': { + node('mxnetlinux-cpu') { + ws('workspace/ut-python3-cpu-debug') { + try { + init_git() + unpack_lib('cpu_debug', mx_cmake_lib_debug) + python3_ut('ubuntu_cpu') + } finally { + collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_debug_unittest.xml') + collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_debug_quantization.xml') + } + } + } + }, 'Python2: GPU': { node('mxnetlinux-gpu') { ws('workspace/ut-python2-gpu') { diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 52a2650a1cc4..371d2cf63af1 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -345,6 +345,25 @@ build_ubuntu_cpu_openblas() { report_ccache_usage } +build_ubuntu_cpu_cmake_debug() { + set -ex + pushd . + cd /work/build + cmake \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DUSE_CUDA=OFF \ + -DUSE_MKL_IF_AVAILABLE=OFF \ + -DUSE_OPENMP=OFF \ + -DUSE_OPENCV=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -G Ninja \ + /work/mxnet + + ninja -v + popd +} + build_ubuntu_cpu_clang39() { set -ex From c9372776ace1eaeeb12aab69bb2151a30fcde93a Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Thu, 2 Aug 2018 18:23:09 -0700 Subject: [PATCH 06/11] Website broken link fixes (#12014) * fix broken link * fix broken link * switch to .md links * fix broken link --- docs/community/ecosystem.md | 2 +- docs/community/mxnet_channels.md | 2 +- docs/tutorials/scala/index.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/community/ecosystem.md b/docs/community/ecosystem.md index 5ca6d7a0b479..54f8c8993ea9 100644 --- a/docs/community/ecosystem.md +++ b/docs/community/ecosystem.md @@ -57,7 +57,7 @@ Community contributions to MXNet have added many new valuable features and funct ## Model Serving -* [MXNet Model Server (MMS)](https://github.com/apache/incubator-mxnet/tree/master/example/model-server/mms.md) - simple yet scalable solution for model inference. +* [MXNet Model Server (MMS)](https://github.com/awslabs/mxnet-model-server) - simple yet scalable solution for model inference. ## Model Zoos diff --git a/docs/community/mxnet_channels.md b/docs/community/mxnet_channels.md index ef3963f7dabc..18dc1bc55ec8 100644 --- a/docs/community/mxnet_channels.md +++ b/docs/community/mxnet_channels.md @@ -2,7 +2,7 @@ Converse with the MXNet community via the following channels: -- [Forum](https://discuss.mxnet.io/): [discuss.mxnet.io](discuss.mxnet.io) +- [Forum](https://discuss.mxnet.io/): [discuss.mxnet.io](https://discuss.mxnet.io/) - [MXNet Apache developer mailing list](https://lists.apache.org/list.html?dev@mxnet.apache.org) (dev@mxnet.apache.org): To subscribe, send an email to dev-subscribe@mxnet.apache.org - [MXNet Apache user mailing list](https://lists.apache.org/list.html?user@mxnet.apache.org) (user@mxnet.apache.org): To subscribe, send an email to user-subscribe@mxnet.apache.org - [MXNet Slack channel](https://apache-mxnet.slack.com): To request an invitation to the channel please subscribe to the mailing list above and then email: dev@mxnet.apache.org diff --git a/docs/tutorials/scala/index.md b/docs/tutorials/scala/index.md index cd9b2e219fcc..f14337f90f08 100644 --- a/docs/tutorials/scala/index.md +++ b/docs/tutorials/scala/index.md @@ -6,8 +6,8 @@ Using MXNet-Scala is easiest with Maven. You have a couple of options for settin **Note:** Windows is not yet supported. -* [MXNet-Scala Setup Guide Using Maven](../install/scala_setup.html) -* [Setup Scala with MXNet and Create a MXNet-Scala Project with IntelliJ](mxnet_scala_on_intellij.html) +* [MXNet-Scala Setup Guide Using Maven](../../install/scala_setup.md) +* [Setup Scala with MXNet and Create a MXNet-Scala Project with IntelliJ](mxnet_scala_on_intellij.md) ## Tutorials From 1818280d33201ee369a41ab1be324e22b162c46f Mon Sep 17 00:00:00 2001 From: Ankit Khedia <36249596+ankkhedia@users.noreply.github.com> Date: Thu, 2 Aug 2018 20:47:50 -0700 Subject: [PATCH 07/11] removed seed from flaky test (#11975) --- tests/python/unittest/test_operator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 418951fcbb0a..90e85d123d59 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -4963,8 +4963,9 @@ def _make_lower_triangle_symm(a, ndims, m, dtype=np.float32): lt_mask = mx.sym.reshape(lt_mask, shape=shp) return mx.sym.broadcast_mul(a, lt_mask) -# Seed set because the test is not robust enough to operate on random data -@with_seed(42) +# @ankkhedia: Getting rid of fixed seed as flakiness could not be reproduced +# tracked at https://github.com/apache/incubator-mxnet/issues/11718 +@with_seed() def test_laop(): dtype = np.float64 rtol_fw = 1e-7 From 619700a5ba0f482f4e48d985f0beb61e3e861655 Mon Sep 17 00:00:00 2001 From: Marco de Abreu Date: Fri, 3 Aug 2018 06:24:53 +0200 Subject: [PATCH 08/11] Disable ccache log print due to threadunsafety (#11997) --- ci/docker/runtime_functions.sh | 53 ++-------------------------------- 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 371d2cf63af1..21471902c356 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -110,23 +110,6 @@ build_jetson() { popd } -report_ccache_usage() { - set -ex - pushd . - - # Show global ccache summary at the end of each run. - ccache -s - if [ -e $CCACHE_LOGFILE ] - then - # Display local ccache log, excluding some overly verbose output. - cat $CCACHE_LOGFILE | grep -v "Config:" | grep -v "stats.lock" - else - echo "No ccache log found." - fi - - popd -} - # # ARM builds # @@ -159,7 +142,6 @@ build_armv6() { -G Ninja /work/mxnet ninja -v - report_ccache_usage build_wheel popd } @@ -191,7 +173,6 @@ build_armv7() { -G Ninja /work/mxnet ninja -v - report_ccache_usage build_wheel popd } @@ -210,7 +191,6 @@ build_armv8() { -DUSE_MKL_IF_AVAILABLE=OFF\ -G Ninja /work/mxnet ninja -v - report_ccache_usage build_wheel } @@ -237,7 +217,6 @@ build_android_armv7() { -DUSE_MKL_IF_AVAILABLE=OFF\ -G Ninja /work/mxnet ninja -v - report_ccache_usage } build_android_armv8() { @@ -270,8 +249,6 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) - - report_ccache_usage } build_amzn_linux_cpu() { @@ -289,7 +266,6 @@ build_amzn_linux_cpu() { -DUSE_DIST_KVSTORE=ON\ -G Ninja /work/mxnet ninja -v - report_ccache_usage } @@ -306,8 +282,6 @@ build_centos7_mkldnn() { USE_MKLDNN=1 \ USE_BLAS=openblas \ -j$(nproc) - - report_ccache_usage } build_centos7_gpu() { @@ -341,8 +315,6 @@ build_ubuntu_cpu_openblas() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_cpu_cmake_debug() { @@ -366,20 +338,15 @@ build_ubuntu_cpu_cmake_debug() { build_ubuntu_cpu_clang39() { set -ex - - export CXX=clang++-3.9 + export CXX=clang++-3.9 export CC=clang-3.9 - - build_ccache_wrappers - - make \ + build_ccache_wrappers + make \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_OPENMP=0 \ USE_DIST_KVSTORE=1 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_cpu_clang50() { @@ -396,8 +363,6 @@ build_ubuntu_cpu_clang50() { USE_OPENMP=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_cpu_clang39_mkldnn() { @@ -414,8 +379,6 @@ build_ubuntu_cpu_clang39_mkldnn() { USE_MKLDNN=1 \ USE_OPENMP=0 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_cpu_clang50_mkldnn() { @@ -432,8 +395,6 @@ build_ubuntu_cpu_clang50_mkldnn() { USE_MKLDNN=1 \ USE_OPENMP=1 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_cpu_mkldnn() { @@ -447,8 +408,6 @@ build_ubuntu_cpu_mkldnn() { USE_BLAS=openblas \ USE_MKLDNN=1 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_gpu() { @@ -469,8 +428,6 @@ build_ubuntu_gpu_mkldnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_gpu_mkldnn_nocudnn() { @@ -486,8 +443,6 @@ build_ubuntu_gpu_mkldnn_nocudnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=0 \ -j$(nproc) - - report_ccache_usage } build_ubuntu_gpu_cuda91_cudnn7() { @@ -534,7 +489,6 @@ build_ubuntu_gpu_cmake_mkldnn() { /work/mxnet ninja -v - report_ccache_usage # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0. cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0 @@ -556,7 +510,6 @@ build_ubuntu_gpu_cmake() { /work/mxnet ninja -v - report_ccache_usage } From 25341648365598a9a123f033bf92ce7fb51c0a39 Mon Sep 17 00:00:00 2001 From: Piyush Ghai Date: Fri, 3 Aug 2018 03:08:05 -0700 Subject: [PATCH 09/11] Added default tolerance levels for regression checks for MBCC (#12006) * Added tolerance level for assert_almost_equal for MBCC * Nudge to CI --- .../nightly/model_backwards_compatibility_check/common.py | 2 ++ .../model_backwards_compat_inference.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/nightly/model_backwards_compatibility_check/common.py b/tests/nightly/model_backwards_compatibility_check/common.py index 4c61cc4e3267..8950a9270839 100644 --- a/tests/nightly/model_backwards_compatibility_check/common.py +++ b/tests/nightly/model_backwards_compatibility_check/common.py @@ -41,6 +41,8 @@ backslash = '/' s3 = boto3.resource('s3') ctx = mx.cpu(0) +atol_default = 1e-5 +rtol_default = 1e-5 def get_model_path(model_name): diff --git a/tests/nightly/model_backwards_compatibility_check/model_backwards_compat_inference.py b/tests/nightly/model_backwards_compatibility_check/model_backwards_compat_inference.py index ae368e3a0fc6..5d63e7e9bca3 100644 --- a/tests/nightly/model_backwards_compatibility_check/model_backwards_compat_inference.py +++ b/tests/nightly/model_backwards_compatibility_check/model_backwards_compat_inference.py @@ -44,7 +44,7 @@ def test_module_checkpoint_api(): old_inference_results = load_inference_results(model_name) inference_results = loaded_model.predict(data_iter) # Check whether they are equal or not ? - assert_almost_equal(inference_results.asnumpy(), old_inference_results.asnumpy()) + assert_almost_equal(inference_results.asnumpy(), old_inference_results.asnumpy(), rtol=rtol_default, atol=atol_default) clean_model_files(model_files, model_name) logging.info('=================================') @@ -69,7 +69,7 @@ def test_lenet_gluon_load_params_api(): loaded_model.load_params(model_name + '-params') output = loaded_model(test_data) old_inference_results = mx.nd.load(model_name + '-inference')['inference'] - assert_almost_equal(old_inference_results.asnumpy(), output.asnumpy()) + assert_almost_equal(old_inference_results.asnumpy(), output.asnumpy(), rtol=rtol_default, atol=atol_default) clean_model_files(model_files, model_name) logging.info('=================================') logging.info('Assertion passed for model : %s' % model_name) @@ -92,7 +92,7 @@ def test_lenet_gluon_hybrid_imports_api(): loaded_model = gluon.SymbolBlock.imports(model_name + '-symbol.json', ['data'], model_name + '-0000.params') output = loaded_model(test_data) old_inference_results = mx.nd.load(model_name + '-inference')['inference'] - assert_almost_equal(old_inference_results.asnumpy(), output.asnumpy()) + assert_almost_equal(old_inference_results.asnumpy(), output.asnumpy(), rtol=rtol_default, atol=atol_default) clean_model_files(model_files, model_name) logging.info('=================================') logging.info('Assertion passed for model : %s' % model_name) @@ -124,7 +124,7 @@ def test_lstm_gluon_load_parameters_api(): loaded_model.load_parameters(model_name + '-params') output = loaded_model(test_data) old_inference_results = mx.nd.load(model_name + '-inference')['inference'] - assert_almost_equal(old_inference_results.asnumpy(), output.asnumpy()) + assert_almost_equal(old_inference_results.asnumpy(), output.asnumpy(), rtol=rtol_default, atol=atol_default) clean_model_files(model_files, model_name) logging.info('=================================') logging.info('Assertion passed for model : %s' % model_name) From 32c2e159ae63458b9aa0231761c8fec38be42df4 Mon Sep 17 00:00:00 2001 From: Kellen Sunderland Date: Fri, 3 Aug 2018 13:02:15 +0200 Subject: [PATCH 10/11] Disable flaky mkldnn test_requantize_int32_to_int8 (#11748) --- tests/python/quantization/test_quantization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py index 359bbee569f8..08303c816af1 100644 --- a/tests/python/quantization/test_quantization.py +++ b/tests/python/quantization/test_quantization.py @@ -77,6 +77,7 @@ def test_dequantize_int8_to_float32(): @with_seed() +@unittest.skip('Flaky test, tracked in: https://github.com/apache/incubator-mxnet/issues/11747') def test_requantize_int32_to_int8(): def quantized_int32_to_float(qdata, min_range, max_range): assert qdata.dtype == 'int32' From 1fa04f2c9a7ba0c3273d080afa7fc993b927f114 Mon Sep 17 00:00:00 2001 From: Pedro Larroy <928489+larroy@users.noreply.github.com> Date: Fri, 3 Aug 2018 14:05:41 +0200 Subject: [PATCH 11/11] [MXNET-769] Usability improvements to windows builds (#11947) * Windows scripted build Adjust Jenkins builds to use ci/build_windows.py Issues: #8714 #11100 #10166 #10049 * Fix bug * Fix non-portable ut * add xunit --- CMakeLists.txt | 2 +- Jenkinsfile | 152 +++---------------- ci/build.py | 16 +- ci/build_windows.py | 253 +++++++++++++++++++++++++++++++ ci/util.py | 43 ++++++ ci/windows/test_py2_cpu.ps1 | 25 +++ ci/windows/test_py2_gpu.ps1 | 29 ++++ ci/windows/test_py3_cpu.ps1 | 25 +++ ci/windows/test_py3_gpu.ps1 | 29 ++++ tests/python/gpu/test_forward.py | 6 +- tests/requirements.txt | 3 + tools/license_header.py | 2 +- 12 files changed, 438 insertions(+), 147 deletions(-) create mode 100755 ci/build_windows.py create mode 100644 ci/util.py create mode 100644 ci/windows/test_py2_cpu.ps1 create mode 100644 ci/windows/test_py2_gpu.ps1 create mode 100644 ci/windows/test_py3_cpu.ps1 create mode 100644 ci/windows/test_py3_gpu.ps1 create mode 100644 tests/requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 483108a68419..000bbbf17ea5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -372,13 +372,13 @@ endif() # ---[ LAPack if(USE_LAPACK) + message("USE_LAPACK is ON") add_definitions(-DMXNET_USE_LAPACK=1) if (NOT MSVC) list(APPEND mxnet_LINKER_LIBS lapack) endif() endif() -message("USE LAPACK ${USE_LAPACK}") # ---[ jemalloc if(USE_JEMALLOC) diff --git a/Jenkinsfile b/Jenkinsfile index 6d9c60732d6b..9d7792066e37 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -391,28 +391,8 @@ try { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { init_git_win() - bat """mkdir build_vc14_cpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_cpu - cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_cpu.bat' - - bat '''rmdir /s/q pkg_vc14_cpu - mkdir pkg_vc14_cpu\\lib - mkdir pkg_vc14_cpu\\python - mkdir pkg_vc14_cpu\\include - mkdir pkg_vc14_cpu\\build - copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib - copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build - xcopy python pkg_vc14_cpu\\python /E /I /Y - xcopy include pkg_vc14_cpu\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ - ''' - stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' + powershell 'python ci/build_windows.py -f WIN_CPU' + stash includes: 'windows_package.7z', name: 'windows_package_cpu' } } } @@ -424,28 +404,9 @@ try { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() - bat """mkdir build_vc14_gpu - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_vc14_gpu - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_gpu.bat' - bat '''rmdir /s/q pkg_vc14_gpu - mkdir pkg_vc14_gpu\\lib - mkdir pkg_vc14_gpu\\python - mkdir pkg_vc14_gpu\\include - mkdir pkg_vc14_gpu\\build - copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib - copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build - xcopy python pkg_vc14_gpu\\python /E /I /Y - xcopy include pkg_vc14_gpu\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ - ''' - stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' + init_git_win() + powershell 'python ci/build_windows.py -f WIN_GPU' + stash includes: 'windows_package.7z', name: 'windows_package_gpu' } } } @@ -456,37 +417,9 @@ try { timeout(time: max_time, unit: 'MINUTES') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) { - init_git_win() - bat """mkdir build_%BUILD_NAME% - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_%BUILD_NAME% - copy ${env.WORKSPACE}\\3rdparty\\mkldnn\\config_template.vcxproj.user ${env.WORKSPACE}\\config_template.vcxproj.user /y - cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" - bat ''' - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" - cd build_%BUILD_NAME% - set /a cores=%NUMBER_OF_PROCESSORS% * 2 - jom -j %cores% - ''' - bat '''rmdir /s/q pkg_%BUILD_NAME% - mkdir pkg_%BUILD_NAME%\\lib - mkdir pkg_%BUILD_NAME%\\python - mkdir pkg_%BUILD_NAME%\\include - mkdir pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\libmxnet.lib pkg_%BUILD_NAME%\\lib - copy build_%BUILD_NAME%\\libmxnet.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\3rdparty\\mkldnn\\src\\mkldnn.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\libiomp5md.dll pkg_%BUILD_NAME%\\build - copy build_%BUILD_NAME%\\mklml.dll pkg_%BUILD_NAME%\\build - xcopy python pkg_%BUILD_NAME%\\python /E /I /Y - xcopy include pkg_%BUILD_NAME%\\include /E /I /Y - xcopy 3rdparty\\dmlc-core\\include pkg_%BUILD_NAME%\\include /E /I /Y - xcopy 3rdparty\\mshadow\\mshadow pkg_%BUILD_NAME%\\include\\mshadow /E /I /Y - xcopy 3rdparty\\nnvm\\include pkg_%BUILD_NAME%\\nnvm\\include /E /I /Y - del /Q *.7z - 7z.exe a %BUILD_NAME%.7z pkg_%BUILD_NAME%\\ - ''' - stash includes: 'vc14_gpu_mkldnn.7z', name: 'vc14_gpu_mkldnn' + init_git_win() + powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN' + stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn' } } } @@ -870,16 +803,8 @@ try { ws('workspace/ut-python-cpu') { try { init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - pip install mock - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + unstash 'windows_package_cpu' + powershell 'ci/windows/test_py2_cpu.ps1' } finally { collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml') } @@ -893,15 +818,8 @@ try { ws('workspace/ut-python-cpu') { try { init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu - 7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc - C:\\mxnet\\test_cpu.bat""" + unstash 'windows_package_cpu' + powershell 'ci/windows/test_py3_cpu.ps1' } finally { collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml') } @@ -915,19 +833,11 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 - pip install mock - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu' + powershell 'ci/windows/test_py2_gpu.ps1' } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') + collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml') + collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml') } } } @@ -939,18 +849,11 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu - 7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu' + powershell 'ci/windows/test_py3_gpu.ps1' } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') + collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml') + collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml') } } } @@ -962,18 +865,11 @@ try { ws('workspace/ut-python-gpu') { try { init_git_win() - unstash 'vc14_gpu_mkldnn' - bat '''rmdir /s/q pkg_vc14_gpu_mkldnn - 7z x -y vc14_gpu_mkldnn.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y - xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 - set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python - del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc - C:\\mxnet\\test_gpu.bat""" + unstash 'windows_package_gpu_mkldnn' + powershell 'ci/windows/test_py3_gpu.ps1' } finally { - collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') - collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') + collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml') + collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml') } } } diff --git a/ci/build.py b/ci/build.py index 09f2d4709bdd..a9d6a63537f2 100755 --- a/ci/build.py +++ b/ci/build.py @@ -39,6 +39,7 @@ from itertools import chain from subprocess import call, check_call from typing import * +from util import * CCACHE_MAXSIZE = '500G' @@ -138,24 +139,9 @@ def _get_local_image_id(docker_binary, docker_tag): return image_id -def get_mxnet_root() -> str: - curpath = os.path.abspath(os.path.dirname(__file__)) - - def is_mxnet_root(path: str) -> bool: - return os.path.exists(os.path.join(path, ".mxnet_root")) - - while not is_mxnet_root(curpath): - parent = os.path.abspath(os.path.join(curpath, os.pardir)) - if parent == curpath: - raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root") - curpath = parent - return curpath - - def buildir() -> str: return os.path.join(get_mxnet_root(), "build") - def default_ccache_dir() -> str: # Share ccache across containers if 'CCACHE_DIR' in os.environ: diff --git a/ci/build_windows.py b/ci/build_windows.py new file mode 100755 index 000000000000..5eca58db7b74 --- /dev/null +++ b/ci/build_windows.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""User friendly / multi platform builder script""" + +import subprocess +import logging +import os +import tempfile +import sys +from distutils import spawn +import logging +from subprocess import check_call +import platform +import argparse +from util import * +import json +from enum import Enum +import time +import datetime +import shutil +import glob +from distutils.dir_util import copy_tree + +KNOWN_VCVARS = [ + # VS 2015 + r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat' + # VS 2017 + , r'c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat' +] + +class BuildFlavour(Enum): + WIN_CPU = 'WIN_CPU' + WIN_CPU_MKLDNN = 'WIN_CPU_MKLDNN' + WIN_GPU = 'WIN_GPU' + WIN_GPU_MKLDNN = 'WIN_GPU_MKLDNN' + +CMAKE_FLAGS = { + 'WIN_CPU': '-DUSE_CUDA=0 \ + -DUSE_CUDNN=0 \ + -DUSE_NVRTC=0 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DBUILD_CPP_EXAMPLES=1 \ + -DUSE_MKL_IF_AVAILABLE=0' + + ,'WIN_CPU_MKLDNN': '-DUSE_CUDA=0 \ + -DUSE_CUDNN=0 \ + -DUSE_NVRTC=0 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DUSE_MKL_IF_AVAILABLE=1' + + ,'WIN_GPU': '-DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_NVRTC=1 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=52 \ + -DCUDA_ARCH_PTX=52 \ + -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" \ + -DUSE_MKL_IF_AVAILABLE=0 \ + -DCMAKE_BUILD_TYPE=Release' + + ,'WIN_GPU_MKLDNN': '-DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_NVRTC=1 \ + -DUSE_OPENCV=1 \ + -DUSE_OPENMP=1 \ + -DUSE_PROFILER=1 \ + -DUSE_BLAS=open \ + -DUSE_LAPACK=1 \ + -DUSE_DIST_KVSTORE=0 \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=52 \ + -DCUDA_ARCH_PTX=52 \ + -DUSE_MKLDNN=1 \ + -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 \ + /DNDEBUG" \ + -DCMAKE_BUILD_TYPE=Release' + +} + + +def get_vcvars_environment(architecture, vcvars): + """ + Returns a dictionary containing the environment variables set up by vcvars + """ + result = None + python = sys.executable + + vcvars_list = [vcvars] + vcvars_list.extend(KNOWN_VCVARS) + for vcvars in vcvars_list: + if os.path.isfile(vcvars): + process = subprocess.Popen('("%s" %s>nul) && "%s" -c "import os; import json; print(json.dumps(dict(os.environ)))"' % (vcvars, architecture, python), stdout=subprocess.PIPE, shell=True) + stdout, stderr = process.communicate() + exitcode = process.wait() + if exitcode == 0: + logging.info("Using build environment from: %s", vcvars) + return(json.loads(stdout.strip())) + else: + raise RuntimeError('Failed cloning environment from vcvars file: %s stdout: %s stderr: %s', vcvars, stdout, stderr) + raise RuntimeError('Couldn\'t find vcvars batch file: %s', vcvars) + + +def windows_build(args): + vcvars_env = get_vcvars_environment(args.arch, args.vcvars) + logging.debug("vcvars environment: %s", vcvars_env) + os.environ.update(vcvars_env) + + path = args.output + os.makedirs(path, exist_ok=True) + mxnet_root = get_mxnet_root() + logging.info("Found mxnet root: {}".format(mxnet_root)) + with remember_cwd(): + os.chdir(path) + logging.info("Generating project with CMake") + check_call("cmake -G \"Visual Studio 14 2015 Win64\" {} {}".format(CMAKE_FLAGS[args.flavour], mxnet_root), shell=True) + logging.info("Building with visual studio") + t0 = int(time.time()) + check_call(["msbuild", "mxnet.sln","/p:configuration=release;platform=x64", "/maxcpucount","/v:minimal"]) + logging.info("Build flavour: %s complete in directory: \"%s\"", args.flavour, os.path.abspath(path)) + logging.info("Build took %s" , datetime.timedelta(seconds=int(time.time()-t0))) + windows_package(args) + +def windows_package(args): + pkgfile = 'windows_package.7z' + pkgdir = os.path.abspath('windows_package') + logging.info("Packaging libraries and headers in package: %s", pkgfile) + j = os.path.join + pkgdir_lib = os.path.abspath(j(pkgdir, 'lib')) + with remember_cwd(): + os.chdir(args.output) + logging.info("Looking for static libraries and dlls in: \"%s", os.getcwd()) + libs = list(glob.iglob('**/*.lib', recursive=True)) + dlls = list(glob.iglob('**/*.dll', recursive=True)) + os.makedirs(pkgdir_lib, exist_ok=True) + for lib in libs: + logging.info("packing lib: %s", lib) + shutil.copy(lib, pkgdir_lib) + for dll in dlls: + logging.info("packing dll: %s", dll) + shutil.copy(dll, pkgdir_lib) + os.chdir(get_mxnet_root()) + logging.info('packing python bindings') + copy_tree('python', j(pkgdir, 'python')) + logging.info('packing headers') + copy_tree('include', j(pkgdir, 'include')) + copy_tree(j('3rdparty','dmlc-core','include'), j(pkgdir, 'include')) + copy_tree(j('3rdparty','mshadow', 'mshadow'), j(pkgdir, 'include', 'mshadow')) + copy_tree(j('3rdparty','tvm','nnvm', 'include'), j(pkgdir,'include', 'nnvm', 'include')) + logging.info("Compressing package: %s", pkgfile) + check_call(['7z', 'a', pkgfile, pkgdir]) + + +def nix_build(args): + path = args.output + os.makedirs(path, exist_ok=True) + with remember_cwd(): + os.chdir(path) + logging.info("Generating project with CMake") + check_call("cmake \ + -DUSE_CUDA=OFF \ + -DUSE_OPENCV=OFF \ + -DUSE_OPENMP=OFF \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja ..", shell=True) + check_call("ninja", shell=True) + +def main(): + logging.getLogger().setLevel(logging.INFO) + logging.basicConfig(format='%(asctime)-15s %(message)s') + logging.info("MXNet Windows build helper") + + parser = argparse.ArgumentParser() + parser.add_argument("-o", "--output", + help="output directory", + default='build', + type=str) + + parser.add_argument("--vcvars", + help="vcvars batch file location, typically inside vs studio install dir", + default=r'c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat', + type=str) + + parser.add_argument("--arch", + help="architecture", + default='x64', + type=str) + + parser.add_argument("-f", "--flavour", + help="build flavour", + default='WIN_CPU', + choices=[x.name for x in BuildFlavour], + type=str) + + args = parser.parse_args() + logging.info("Build flavour: %s", args.flavour) + + system = platform.system() + if system == 'Windows': + logging.info("Detected Windows platform") + if 'OpenBLAS_HOME' not in os.environ: + os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas" + if 'OpenCV_DIR' not in os.environ: + os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14" + if 'CUDA_PATH' not in os.environ: + os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0" + windows_build(args) + + elif system == 'Linux' or system == 'Darwin': + nix_build(args) + + else: + logging.error("Don't know how to build for {} yet".format(platform.system())) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/ci/util.py b/ci/util.py new file mode 100644 index 000000000000..22631f30435f --- /dev/null +++ b/ci/util.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import contextlib + +def get_mxnet_root() -> str: + curpath = os.path.abspath(os.path.dirname(__file__)) + + def is_mxnet_root(path: str) -> bool: + return os.path.exists(os.path.join(path, ".mxnet_root")) + + while not is_mxnet_root(curpath): + parent = os.path.abspath(os.path.join(curpath, os.pardir)) + if parent == curpath: + raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root") + curpath = parent + return curpath + +@contextlib.contextmanager +def remember_cwd(): + ''' + Restore current directory when exiting context + ''' + curdir = os.getcwd() + try: yield + finally: os.chdir(curdir) + + diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1 new file mode 100644 index 000000000000..1623d2956103 --- /dev/null +++ b/ci/windows/test_py2_cpu.ps1 @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt +c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train +if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1 new file mode 100644 index 000000000000..13cd5366e0db --- /dev/null +++ b/ci/windows/test_py2_gpu.ps1 @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt +c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\train +if (! $?) { Throw ("Error running tests") } diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1 new file mode 100644 index 000000000000..98d4e410e8f5 --- /dev/null +++ b/ci/windows/test_py3_cpu.ps1 @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt +c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train +if (! $?) { Throw ("Error running train tests") } diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1 new file mode 100644 index 000000000000..b94b4f389be8 --- /dev/null +++ b/ci/windows/test_py3_gpu.ps1 @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z +$env:PYTHONPATH=join-path $pwd.Path windows_package\python +$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 +c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt +c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest +if (! $?) { Throw ("Error running unittest") } +c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py +if (! $?) { Throw ("Error running tests") } +c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train +if (! $?) { Throw ("Error running tests") } diff --git a/tests/python/gpu/test_forward.py b/tests/python/gpu/test_forward.py index 126ccabaa7b5..02b0256024d3 100644 --- a/tests/python/gpu/test_forward.py +++ b/tests/python/gpu/test_forward.py @@ -24,11 +24,13 @@ sys.path.insert(0, os.path.join(curr_path, '../unittest')) from common import setup_module, with_seed, teardown from mxnet.gluon import utils +import tarfile def _get_model(): if not os.path.exists('model/Inception-7-symbol.json'): - download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz', dirname='model') - os.system("cd model; tar -xf inception-v3.tar.gz --strip-components 1") + download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz') + with tarfile.open(name="inception-v3.tar.gz", mode="r:gz") as tf: + tf.extractall() def _dump_images(shape): import skimage.io diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000000..0eca73fbb02a --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +# Requirements for tests, those are installed before running on the virtualenv +mock +nose diff --git a/tools/license_header.py b/tools/license_header.py index 0ee4049338b1..7aef33b71213 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -82,7 +82,7 @@ _LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#', '.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#', '.java':'*', '.sh':'#', '.cpp':'*', '.hpp':'*', '.c':'*', - '.bat':'rem', '.pl':'#', '.m':'%', '.R':'#', '.mk':'#', '.cfg':'#', '.t':'#'} + '.bat':'rem', '.pl':'#', '.m':'%', '.R':'#', '.mk':'#', '.cfg':'#', '.t':'#', '.ps1': '#'} # Previous license header, which will be removed _OLD_LICENSE = re.compile('.*Copyright.*by Contributors')