diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 10e2f21138..f38dd6ccb6 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -13,6 +13,11 @@ jobs:
   - script: |
       source install.sh
     displayName: 'Install nni toolkit via source code'
+  - script: |
+      python3 -m pip install flake8 --user
+      IGNORE=./tools/nni_annotation/testcase/*:F821,./examples/trials/mnist-nas/mnist.py:F821
+      python3 -m flake8 . --count --per-file-ignores=$IGNORE --select=E9,F63,F72,F82 --show-source --statistics
+    displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
       source unittest.sh