diff --git a/tests/test_config.py b/tests/test_config.py
index bf71f48..e69ea70 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -6,7 +6,7 @@
 import os
 import stat
 import unittest
-import tempfile
+import shutil
 try:
     import mock
 except ImportError:
@@ -14,12 +14,24 @@
 from six.moves import configparser
 
 from knack.config import CLIConfig, get_config_parser
+from .util import TEMP_FOLDER_NAME, new_temp_folder
+
+
+def clean_local_temp_folder():
+    local_temp_folders = os.path.join(os.getcwd(), TEMP_FOLDER_NAME)
+    if os.path.exists(local_temp_folders):
+        shutil.rmtree(local_temp_folders)
 
 
 class TestCLIConfig(unittest.TestCase):
 
     def setUp(self):
-        self.cli_config = CLIConfig(config_dir=tempfile.mkdtemp())
+        self.cli_config = CLIConfig(config_dir=new_temp_folder())
+        # In case the previous test is stopped and doesn't clean up
+        clean_local_temp_folder()
+
+    def tearDown(self):
+        clean_local_temp_folder()
 
     def test_has_option(self):
         section = 'MySection'
diff --git a/tests/util.py b/tests/util.py
index 0c03c34..57f8c7c 100644
--- a/tests/util.py
+++ b/tests/util.py
@@ -9,10 +9,15 @@
     from unittest import mock
 import sys
 import tempfile
+import shutil
+import os
 from six import StringIO
 
 from knack.cli import CLI, CLICommandsLoader, CommandInvoker
 
+TEMP_FOLDER_NAME = "knack_temp"
+
+
 def redirect_io(func):
 
     original_stderr = sys.stderr
@@ -30,7 +35,7 @@ def wrapper(self):
 class MockContext(CLI):
 
     def __init__(self):
-        super(MockContext, self).__init__(config_dir=tempfile.mkdtemp())
+        super(MockContext, self).__init__(config_dir=new_temp_folder())
         loader = CLICommandsLoader(cli_ctx=self)
         invocation = mock.MagicMock(spec=CommandInvoker)
         invocation.data = {}
@@ -44,5 +49,13 @@ def get_cli_version(self):
         return '0.1.0'
 
     def __init__(self, **kwargs):
-        kwargs['config_dir'] = tempfile.mkdtemp()
+        kwargs['config_dir'] = new_temp_folder()
         super(DummyCLI, self).__init__(**kwargs)
+
+
+def new_temp_folder():
+    temp_dir = os.path.join(tempfile.gettempdir(), TEMP_FOLDER_NAME)
+    if os.path.exists(temp_dir):
+        shutil.rmtree(temp_dir)
+    os.mkdir(temp_dir)
+    return temp_dir