Merge pull request #51313 from s0undt3ch/features/tox-runtests-2018.3

[2018.3] Allow running runtests.py using tox

.coveragerc

@@ -0,0 +1,34 @@
+[run]
+branch = True
+cover_pylib = False
+source =
+  salt
+parallel = True
+concurrency = multiprocessing
+omit =
+  tests/*.py
+  setup.py
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+
+
+ignore_errors = True
+
+[paths]
+source =
+   salt

Gemfile

@@ -2,9 +2,8 @@
 
 source 'https://rubygems.org'
 
-# Point this back at the test-kitchen package after 1.23.3 is relased
-gem 'test-kitchen', :git => 'https://github.com/dwoz/test-kitchen.git', :branch => 'winrm_opts'
-gem 'kitchen-salt', '~>0.2'
+gem 'test-kitchen', '~>1.23.3'
+gem 'kitchen-salt', '~>0.4.1'
 gem 'kitchen-sync'
 gem 'git'
 
@@ -14,7 +13,7 @@ end
 
 group :windows do
   gem 'winrm', '~>2.0'
-  gem 'winrm-fs', '~>1.3.1' 
+  gem 'winrm-fs', '~>1.3.1'
 end
 
 group :ec2 do

requirements/pytest.txt

@@ -5,3 +5,4 @@ pytest-salt == 2018.12.8
 pytest-timeout >= 1.3.3
 pytest-tempdir >= 2018.8.11
 pytest-helpers-namespace >= 2017.11.11
+pytest-salt-from-filenames >= 2019.1.22

requirements/tests.txt

@@ -29,7 +29,6 @@ PyMySQL; sys.platform != 'win32' and sys.platform != 'darwin'
 jsonschema
 strict_rfc3339
 rfc3987
-jinja2
 pyOpenSSL
 ioflo
 dnspython

tests/conftest.py

@@ -55,7 +55,7 @@
 from salt.utils.odict import OrderedDict
 
 # Define the pytest plugins we rely on
-pytest_plugins = ['tempdir', 'helpers_namespace']  # pylint: disable=invalid-name
+pytest_plugins = ['tempdir', 'helpers_namespace', 'salt-from-filenames']  # pylint: disable=invalid-name
 
 # Define where not to collect tests from
 collect_ignore = ['setup.py']

tests/integration/modules/test_supervisord.py

@@ -222,7 +222,7 @@ def test_status_all(self):
         ret = self.run_function(
             'supervisord.status', [], conf_file=self.supervisor_conf,
             bin_env=self.venv_dir)
-        self.assertEqual(list(ret.keys()), ['sleep_service', 'sleep_service2'])
+        self.assertEqual(sorted(ret), ['sleep_service', 'sleep_service2'])
 
     def test_status_one(self):
         '''

tests/support/coverage/sitecustomize.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+'''
+Python will always try to import sitecustomize.
+We use that fact to try and support code coverage for sub-processes
+'''
+from __future__ import absolute_import
+try:
+    import coverage
+    coverage.process_startup()
+except ImportError:
+    pass

tests/tox-helper.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# This script exists so that path handling when running tox works for both Linux and Windows
+
+# Import Python Libs
+from __future__ import absolute_import, unicode_literals
+import os
+import shutil
+import argparse
+import tempfile
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--rootdir',
+        default=os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+    )
+    subparsers = parser.add_subparsers(help='sub-command help', dest='subparser')
+
+    subparsers.add_parser('create-dirs')
+    subparsers.add_parser('move-artifacts')
+
+    options = parser.parse_args()
+    if options.subparser == 'create-dirs':
+        for dirname in ('logs', 'coverage', 'xml-unittests-output'):
+            path = os.path.join(options.rootdir, 'artifacts', dirname)
+            if not os.path.exists(path):
+                os.makedirs(path)
+
+    if options.subparser == 'move-artifacts':
+        tmp_artifacts_dir = os.path.join(tempfile.gettempdir(), 'artifacts')
+        if not os.path.exists(tmp_artifacts_dir):
+            os.makedirs(tmp_artifacts_dir)
+
+        for dirname in ('logs', 'coverage', 'xml-unittests-output'):
+            src = os.path.join(options.rootdir, 'artifacts', dirname)
+            dst = os.path.join(tmp_artifacts_dir, dirname)
+            shutil.copytree(src, dst)
+
+
+if __name__ == '__main__':
+    main()

tests/unit/modules/inspectlib/test_fsdb.py

@@ -135,26 +135,12 @@ def test_create_table(self):
                 csvdb.open()
                 csvdb.create_table_from_object(FoobarEntity())
 
+            sorted_writable_data = sorted(writable.data[0].strip().split(','))
             if six.PY2:
-                assert writable.data[0].strip() == "foo:int,bar:unicode,spam:float"
+                sorted_expected_data = sorted("foo:int,bar:unicode,spam:float".split(','))
             else:
-                # Order in PY3 is not the same for every run
-                writable_data = writable.data[0].strip()
-                assert_order_options = ['bar:str,foo:int,spam:float',
-                                        'bar:str,spam:float,foo:int',
-                                        'foo:int,spam:float,bar:str',
-                                        'foo:int,bar:str,spam:float',
-                                        'spam:float,foo:int,bar:str',
-                                        'spam:float,bar:str,foo:int']
-                while assert_order_options:
-                    assert_option = assert_order_options.pop()
-                    try:
-                        assert writable_data == assert_option
-                        break
-                    except AssertionError:
-                        if not assert_order_options:
-                            raise
-                        continue
+                sorted_expected_data = sorted("foo:int,bar:str,spam:float".split(','))
+            self.assertEqual(sorted_writable_data, sorted_expected_data)
 
     def test_list_databases(self):
         '''

tests/unit/modules/test_cassandra.py

@@ -118,9 +118,12 @@ def get_keyspace_column_families(self, keyspace):
                 self.assertCountEqual(cassandra.column_families(),
                                       {'A': ['a', 'b'], 'B': ['c', 'd']})
             else:
-                self.assertEqual(cassandra.column_families('A'),
+                self.assertEqual(sorted(cassandra.column_families('A')),
                                  ['a', 'b'])
-                self.assertEqual(cassandra.column_families(),
+                column_families = cassandra.column_families()
+                for key in ('A', 'B'):
+                    column_families[key] = sorted(column_families[key])
+                self.assertEqual(column_families,
                                  {'A': ['a', 'b'], 'B': ['c', 'd']})
 
     def test_column_family_definition(self):

tests/unit/modules/test_pillar.py

@@ -54,10 +54,11 @@ def test_obfuscate(self):
     @skipIf(NO_MOCK, NO_MOCK_REASON)
     def test_ls(self):
         with patch('salt.modules.pillar.items', MagicMock(return_value=pillar_value_1)):
+            ls = sorted(pillarmod.ls())
             if six.PY3:
-                self.assertCountEqual(pillarmod.ls(), ['a', 'b'])
+                self.assertCountEqual(ls, ['a', 'b'])
             else:
-                self.assertEqual(pillarmod.ls(), ['a', 'b'])
+                self.assertEqual(ls, ['a', 'b'])
 
     @skipIf(NO_MOCK, NO_MOCK_REASON)
     def test_pillar_get_default_merge(self):

tests/unit/serializers/test_serializers.py

@@ -152,7 +152,7 @@ def test_compare_sls_vs_yaml_with_jinja(self):
 
         # BLAAM! yml_src is not valid !
         final_obj = OrderedDict(yaml.deserialize(yml_src))
-        assert obj != final_obj
+        assert obj != final_obj, 'Objects matched! {} == {}'.format(obj, final_obj)
 
     @skipIf(not yamlex.available, SKIP_MESSAGE % 'sls')
     def test_sls_aggregate(self):

tests/unit/states/test_proxy.py

@@ -14,9 +14,6 @@
     call
 )
 
-# Import 3rd-party libs
-from salt.ext import six
-
 # Import Salt Libs
 import salt.states.proxy as proxy
 
@@ -70,9 +67,7 @@ def test_set_proxy_macos(self):
             with patch.dict(proxy.__salt__, patches):
                 out = proxy.managed('192.168.0.1', '3128', user='frank', password='passw0rd',
                                     bypass_domains=['salt.com', 'test.com'])
-                if six.PY3:
-                    # Sorting is different in Py3
-                    out['changes']['new'][-1]['bypass_domains'] = sorted(out['changes']['new'][-1]['bypass_domains'])
+                out['changes']['new'][-1]['bypass_domains'] = sorted(out['changes']['new'][-1]['bypass_domains'])
 
                 calls = [
                     call('192.168.0.1', '3128', 'frank', 'passw0rd', 'Ethernet'),

tests/unit/test_module_names.py

@@ -47,6 +47,7 @@
     os.path.join('tests', 'modparser.py'),
     os.path.join('tests', 'committer_parser.py'),
     os.path.join('tests', 'zypp_plugin.py'),
+    os.path.join('tests', 'tox-helper.py'),
     os.path.join('tests', 'unit', 'transport', 'mixins.py'),
     os.path.join('tests', 'integration', 'utils', 'testprogram.py'),
 ]