Merge pull request #106 from Breakthrough-Energy/daniel/no_gurobi_req

refactor: remove Gurobi requirement

Dockerfile

@@ -21,7 +21,7 @@ ENV PATH="$PATH:/usr/share/julia-1.5.3/bin" \
 WORKDIR /app
 COPY . .
 
-RUN julia -e 'using Pkg; Pkg.activate("."); Pkg.instantiate(), using REISE' && \
+RUN julia -e 'using Pkg; Pkg.activate("."); Pkg.instantiate(); Pkg.add("Gurobi"); import Gurobi; using REISE' && \
     pip install -r requirements.txt
 
 

Manifest.toml

@@ -39,11 +39,6 @@ git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e"
 uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
 version = "1.0.6+5"
 
-[[CEnum]]
-git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
-uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
-version = "0.4.1"
-
 [[CSV]]
 deps = ["CategoricalArrays", "DataFrames", "Dates", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode"]
 git-tree-sha1 = "f095e44feec53d0ae809714a78c25908d1f370e6"
@@ -154,12 +149,6 @@ version = "0.10.14"
 deps = ["Random"]
 uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
 
-[[Gurobi]]
-deps = ["CEnum", "Libdl", "MathOptInterface"]
-git-tree-sha1 = "1719220ceddc23fafa82e9fcc28543a81505e5f3"
-uuid = "2e9cd046-0924-5485-92f1-d5272153d98b"
-version = "0.9.3"
-
 [[HDF5]]
 deps = ["Blosc", "HDF5_jll", "Libdl", "Mmap", "Random"]
 git-tree-sha1 = "0b812e7872e2199a5a04944f486b4048944f1ed8"
@@ -347,6 +336,12 @@ git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0"
 uuid = "189a3867-3050-52da-a836-e630ba90ab69"
 version = "0.2.0"
 
+[[Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "cfbac6c1ed70c002ec6361e7fd334f02820d6419"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.1.2"
+
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 

Project.toml

@@ -7,14 +7,13 @@ version = "0.1.0"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
-Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 MAT = "23992714-dd62-5051-b70f-ba57cb901cac"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 DataFrames = "0.21"
-Gurobi = "0.9"
 JuMP = "0.21.3"

pyreisejl/utility/call.py

@@ -34,89 +34,109 @@ def _record_scenario(scenario_id, runtime):
     )
 
 
-def launch_scenario(
-    start_date, end_date, interval, input_dir, execute_dir=None, threads=None
-):
-    """Launches the scenario.
+class Launcher:
+    """Parent class for solver-specific scenario launchers.
 
     :param str start_date: start date of simulation as 'YYYY-MM-DD HH:MM:SS',
-    where HH, MM, and SS are optional.
+        where HH, MM, and SS are optional.
     :param str end_date: end date of simulation as 'YYYY-MM-DD HH:MM:SS',
-    where HH, MM, and SS are optional.
+        where HH, MM, and SS are optional.
     :param int interval: length of each interval in hours
     :param str input_dir: directory with input data
-    :param None/str execute_dir: directory for execute data. None defaults to an
-    execute folder that will be created in the input directory
-    :param None/int threads: number of threads to use, None defaults to auto.
-    :return: (*int*) runtime of scenario in seconds
     :raises InvalidDateArgument: if start_date is posterior to end_date
-    :raises InvalidInterval: if the interval does not evently divide the given date range
+    :raises InvalidInterval: if the interval doesn't evently divide the given date range
     """
-    # extract time limits from 'demand.csv'
-    with open(os.path.join(input_dir, "demand.csv")) as profile:
-        min_ts, max_ts, freq = extract_date_limits(profile)
 
-    dates = pd.date_range(start=min_ts, end=max_ts, freq=freq)
-
-    start_ts = validate_time_format(start_date)
-    end_ts = validate_time_format(end_date, end_date=True)
-
-    # make sure the dates are within the time frame we have data for
-    validate_time_range(start_ts, min_ts, max_ts)
-    validate_time_range(end_ts, min_ts, max_ts)
-
-    if start_ts > end_ts:
-        raise InvalidDateArgument(
-            f"The start date ({start_ts}) cannot be after the end date ({end_ts})."
+    def __init__(self, start_date, end_date, interval, input_dir):
+        """Constructor."""
+        # extract time limits from 'demand.csv'
+        with open(os.path.join(input_dir, "demand.csv")) as profile:
+            min_ts, max_ts, freq = extract_date_limits(profile)
+
+        dates = pd.date_range(start=min_ts, end=max_ts, freq=freq)
+
+        start_ts = validate_time_format(start_date)
+        end_ts = validate_time_format(end_date, end_date=True)
+
+        # make sure the dates are within the time frame we have data for
+        validate_time_range(start_ts, min_ts, max_ts)
+        validate_time_range(end_ts, min_ts, max_ts)
+
+        if start_ts > end_ts:
+            raise InvalidDateArgument(
+                f"The start date ({start_ts}) cannot be after the end date ({end_ts})."
+            )
+
+        # Julia starts at 1
+        start_index = dates.get_loc(start_ts) + 1
+        end_index = dates.get_loc(end_ts) + 1
+
+        # Calculate number of intervals
+        ts_range = end_index - start_index + 1
+        if ts_range % interval > 0:
+            raise InvalidInterval(
+                "This interval does not evenly divide the given date range."
+            )
+        self.start_index = start_index
+        self.interval = interval
+        self.n_interval = int(ts_range / interval)
+        self.input_dir = input_dir
+        print("Validation complete!")
+
+    def _print_settings(self):
+        print("Launching scenario with parameters:")
+        print(
+            {
+                "interval": self.interval,
+                "n_interval": self.n_interval,
+                "start_index": self.start_index,
+                "input_dir": self.input_dir,
+                "execute_dir": self.execute_dir,
+                "threads": self.threads,
+            }
         )
 
-    # Julia starts at 1
-    start_index = dates.get_loc(start_ts) + 1
-    end_index = dates.get_loc(end_ts) + 1
-
-    # Calculate number of intervals
-    ts_range = end_index - start_index + 1
-    if ts_range % interval > 0:
-        raise InvalidInterval(
-            "This interval does not evenly divide the given date range."
+    def launch_scenario(self):
+        # This should be defined in sub-classes
+        raise NotImplementedError
+
+
+class GurobiLauncher(Launcher):
+    def launch_scenario(self, execute_dir=None, threads=None, solver_kwargs=None):
+        """Launches the scenario.
+
+        :param None/str execute_dir: directory for execute data. None defaults to an
+            execute folder that will be created in the input directory
+        :param None/int threads: number of threads to use.
+        :param None/dict solver_kwargs: keyword arguments to pass to solver (if any).
+        :return: (*int*) runtime of scenario in seconds
+        """
+        self.execute_dir = execute_dir
+        self.threads = threads
+        self._print_settings()
+        # Import these within function because there is a lengthy compilation step
+        from julia.api import Julia
+
+        Julia(compiled_modules=False)
+        from julia import Gurobi  # noqa: F401
+        from julia import REISE
+
+        start = time()
+        REISE.run_scenario_gurobi(
+            interval=self.interval,
+            n_interval=self.n_interval,
+            start_index=self.start_index,
+            inputfolder=self.input_dir,
+            outputfolder=self.execute_dir,
+            threads=self.threads,
         )
+        end = time()
 
-    n_interval = int(ts_range / interval)
-
-    # Import these within function because there is a lengthy compilation step
-    from julia.api import Julia
-
-    Julia(compiled_modules=False)
-    from julia import REISE
-
-    print("Validation complete! Launching scenario with parameters:")
-    print(
-        {
-            "interval": interval,
-            "n_interval": n_interval,
-            "start_index": start_index,
-            "input_dir": input_dir,
-            "execute_dir": execute_dir,
-            "threads": threads,
-        }
-    )
-
-    start = time()
-    REISE.run_scenario(
-        interval=interval,
-        n_interval=n_interval,
-        start_index=start_index,
-        inputfolder=input_dir,
-        outputfolder=execute_dir,
-        threads=threads,
-    )
-    end = time()
-
-    runtime = round(end - start)
-    hours, minutes, seconds = sec2hms(runtime)
-    print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}")
+        runtime = round(end - start)
+        hours, minutes, seconds = sec2hms(runtime)
+        print(f"Run time: {hours}:{minutes:02d}:{seconds:02d}")
 
-    return runtime
+        return runtime
 
 
 def main(args):
@@ -142,14 +162,13 @@ def main(args):
         )
         raise WrongNumberOfArguments(err_str)
 
-    runtime = launch_scenario(
+    launcher = GurobiLauncher(
         args.start_date,
         args.end_date,
         args.interval,
         args.input_dir,
-        args.execute_dir,
-        args.threads,
     )
+    runtime = launcher.launch_scenario(args.execute_dir, args.threads)
 
     # If using PowerSimData, record the runtime
     if args.scenario_id:

src/REISE.jl

@@ -4,8 +4,8 @@ import CSV
 import DataFrames
 import Dates
 import JuMP
-import Gurobi
 import MAT
+import Requires
 import SparseArrays: sparse, SparseMatrixCSC
 
 
@@ -19,6 +19,13 @@ include("query.jl")         # Defines get_results (used in interval_loop)
 include("save.jl")          # Defines save_input_mat, save_results
 
 
+function __init__()
+    Requires.@require Gurobi="2e9cd046-0924-5485-92f1-d5272153d98b" begin
+        include(joinpath("solver_specific", "gurobi.jl"))
+    end
+end
+
+
 """
     REISE.run_scenario(;
         interval=24, n_interval=3, start_index=1, outputfolder="output",
@@ -40,20 +47,18 @@ Run a scenario consisting of several intervals.
 function run_scenario(;
         num_segments::Int=1, interval::Int, n_interval::Int, start_index::Int,
         inputfolder::String, outputfolder::Union{String, Nothing}=nothing,
-        threads::Union{Int, Nothing}=nothing, optimizer_factory=nothing)
+        threads::Union{Int, Nothing}=nothing, optimizer_factory=nothing,
+        solver_kwargs::Union{Dict, Nothing}=nothing)
+    isnothing(optimizer_factory) && error("optimizer_factory must be specified")
     # Setup things that build once
+    # If no solver kwargs passed, instantiate an empty dict
+    solver_kwargs = something(solver_kwargs, Dict())
     # If outputfolder not given, by default assign it inside inputfolder
     isnothing(outputfolder) && (outputfolder = joinpath(inputfolder, "output"))
     # If outputfolder doesn't exist (isdir evaluates false) create it (mkdir)
     isdir(outputfolder) || mkdir(outputfolder)
     stdout_filepath = joinpath(outputfolder, "stdout.log")
     stderr_filepath = joinpath(outputfolder, "stderr.err")
-    if isnothing(optimizer_factory)
-        optimizer_factory = Gurobi.Env()
-        solver_kwargs = Dict("Method" => 2, "Crossover" => 0)
-    else
-        solver_kwargs = Dict()
-    end
     case = read_case(inputfolder)
     storage = read_storage(inputfolder)
     println("All scenario files loaded!")
@@ -71,14 +76,10 @@ function run_scenario(;
     println("Redirecting outputs, see stdout.log & stderr.err in outputfolder")
     redirect_stdout_stderr(stdout_filepath, stderr_filepath) do
         # Loop through intervals
-        interval_loop(optimizer_factory, model_kwargs, solver_kwargs, interval,
-                      n_interval, start_index, inputfolder, outputfolder)
-        GC.gc()
-        if isa(optimizer_factory, Gurobi.Env)
-            Gurobi.finalize(optimizer_factory)
-            println("Connection closed successfully!")
-        end
+        m = interval_loop(optimizer_factory, model_kwargs, solver_kwargs, interval,
+                          n_interval, start_index, inputfolder, outputfolder)
     end
+    return m
 end
 
 # Module end

src/loop.jl

@@ -4,12 +4,8 @@ function symbolize(d::Dict{String,Any})::NamedTuple
 end
 
 
-function new_model(factory_like)::Union{JuMP.Model, JuMP.MOI.AbstractOptimizer}
-    if isa(factory_like, Gurobi.Env)
-        return JuMP.direct_model(Gurobi.Optimizer(factory_like))
-    else
-        return JuMP.Model(factory_like)
-    end
+function new_model(optimizer_factory)::JuMP.Model
+    return JuMP.Model(optimizer_factory)
 end
 
 
@@ -19,7 +15,7 @@ end
 
 Given:
 - optimizer instantiation object `factory_like`:
-    either a Gurobi environment or something that can be passed to JuMP.Model
+    something that can be passed to new_model (goes to JuMP.Model by default)
 - a dictionary of model keyword arguments `model_kwargs`
 - a dictionary of solver keyword arguments `solver_kwargs`
 - an interval length `interval` (hours)
@@ -139,7 +135,7 @@ function interval_loop(factory_like, model_kwargs::Dict,
                 results = get_results(f, voi, model_kwargs["case"])
                 break
             elseif ((status in numeric_statuses)
-                    & isa(factory_like, Gurobi.Env)
+                    & JuMP.solver_name(m) == "Gurobi"
                     & !("BarHomogeneous" in keys(solver_kwargs)))
                 # if Gurobi, and BarHomogeneous is not enabled, enable it and re-solve
                 solver_kwargs["BarHomogeneous"] = 1
@@ -154,7 +150,7 @@ function interval_loop(factory_like, model_kwargs::Dict,
                 JuMP.set_optimizer_attributes(m, pairs(solver_kwargs)...)
                 m, voi = _build_model(m; symbolize(model_kwargs)...)
                 intervals_without_loadshed = 0
-            elseif (isa(factory_like, Gurobi.Env)
+            elseif (JuMP.solver_name(m) == "Gurobi"
                     & !("BarHomogeneous" in keys(solver_kwargs)))
                 # if Gurobi, and BarHomogeneous is not enabled, enable it and re-solve
                 solver_kwargs["BarHomogeneous"] = 1
@@ -219,4 +215,6 @@ function interval_loop(factory_like, model_kwargs::Dict,
             end
         end
     end
+
+    return m
 end