diff --git a/doc/source/examples/ps_grid_sampling.rst b/doc/source/examples/ps_grid_sampling.rst index 4f5cb245..fe164ba3 100644 --- a/doc/source/examples/ps_grid_sampling.rst +++ b/doc/source/examples/ps_grid_sampling.rst @@ -43,7 +43,7 @@ where :math:`l_b=0` and :math:`u_b=15`, the grid of sample looks like: all_trials = [] while True: - trial = gen.ask(1) + trial = gen.ask_trials(1) if trial: all_trials.append(trial[0]) else: diff --git a/doc/source/examples/ps_line_sampling.rst b/doc/source/examples/ps_line_sampling.rst index 36f6a6eb..2175b388 100644 --- a/doc/source/examples/ps_line_sampling.rst +++ b/doc/source/examples/ps_line_sampling.rst @@ -46,7 +46,7 @@ where :math:`x_0` and :math:`x_1` have a default values of :math:`5` and all_trials = [] while True: - trial = gen.ask(1) + trial = gen.ask_trials(1) if trial: all_trials.append(trial[0]) else: diff --git a/doc/source/examples/ps_random_sampling.rst b/doc/source/examples/ps_random_sampling.rst index 1593c6d7..04bc1844 100644 --- a/doc/source/examples/ps_random_sampling.rst +++ b/doc/source/examples/ps_random_sampling.rst @@ -45,7 +45,7 @@ distribution such as: all_trials = [] while len(all_trials) <= 100: - trial = gen.ask(1) + trial = gen.ask_trials(1) if trial: all_trials.append(trial[0]) else: diff --git a/optimas/core/trial.py b/optimas/core/trial.py index 6879cb32..efcafa4d 100644 --- a/optimas/core/trial.py +++ b/optimas/core/trial.py @@ -81,6 +81,74 @@ def __init__( self._mapped_evaluations[ev.parameter.name] = ev self.mark_as(TrialStatus.CANDIDATE) + def to_dict(self) -> Dict: + """Convert the trial to a dictionary.""" + trial_dict = { + **self.parameters_as_dict(), + **self.objectives_as_dict(), + **self.analyzed_parameters_as_dict(), + **self.custom_parameters_as_dict(), + "_id": self._index, + "_ignored": self._ignored, + "_ignored_reason": self._ignored_reason, + "_status": self._status, + } + return trial_dict + + @classmethod + def from_dict( + cls, + trial_dict: Dict, + varying_parameters: List[VaryingParameter], + objectives: List[Objective], + analyzed_parameters: List[Parameter], + custom_parameters: Optional[List[TrialParameter]] = None, + ) -> "Trial": + """Create a trial from a dictionary. + + Parameters + ---------- + trial_dict : dict + Dictionary containing the trial information. + varying_parameters : list of VaryingParameter + The varying parameters of the optimization. + objectives : list of Objective + The optimization objectives. + analyzed_parameters : list of Parameter, optional + Additional parameters to be analyzed during the optimization. + custom_parameters : list of TrialParameter, optional + Additional parameters needed to identify or carry out the trial, and + which will be included in the optimization history. + """ + # Prepare values of the input parameters + parameter_values = [trial_dict[var.name] for var in varying_parameters] + # Prepare evaluations + evaluations = [ + Evaluation(parameter=par, value=trial_dict[par.name]) + for par in objectives + analyzed_parameters + if par.name in trial_dict + ] + # Create the trial object + trial = cls( + varying_parameters=varying_parameters, + objectives=objectives, + analyzed_parameters=analyzed_parameters, + parameter_values=parameter_values, + evaluations=evaluations, + custom_parameters=custom_parameters, + ) + if "_id" in trial_dict: + trial._index = trial_dict["_id"] + if "_ignored" in trial_dict: + trial._ignored = trial_dict["_ignored"] + if "_ignored_reason" in trial_dict: + trial._ignored_reason = trial_dict["_ignored_reason"] + if "_status" in trial_dict: + trial._status = trial_dict["_status"] + for custom_param in custom_parameters: + setattr(trial, custom_param.name, trial_dict[custom_param.name]) + return trial + @property def varying_parameters(self) -> List[VaryingParameter]: """Get the list of varying parameters.""" @@ -225,7 +293,8 @@ def objectives_as_dict(self) -> Dict: params = {} for obj in self._objectives: ev = self._mapped_evaluations[obj.name] - params[obj.name] = (ev.value, ev.sem) + if ev is not None: + params[obj.name] = ev.value return params def analyzed_parameters_as_dict(self) -> Dict: @@ -237,5 +306,13 @@ def analyzed_parameters_as_dict(self) -> Dict: params = {} for par in self._analyzed_parameters: ev = self._mapped_evaluations[par.name] - params[par.name] = (ev.value, ev.sem) + if ev is not None: + params[par.name] = ev.value + return params + + def custom_parameters_as_dict(self) -> Dict: + """Get a mapping between names and values of the custom parameters.""" + params = {} + for param in self._custom_parameters: + params[param.name] = getattr(self, param.name) return params diff --git a/optimas/gen_functions.py b/optimas/gen_functions.py index c3e37eb3..7879a7e7 100644 --- a/optimas/gen_functions.py +++ b/optimas/gen_functions.py @@ -20,7 +20,7 @@ def persistent_generator(H, persis_info, gen_specs, libE_info): """Generate and launch evaluations with the optimas generators. This function gets the generator object and uses it to generate new - evaluations via the `ask` method. Once finished, the result of the + evaluations via the `ask_trials` method. Once finished, the result of the evaluations is communicated back to the generator via the `tell` method. This is a persistent generator function, i.e., it is called by a dedicated @@ -68,7 +68,7 @@ def persistent_generator(H, persis_info, gen_specs, libE_info): # Ask the optimizer to generate `batch_size` new points # Store this information in the format expected by libE H_o = np.zeros(number_of_gen_points, dtype=gen_specs["out"]) - generated_trials = generator.ask(number_of_gen_points) + generated_trials = generator.ask_trials(number_of_gen_points) for i, trial in enumerate(generated_trials): for var, val in zip( trial.varying_parameters, trial.parameter_values @@ -108,7 +108,7 @@ def persistent_generator(H, persis_info, gen_specs, libE_info): ev = Evaluation(parameter=par, value=y) trial.complete_evaluation(ev) # Register trial with unknown SEM - generator.tell([trial]) + generator.tell_trials([trial]) # Set the number of points to generate to that number: number_of_gen_points = min(n + n_failed_gens, max_evals - n_gens) n_failed_gens = 0 diff --git a/optimas/generators/ax/developer/multitask.py b/optimas/generators/ax/developer/multitask.py index de4c76ab..4d0a3734 100644 --- a/optimas/generators/ax/developer/multitask.py +++ b/optimas/generators/ax/developer/multitask.py @@ -162,7 +162,7 @@ def _check_inputs( "to the number of high-fidelity trials" ) - def _ask(self, trials: List[Trial]) -> List[Trial]: + def ask(self, trials: List[Trial]) -> List[Trial]: """Fill in the parameter values of the requested trials.""" for trial in trials: next_trial = self._get_next_trial_arm() @@ -177,7 +177,7 @@ def _ask(self, trials: List[Trial]) -> List[Trial]: trial.trial_index = trial_index return trials - def _tell(self, trials: List[Trial]) -> None: + def tell(self, trials: List[Trial]) -> None: """Incorporate evaluated trials into experiment.""" if self.gen_state == NOT_STARTED: self._incorporate_external_data(trials) diff --git a/optimas/generators/ax/service/base.py b/optimas/generators/ax/service/base.py index 3d2bf42b..8f352588 100644 --- a/optimas/generators/ax/service/base.py +++ b/optimas/generators/ax/service/base.py @@ -23,7 +23,7 @@ Trial, VaryingParameter, Parameter, - TrialStatus, + TrialParameter, ) from optimas.generators.ax.base import AxGenerator from optimas.utils.ax import AxModelManager @@ -109,6 +109,9 @@ def __init__( model_save_period: Optional[int] = 5, model_history_dir: Optional[str] = "model_history", ) -> None: + custom_trial_parameters = [ + TrialParameter("ax_trial_id", dtype=int), + ] super().__init__( varying_parameters=varying_parameters, objectives=objectives, @@ -119,6 +122,7 @@ def __init__( save_model=save_model, model_save_period=model_save_period, model_history_dir=model_history_dir, + custom_trial_parameters=custom_trial_parameters, allow_fixed_parameters=True, allow_updating_parameters=True, ) @@ -142,21 +146,33 @@ def model(self) -> AxModelManager: """Get access to the underlying model using an `AxModelManager`.""" return self._model - def _ask(self, trials: List[Trial]) -> List[Trial]: - """Fill in the parameter values of the requested trials.""" - for trial in trials: + def ask(self, num_points: Optional[int]) -> List[dict]: + """Request the next set of points to evaluate.""" + points = [] + for _ in range(num_points): parameters, trial_id = self._ax_client.get_next_trial( fixed_features=self._fixed_features ) - trial.parameter_values = [ - parameters.get(var.name) for var in self._varying_parameters - ] - trial.ax_trial_id = trial_id - return trials + point = { + var.name: parameters.get(var.name) + for var in self._varying_parameters + } + point["ax_trial_id"] = trial_id + points.append(point) + return points - def _tell(self, trials: List[Trial]) -> None: - """Incorporate evaluated trials into Ax client.""" - for trial in trials: + def tell(self, results: List[dict]) -> None: + """ + Send the results of evaluations to the generator. + """ + for result in results: + trial = Trial.from_dict( + trial_dict=result, + varying_parameters=self._varying_parameters, + objectives=self._objectives, + analyzed_parameters=self._analyzed_parameters, + custom_parameters=self._custom_trial_parameters, + ) try: trial_id = trial.ax_trial_id ax_trial = self._ax_client.get_trial(trial_id) diff --git a/optimas/generators/base.py b/optimas/generators/base.py index 931a64e1..d856de53 100644 --- a/optimas/generators/base.py +++ b/optimas/generators/base.py @@ -187,7 +187,7 @@ def n_evaluated_trials(self) -> int: n_evaluated += 1 return n_evaluated - def ask(self, n_trials: int) -> List[Trial]: + def ask_trials(self, n_trials: int) -> List[Trial]: """Ask the generator to suggest the next ``n_trials`` to evaluate. Parameters @@ -204,18 +204,19 @@ def ask(self, n_trials: int) -> List[Trial]: # Generate as many trials as needed and add them to the queue. if n_trials > self.n_queued_trials: n_gen = n_trials - self.n_queued_trials + # Ask generator for n_gen points, using the standardized API + gen_points = self.ask(n_gen) + # Convert the points to the Trial format gen_trials = [] - for _ in range(n_gen): - gen_trials.append( - Trial( - varying_parameters=self._varying_parameters, - objectives=self._objectives, - analyzed_parameters=self._analyzed_parameters, - custom_parameters=self._custom_trial_parameters, - ) + for point in gen_points: + trial = Trial.from_dict( + trial_dict=point, + varying_parameters=self._varying_parameters, + objectives=self._objectives, + analyzed_parameters=self._analyzed_parameters, + custom_parameters=self._custom_trial_parameters, ) - # Ask the generator to fill them. - gen_trials = self._ask(gen_trials) + gen_trials.append(trial) # Keep only trials that have been given data. for trial in gen_trials: if len(trial.parameter_values) > 0: @@ -236,7 +237,7 @@ def ask(self, n_trials: int) -> List[Trial]: trials.append(trial) return trials - def tell( + def tell_trials( self, trials: List[Trial], allow_saving_model: Optional[bool] = True ) -> None: """Give trials back to generator once they have been evaluated. @@ -250,7 +251,10 @@ def tell( incorporating the evaluated trials. By default ``True``. """ - self._tell(trials) + # Feed data to the generator, using the standardized API + self.tell([trial.to_dict() for trial in trials]) + + # Perform additional checks that rely on the trial format for trial in trials: if trial not in self._given_trials: self._add_external_evaluated_trial(trial) @@ -290,7 +294,7 @@ def incorporate_history(self, history: np.ndarray) -> None: trials = self._create_trials_from_external_data( history_ended, ignore_unrecognized_parameters=True ) - self.tell(trials, allow_saving_model=False) + self.tell_trials(trials, allow_saving_model=False) # Communicate to history array whether the trial has been ignored. for trial in trials: idxs = np.where(history["trial_index"] == trial.index)[0] @@ -306,7 +310,7 @@ def attach_trials( The given trials are placed at the top of the queue of trials that will be proposed by the generator (that is, they will be the first - ones to be proposed the next time that `ask` is called). + ones to be proposed the next time that `ask_trials` is called). Parameters ---------- @@ -409,7 +413,7 @@ def _add_trial_to_queue( By default, the trial will be appended to the end of the queue, unless a `queue_index` is given. Trials at the top of the queue will be the - first ones to be given for evaluation when `ask` is called. + first ones to be given for evaluation when `ask_trials` is called. Parameters ---------- @@ -579,28 +583,27 @@ def get_libe_specs(self) -> Dict: libE_specs = {} return libE_specs - def _ask(self, trials: List[Trial]) -> List[Trial]: - """Ask method to be implemented by the Generator subclasses. + def ask(self, num_points: Optional[int]) -> List[dict]: + """Request the next set of points to evaluate. Parameters ---------- - trials : list of Trial - A list with as many trials as requested to the generator. The - trials do not yet contain the values of the varying parameters. - These values should instead be supplied in this method. + num_points : int + Number of points to generate. """ - return trials + return [] - def _tell(self, trials: List[Trial]) -> None: - """Tell method to be implemented by the Generator subclasses. + def tell(self, results: List[dict]) -> None: + """ + Send the results of evaluations to the generator. Parameters ---------- - trials : list of Trial - A list with all evaluated trials. All evaluations included in the - trials should be incorporated to the generator model in this - method. + results : list of dict + List with the results of the evaluations. + All evaluations included in the results should be incorporated + to the generator model in this method. """ pass diff --git a/optimas/generators/grid_sampling.py b/optimas/generators/grid_sampling.py index 8d7e5e7c..82d43c0e 100644 --- a/optimas/generators/grid_sampling.py +++ b/optimas/generators/grid_sampling.py @@ -67,15 +67,14 @@ def _create_configurations(self) -> None: all_configs.append(config) self._all_configs = all_configs - def _ask(self, trials: List[Trial]) -> List[Trial]: - """Fill in the parameter values of the requested trials.""" - for trial in trials: + def ask(self, num_points: Optional[int]) -> List[dict]: + """Request the next set of points to evaluate.""" + points = [] + for _ in range(num_points): if self._all_configs: config = self._all_configs.pop(0) - trial.parameter_values = [ - config[var.name] for var in trial.varying_parameters - ] - return trials + points.append(config) + return points def _mark_trial_as_failed(self, trial: Trial): """No need to do anything, since there is no surrogate model.""" diff --git a/optimas/generators/line_sampling.py b/optimas/generators/line_sampling.py index dd67a0e4..deb88d64 100644 --- a/optimas/generators/line_sampling.py +++ b/optimas/generators/line_sampling.py @@ -103,15 +103,14 @@ def _create_configurations(self) -> None: # Store configurations. self._all_configs = all_configs - def _ask(self, trials: List[Trial]) -> List[Trial]: - """Fill in the parameter values of the requested trials.""" - for trial in trials: + def ask(self, num_points: Optional[int]) -> List[dict]: + """Request the next set of points to evaluate.""" + points = [] + for _ in range(num_points): if self._all_configs: config = self._all_configs.pop(0) - trial.parameter_values = [ - config[var.name] for var in trial.varying_parameters - ] - return trials + points.append(config) + return points def _mark_trial_as_failed(self, trial: Trial): """No need to do anything, since there is no surrogate model.""" diff --git a/optimas/generators/random_sampling.py b/optimas/generators/random_sampling.py index 0bbced4d..b981139f 100644 --- a/optimas/generators/random_sampling.py +++ b/optimas/generators/random_sampling.py @@ -57,13 +57,16 @@ def __init__( self._rng = np.random.default_rng(seed) self._define_generator_parameters() - def _ask(self, trials: List[Trial]) -> List[Trial]: - """Fill in the parameter values of the requested trials.""" - n_trials = len(trials) - configs = self._generate_sampling[self._distribution](n_trials) - for trial, config in zip(trials, configs): - trial.parameter_values = config - return trials + def ask(self, num_points: Optional[int]) -> List[dict]: + """Request the next set of points to evaluate.""" + configs = self._generate_sampling[self._distribution](num_points) + points = [] + for config in configs: + point = {} + for var, value in zip(self._varying_parameters, config): + point[var.name] = value + points.append(point) + return points def _check_inputs( self,