Add Pca-bo (#1179)

* first_draft_pcabo

* Fix

* set_name

* black

* modifications_by_olivier

* fix

* minor_fixes

* minor_fixes

* fix

* fixes

* fixes

* Fix

* fix

* requirements fix

* trial fix bayes_optim import

* fix and adding in requirements

* fix

* Update bayes-optim package

* Working PCA-BO, still to clean and check in detail

* black

* trial to solve suggest problem

* removed files added by mistake

* Update mypy.ini

* fix

* fix

* _internal_tell_not_asked added to PCABO

* black

* update recorderd_recommendations.csv

* pull master

* requirements fix

* update experiments.py

* AX and BOBYQA ad in recastlib.py

* Removed AX and BOBYQA from recastlib.py

* Added bayes-optim BO optimizer

* fix in benchmark/experiments.py

* black

* fix tp.List

* Better version of PCA-BO with _PCABO and ParametrizedPCABO. PCA-BO variants for testing purposes.

* fix

* black

* Merged PCABO and BayesOptimBO classes

* black

* fix

* fix

* trial to solve import bayes_optim failing

* Refinements in optimizerlib.py

* little fix

* fix

* Refinement

* Update nevergrad/optimization/optimizerlib.py

Co-authored-by: Jérémy Rapin <jrapin.github@gmail.com>

* Update nevergrad/optimization/optimizerlib.py

Co-authored-by: Jérémy Rapin <jrapin.github@gmail.com>

* Update nevergrad/optimization/optimizerlib.py

Co-authored-by: Jérémy Rapin <jrapin.github@gmail.com>

* Changed ParametrizedBayesOptimBO in ParametrizedBayesOptim

* fix according to insertion of self._config in _BayesOptim class

* fix

* fix

* fix

* trial to fix CI pytest error

* ArctanBound from transforms is now used

* minor changes

* updated bayes-optim

* back to old version of bayes-optim

* fix

* black

* update in bench.txt

* name change for BayesOptim class

* updated bayes-optim version in the requirements

* fix

* added BayesOptim to families.py for documentation

* black

* Update nevergrad/optimization/optimizerlib.py

Co-authored-by: Elena Raponi <elenaraponi@fb.com>
Co-authored-by: Elena <71965364+elenaraponi@users.noreply.github.com>
Co-authored-by: Jérémy Rapin <jrapin.github@gmail.com>

mypy.ini

@@ -1,6 +1,6 @@
 [mypy]
 
-[mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym,gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*]
+[mypy-scipy.*,requests,pandas,compiler_gym,compiler_gym.*,gym,gym.*,gym_anm,matplotlib.*,pytest,cma,bayes_opt.*,torchvision.models,torch.*,mpl_toolkits.*,fcmaes.*,tqdm,pillow,PIL,PIL.Image,sklearn.*,pyomo.*,pyproj,IOHexperimenter.*,tensorflow,koncept.models,cv2,imquality,imquality.brisque,lpips,mixsimulator.*,networkx.*,cdt.*,pymoo,pymoo.*,bayes_optim.*]
 ignore_missing_imports = True
 
 [mypy-nevergrad.functions.rl.agents,torchvision,torchvision.*,nevergrad.functions.games.*,nevergrad.functions.multiobjective.pyhv,nevergrad.optimization.test_doc,,pymoo,pymoo.*]

nevergrad/benchmark/experiments.py

@@ -664,7 +664,8 @@ def yabbob(
         optims += get_optimizers("splitters", seed=next(seedg))  # type: ignore
 
     if hd and small:
-        optims = ["BO", "BOSplit", "CMA", "PSO", "DE"]
+        optims = ["BO", "CMA", "PSO", "DE"]
+
     # List of objective functions.
     functions = [
         ArtificialFunction(name, block_dimension=d, rotation=rotation, noise_level=noise_level, split=split)

nevergrad/benchmark/optimizer_groups.txt

@@ -1,3 +1,6 @@
+
+progressive = [ProgDiscAuto, ProgDisc13, ProgDiscInf, ProgAuto, Prog13, ProgInf]
+
 all_bo = ['BO', 'BOSplit', 'ChainBOwithLHS', 'ChainBOwithLHS30', 'ChainBOwithLHSdim', 'ChainBOwithLHSsqrt', 'ChainBOwithMetaRecentering', 'ChainBOwithMetaRecentering30', 'ChainBOwithMetaRecenteringdim', 'ChainBOwithMetaRecenteringsqrt', 'ChainBOwithMetaTuneRecentering', 'ChainBOwithMetaTuneRecentering30', 'ChainBOwithMetaTuneRecenteringdim', 'ChainBOwithMetaTuneRecenteringsqrt', 'ChainBOwithR', 'ChainBOwithR30', 'ChainBOwithRdim', 'ChainBOwithRsqrt', 'LBO', 'MidQRBO', 'QRBO', 'RBO']
 
 baselines = ['OnePlusOne', 'DiscreteOnePlusOne', 'NoisyDiscreteOnePlusOne', 'PSO', 'DE', 'TwoPointsDE']
@@ -38,4 +41,4 @@ spsa = ['NaiveTBPSA', 'SPSA', 'TBPSA']
 
 structure = ['RecES', 'RecMixES', 'RecMutDE', 'ParametrizationDE']
 
-structured_moo = ['CMA', 'NGOpt10', 'MetaNGOpt10', 'DE', 'PSO', 'RecES', 'RecMixES', 'RecMutDE', 'ParametrizationDE']
+structured_moo = ['CMA', 'NGOpt10', 'MetaNGOpt10', 'DE', 'PSO', 'RecES', 'RecMixES', 'RecMutDE', 'ParametrizationDE']

nevergrad/functions/images/core.py

@@ -88,7 +88,7 @@ def __init__(
         self.loss_function = loss(reference=self.data)
 
     def _generate_images(self, x: np.ndarray) -> np.ndarray:
-        """ Generates images tensor of shape [nb_images, x, y, 3] with pixels between 0 and 255"""
+        """Generates images tensor of shape [nb_images, x, y, 3] with pixels between 0 and 255"""
         # pylint: disable=not-callable
         noise = torch.tensor(x.astype("float32"))
         return (
@@ -337,7 +337,7 @@ def _loss(self, x: np.ndarray) -> float:
         return loss
 
     def _generate_images(self, x: np.ndarray) -> np.ndarray:
-        """ Generates images tensor of shape [nb_images, x, y, 3] with pixels between 0 and 255"""
+        """Generates images tensor of shape [nb_images, x, y, 3] with pixels between 0 and 255"""
         # pylint: disable=not-callable
         noise = torch.tensor(x.astype("float32"))
         return ((self.pgan_model.test(noise).clamp(min=-1, max=1) + 1) * 255.99 / 2).permute(0, 2, 3, 1).cpu().numpy()  # type: ignore

nevergrad/functions/leaderboard.csv

@@ -6,7 +6,7 @@
 "arcoating,8,400",13.100717405397,"[5.0,6.333333333333333,7.666666666666666,9.0,9.0,9.0,9.0,9.0]"
 "bragg,16",0.932163816851,"[2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5]"
 "bragg,80",0.000694261986,"[2.999999999999995,2.0,2.9999999999999982,2.0,2.9999999999999942,2.0000000000000204,2.9999999999999982,2.0000000000000204,2.999999999999997,2.0000000000000577,2.999999999999994,2.0000000000000178,2.9999999999999996,2.0000000000000258,2.9999999999999956,2.0,3.0,2.0,3.0,2.0000000000000258,2.9999999999998677,2.0,2.999999999999998,2.0000000000000524,2.999999999999998,2.0,2.999999999999731,2.0000000000000204,3.0,2.0,3.0,2.0,2.999999999999869,2.0000000000000524,2.999999999999998,2.0,2.999999999999731,2.0,2.9999999999999982,2.0,86.60255045244762,106.06599965930195,86.60255443305651,106.06600747129924,86.60257272790835,106.06594450536937,86.60261007043326,106.06596664220048,86.60256991846003,106.06601083817202,86.60251126812021,106.06605239950612,86.60252027566017,106.06598075034444,86.60259101280616,106.06599907105759,86.60256520284221,106.06597421940252,86.60257498882078,106.06598483902887,86.60258117175987,106.06597850626063,86.60257221703684,106.06597517269579,86.60257273903393,106.06600072357362,86.60255405767539,106.06597375534662,86.60256934296459,106.06596027660167,86.60259348834224,106.06598724294983,86.60256638431304,106.06597523757073,86.60257261435468,106.06600050328808,86.60255444923789,106.06596589510059,86.60257650073991,106.06599053476317]"
-"bragg_as_tuple,80",0.046778286123,"[2.0,106.66757088749442,2.0,105.57729947300236,3.0,103.65402137816143,2.6440397966334315,107.98592838668976,3.0,104.8709863158532,2.5016953489723783,101.70490179587361,2.0,106.6400323861937,2.6756343089825743,102.87643001088634,2.0,105.49402224816971,2.4894682374727557,104.15987513997219,2.0,105.19206525072735,2.0,102.67129549331229,3.0,107.35548678882243,2.0,106.06602704283594,2.0,104.73708599677573,3.0,103.02092908425642,2.0,106.40134481056202,3.0,101.60387313114889,2.0,102.12797317754806,2.746321933618765,103.47729769177475,2.06880160474478,101.71240298693832,2.0,102.31903969506153,3.0,105.63449280773244,3.0,109.18919007786876,3.0,104.41417749736138,2.0,107.2685376632528,3.0,105.54264923377298,2.0284783071875836,106.12096882195686,2.0,105.52702869293209,2.0,107.67878362202327,3.0,103.39302542120915,3.0,105.9876221605065,3.0,106.25401313006192,2.0,106.78112462558478,2.175801565236613,105.57999850114906,3.0,106.33259748171652,3.0,104.5585256686101,3.0,103.31102896259058,2.0,100.7061169786474,3.0,104.90308290718994]"
+"bragg_as_tuple,80",0.044269947562,"[2.0,100.88538377600655,3.0,104.05255078972668,3.0,102.71576752436641,3.0,104.69833319792912,3.0,107.0349117729925,2.5105404616268534,107.71668657138052,3.0,102.94447942161379,3.0,105.23808092441166,2.0,102.6170647898369,3.0,103.92440313292103,3.0,105.32108633207582,3.0,106.26506678413494,3.0,102.49701216278135,3.0,103.71412983666362,3.0,105.0129901567737,3.0,102.67278105118274,2.0,105.92880679692507,2.0,102.59584356379756,3.0,103.68719340672614,3.0,107.68981189438946,3.0,105.53063212634463,3.0,103.96122890005357,3.0,106.67947841146885,2.0,106.8536676554173,2.428587245566405,102.60845071710577,2.0,102.89596860249945,2.6225783427514004,107.57129586325682,2.0,106.1277456717187,2.0,106.5274246728118,2.0,100.22110408502853,3.0,104.0064043449358,2.0,103.90536396712679,2.0,108.0388562301486,3.0,107.34699506053927,3.0,107.02554866444473,2.0,107.69763191464625,3.0,106.27055136036367,2.484191834701592,102.56399086704353,2.0,105.41253125761362,3.0,105.22393493081293]"
 "cf_photosic_realistic,20",0.0860257425449999,"[2.0,3.0,2.1076,2.0,3.0,2.5783,2.0,3.0,2.0,3.0,90.0231,78.9789,72.8369,99.9577,82.7487,62.7583,104.1682,139.9002,93.3356,75.6039]"
 "cf_photosic_reference,16",0.431072337632,"[109.72985831489979,98.18574120560555,107.42439496690479,87.99264365961669,112.531428339492,89.65278659791956,105.05127078132273,103.79772329843809,96.93018121427048,133.71819394988918,99.02177080593655,109.72456993535758,115.95956118008563,92.84831198907784,118.42356371437981,103.77850212331678]"
 "chirped,16",0.594587712739,"[109.72985831489979,98.18574120560555,107.42439496690479,87.99264365961669,112.531428339492,89.65278659791956,105.05127078132273,103.79772329843809,96.93018121427048,133.71819394988918,99.02177080593655,109.72456993535758,115.95956118008563,92.84831198907784,118.42356371437981,103.77850212331678]"

nevergrad/optimization/experimentalvariants.py

@@ -13,6 +13,7 @@
     EMNA,
     NGOpt10,
     NGOpt12,
+    BayesOptim,
 )
 from . import optimizerlib as opts
 from .optimizerlib import CMA, Chaining, PSO, BO
@@ -306,3 +307,12 @@
 DiscreteNoisyInfSplits = opts.NoisySplit(num_optims=float("inf"), discrete=True).set_name(
     "DiscreteNoisyInfSplits", register=True
 )
+
+# PCA-BO
+# Testing the influence of n_components on the performance of PCABO
+PCABO80 = BayesOptim(pca=True, n_components=0.80).set_name("PCABO80", register=True)
+
+# Testing the influence of the DoE size on the performance of PCABO
+PCABO95DoE20 = BayesOptim(pca=True, n_components=0.95, prop_doe_factor=0.20).set_name(
+    "PCABO95DoE20", register=True
+)

nevergrad/optimization/families.py

@@ -17,6 +17,7 @@
 from .optimizerlib import NoisySplit
 from .optimizerlib import ConfSplitOptimizer
 from .optimizerlib import ConfPortfolio
+from .optimizerlib import BayesOptim
 from .differentialevolution import DifferentialEvolution
 from .es import EvolutionStrategy
 from .recastlib import ScipyOptimizer
@@ -41,4 +42,5 @@
     "NoisySplit",
     "ConfSplitOptimizer",
     "ConfPortfolio",
+    "BayesOptim",
 ]

nevergrad/optimization/multiobjective/hypervolume.py

@@ -71,7 +71,7 @@ def pop(self, index: int) -> None:
 
 
 class VectorLinkedList:
-    """ Linked list structure with list of VectorNodes as elements."""
+    """Linked list structure with list of VectorNodes as elements."""
 
     def __init__(self, dimension: int) -> None:
         self.dimension = dimension
@@ -107,7 +107,7 @@ def chain_length(self, index: int) -> int:
         return length
 
     def append(self, node: VectorNode, index: int) -> None:
-        """ Append a node to the `index`-th position."""
+        """Append a node to the `index`-th position."""
         current_last = self.sentinel.prev[index]
         assert current_last is not None
         node.next[index] = self.sentinel
@@ -302,7 +302,7 @@ def recursive_hypervolume(self, dimension: int) -> float:
         return hypervolume
 
     def skip_dominated_points(self, node: VectorNode, dimension: int) -> None:
-        """ Implements Algorithm 2, _skipdom_, for skipping dominated points."""
+        """Implements Algorithm 2, _skipdom_, for skipping dominated points."""
         if node.dominated_flag >= dimension:
             node.area[dimension] = node.prev[dimension].area[dimension]
         else:

nevergrad/optimization/multiobjective/nsga2.py

@@ -118,7 +118,7 @@ def sort(self, candidates: tp.List[p.Parameter], in_place: bool = True) -> tp.Li
 
 
 class FastNonDominatedRanking:
-    """ Non-dominated ranking of NSGA-II proposed by Deb et al., see [Deb2002] """
+    """Non-dominated ranking of NSGA-II proposed by Deb et al., see [Deb2002]"""
 
     def compare(self, candidate1: p.Parameter, candidate2: p.Parameter) -> int:
         """Compare the domainance relation of two candidates.

nevergrad/optimization/optimizerlib.py

@@ -26,7 +26,6 @@
 from .base import addCompare  # pylint: disable=unused-import
 from .base import IntOrParameter
 
-
 # families of optimizers
 # pylint: disable=unused-wildcard-import,wildcard-import,too-many-lines,too-many-arguments,too-many-branches
 # pylint: disable=import-outside-toplevel,too-many-nested-blocks,too-many-instance-attributes,
@@ -45,6 +44,7 @@
 logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
 logger = logging.getLogger(__name__)
 
+
 # # # # # optimizers # # # # #
 
 
@@ -379,6 +379,7 @@ def __init__(
     mutation="portfolio",
 ).set_name("RecombiningPortfolioDiscreteOnePlusOne", register=True)
 
+
 # pylint: too-many-arguments,too-many-instance-attributes
 
 
@@ -1815,6 +1816,148 @@ def __init__(
 )
 
 
+class _BayesOptim(base.Optimizer):
+    def __init__(
+        self,
+        parametrization: IntOrParameter,
+        budget: tp.Optional[int] = None,
+        num_workers: int = 1,
+        *,
+        config: tp.Optional["BayesOptim"] = None,
+    ) -> None:
+        self._config = BayesOptim() if config is None else config
+        cfg = self._config
+        super().__init__(parametrization, budget=budget, num_workers=num_workers)
+        self._transform = transforms.ArctanBound(0, 1)
+
+        from bayes_optim import RealSpace
+        from bayes_optim.surrogate import GaussianProcess
+
+        # lb, ub = 1e-7 - np.pi / 2, np.pi / 2 - 1e-7
+        lb, ub = 1e-7, 1 - 1e-7
+        space = RealSpace([lb, ub]) * self.dimension
+
+        self._buffer: tp.List[float] = []
+        self._newX: tp.List[float] = []
+        self._losses: tp.List[float] = []
+
+        # Setting DoE size as a percentage of the total budget if prop_doe_factor is not None
+        init_budget = cfg.init_budget
+        if cfg.prop_doe_factor and budget is not None:
+            init_budget = round(cfg.prop_doe_factor * budget) if budget >= 10 else 5
+
+        if cfg.pca:
+            from bayes_optim.extension import PCABO as PcaBO
+
+            self._alg = PcaBO(
+                search_space=space,
+                obj_fun=None,  # Assuming that this is not used :-)
+                DoE_size=init_budget if init_budget is not None else 5,
+                max_FEs=budget,
+                verbose=True,
+                n_point=1,  # We start with a sequential procedure, maybe we'll extend in a second moment
+                n_components=cfg.n_components,
+                acquisition_optimization={"optimizer": "BFGS"},
+            )
+        else:
+            from bayes_optim import BO as BayesOptimBO
+
+            # hyperparameters of the GPR model
+            thetaL = 1e-10 * (ub - lb) * np.ones(self.dimension)
+            thetaU = 10 * (ub - lb) * np.ones(self.dimension)
+            model = GaussianProcess(thetaL=thetaL, thetaU=thetaU)  # create the GPR model
+
+            self._alg = BayesOptimBO(
+                search_space=space,
+                obj_fun=None,  # Assuming that this is not used :-)
+                model=model,
+                DoE_size=init_budget if init_budget is not None else 5,
+                max_FEs=budget,
+                verbose=True,
+            )
+
+    def _internal_ask_candidate(self) -> p.Parameter:
+        if not self._buffer:
+            candidate = self._alg.ask()
+            if not isinstance(candidate, list):
+                candidate = candidate.tolist()
+            self._buffer = candidate
+        x_probe = self._buffer.pop()
+        data = self._transform.backward(np.array(x_probe, copy=False))
+        candidate = self.parametrization.spawn_child().set_standardized_data(data)
+        candidate._meta["x_probe"] = x_probe
+        return candidate
+
+    def _internal_tell_candidate(self, candidate: p.Parameter, loss: tp.FloatLoss) -> None:
+        self._newX.append(candidate._meta["x_probe"])
+        self._losses.append(loss)
+        if not self._buffer:
+            if "x_probe" in candidate._meta:
+                self._alg.tell(self._newX, self._losses)
+            else:
+                data = candidate.get_standardized_data(reference=self.parametrization)
+                # Tell not asked:
+                self._alg.tell(self._transform.forward(data), loss)
+            self._newX = []
+            self._losses = []
+
+    def _internal_tell_not_asked(self, candidate: p.Parameter, loss: tp.FloatLoss) -> None:
+        raise errors.TellNotAskedNotSupportedError
+
+
+class BayesOptim(base.ConfiguredOptimizer):
+    """
+    Algorithms from bayes-optim package.
+
+    We use:
+    - BO
+    - PCA-BO: Principle Component Analysis (PCA) Bayesian Optimization for dimensionality reduction in BO
+
+    References
+
+    [RaponiWB+20]
+        Raponi, Elena, Hao Wang, Mariusz Bujny, Simonetta Boria, and Carola Doerr.
+        "High dimensional bayesian optimization assisted by principal component analysis."
+        In International Conference on Parallel Problem Solving from Nature, pp. 169-183.
+        Springer, Cham, 2020.
+
+
+    Parameters
+    ----------
+    init_budget: int or None
+        Number of initialization algorithm steps
+    pca: bool
+        whether to use the PCA transformation defining PCA-BO rather than BO
+    n_components: float or 0.95
+        Principal axes in feature space, representing the directions of maximum variance in the data.
+        It represents the percentage of explained variance
+    prop_doe_factor: float or None
+        Percentage of the initial budget used for DoE and eventually overwriting init_budget
+    """
+
+    no_parallelization = True
+    recast = True
+
+    # pylint: disable=unused-argument
+    def __init__(
+        self,
+        *,
+        init_budget: tp.Optional[int] = None,
+        pca: tp.Optional[bool] = False,
+        n_components: tp.Optional[float] = 0.95,
+        prop_doe_factor: tp.Optional[float] = None,
+    ) -> None:
+        super().__init__(_BayesOptim, locals(), as_config=True)
+        self.init_budget = init_budget
+        self.pca = pca
+        self.n_components = n_components
+        self.prop_doe_factor = prop_doe_factor
+
+
+PCABO = BayesOptim(pca=True).set_name("PCABO", register=True)
+BayesOptimBO = BayesOptim().set_name("BayesOptimBO", register=True)
+
+
 class _Chain(base.Optimizer):
     def __init__(
         self,

requirements/bench.txt

@@ -22,8 +22,10 @@ IOHexperimenter>=0.2.8.7
 cdt>=0.5.23
 koncept>=0.2.2
 tensorflow-estimator>=2.3.0
+scikit-learn>=0.24
 tensorflow>=2.5.0
 image-quality>=1.2.7
 keras>=2.4.3
 compiler_gym>=0.1.8 ; sys_platform == "linux"
 pymoo>=0.4.2.2
+bayes-optim>=0.2.5.5