Model Building with BoFire

This notebooks shows how to setup and analyze models trained with BoFire. It is still WIP.

Imports

from pydantic import TypeAdapter

import bofire.surrogates.api as surrogates
from bofire.benchmarks.multi import CrossCoupling
from bofire.benchmarks.single import Himmelblau
from bofire.data_models.domain.api import Inputs, Outputs
from bofire.data_models.features.api import CategoricalInput
from bofire.data_models.surrogates.api import (
    AnySurrogate,
    EmpiricalSurrogate,
    RandomForestSurrogate,
    RegressionMLPEnsemble,
    SingleTaskGPSurrogate,
)
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/torch/jit/_script.py:1491: FutureWarning: `torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`.
  warnings.warn(
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/torch/jit/_script.py:1491: FutureWarning: `torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`.
  warnings.warn(

Problem Setup

For didactic purposes, we sample data from a Himmelblau benchmark function and use them to train a SingleTaskGP.

benchmark = Himmelblau()
samples = benchmark.domain.inputs.sample(n=50)
experiments = benchmark.f(samples, return_complete=True)

experiments.head(10)
x_1 x_2 y valid_y
0 0.422517 2.316703 73.796209 1
1 -1.892609 0.897596 107.914507 1
2 -5.182837 2.434295 373.897665 1
3 -0.471844 -0.560956 179.782605 1
4 -5.214499 4.714462 537.271496 1
5 1.599849 2.237764 38.627847 1
6 2.676756 -4.794835 422.938026 1
7 -1.367161 -2.202152 140.811546 1
8 -0.765920 -0.794382 176.520099 1
9 1.353204 2.544395 44.567432 1

Model Fitting

input_features = benchmark.domain.inputs
output_features = benchmark.domain.outputs
input_features.model_dump_json()
'{"type":"Inputs","features":[{"type":"ContinuousInput","key":"x_1","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"x_2","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false}]}'
output_features.model_dump_json()
'{"type":"Outputs","features":[{"type":"ContinuousOutput","key":"y","context":null,"unit":null,"objective":{"type":"MinimizeObjective","w":1.0,"bounds":[0.0,1.0]}}]}'

Single Task GP

Generate the json spec

# we setup the data model, here a Single Task GP
surrogate_data = SingleTaskGPSurrogate(inputs=input_features, outputs=output_features)

# we generate the json spec
jspec = surrogate_data.model_dump_json()

jspec
'{"optimizer":{"type":"LBFGSB","maxiter":null,"ftol":null,"gtol":null,"max_attempts":50},"type":"SingleTaskGPSurrogate","inputs":{"type":"Inputs","features":[{"type":"ContinuousInput","key":"x_1","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"x_2","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false}]},"outputs":{"type":"Outputs","features":[{"type":"ContinuousOutput","key":"y","context":null,"unit":null,"objective":{"type":"MinimizeObjective","w":1.0,"bounds":[0.0,1.0]}}]},"dump":null,"categorical_encodings":{},"engineered_features":{"type":"EngineeredFeatures","features":[]},"scaler":{"type":"Normalize","features":[]},"output_scaler":"STANDARDIZE","kernel":{"type":"RBFKernel","features":null,"lengthscale_prior":{"type":"DimensionalityScaledLogNormalPrior","loc":1.4142135623730951,"loc_scaling":0.5,"scale":1.7320508075688772,"scale_scaling":0.0},"lengthscale_constraint":{"type":"GreaterThan","initial_value":null,"lower_bound":0.025},"ard":true},"mean":{"type":"ConstantMean","prior":null,"constraint":null},"likelihood":{"type":"GaussianLikelihood","noise_prior":{"type":"LogNormalPrior","loc":-4.0,"scale":1.0},"noise_constraint":{"type":"GreaterThan","initial_value":0.006737946999085467,"lower_bound":0.0001}}}'

Load it from the spec

surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)

Map it

surrogate = surrogates.map(surrogate_data)

Fit it. This is not 100% finished. In the future we will call here hyperfit which will return the CV results etc. This has to be finished. So ignore this for now and just call fit.

surrogate.fit(experiments=experiments)
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/bofire/utils/torch_tools.py:51: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/utils/tensor_numpy.cpp:213.)
  return torch.from_numpy(np.ascontiguousarray(df.to_numpy())).to(**tkwargs)

Dump it.

# dump it
dump = surrogate.dumps()

Make predictions.

# predict with it
df_predictions = surrogate.predict(experiments)
# transform to spec
predictions = surrogate.to_predictions(predictions=df_predictions)

Load again from spec and dump and make predictions.

surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
surrogate = surrogates.map(surrogate_data)
surrogate.loads(dump)

# predict with it
df_predictions2 = surrogate.predict(experiments)
# transform to spec
predictions2 = surrogate.to_predictions(predictions=df_predictions2)

# check for equality
predictions == predictions2
True

Random Forest

Generate the json spec

# we setup the data model, here a Single Task GP
surrogate_data = RandomForestSurrogate(
    inputs=input_features,
    outputs=output_features,
    random_state=42,
)

# we generate the json spec
jspec = surrogate_data.model_dump_json()

jspec
'{"type":"RandomForestSurrogate","inputs":{"type":"Inputs","features":[{"type":"ContinuousInput","key":"x_1","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"x_2","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false}]},"outputs":{"type":"Outputs","features":[{"type":"ContinuousOutput","key":"y","context":null,"unit":null,"objective":{"type":"MinimizeObjective","w":1.0,"bounds":[0.0,1.0]}}]},"dump":null,"categorical_encodings":{},"engineered_features":{"type":"EngineeredFeatures","features":[]},"scaler":{"type":"Normalize","features":[]},"output_scaler":"STANDARDIZE","n_estimators":100,"criterion":"squared_error","max_depth":null,"min_samples_split":2,"min_samples_leaf":1,"min_weight_fraction_leaf":0.0,"max_features":1.0,"max_leaf_nodes":null,"min_impurity_decrease":0.0,"bootstrap":true,"oob_score":false,"random_state":42,"ccp_alpha":0.0,"max_samples":null}'
# Load it from the spec
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
# Map it
surrogate = surrogates.map(surrogate_data)
# Fit it
surrogate.fit(experiments=experiments)
# dump it
dump = surrogate.dumps()
# predict with it
df_predictions = surrogate.predict(experiments)
# transform to spec
predictions = surrogate.to_predictions(predictions=df_predictions)
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/botorch/models/ensemble.py:82: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  self.eval()
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/torch/nn/modules/module.py:2932: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  return self.train(False)
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/botorch/models/ensemble.py:82: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  self.eval()
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/torch/nn/modules/module.py:2932: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  return self.train(False)
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
surrogate = surrogates.map(surrogate_data)
surrogate.loads(dump)

# predict with it
df_predictions2 = surrogate.predict(experiments)
# transform to spec
predictions2 = surrogate.to_predictions(predictions=df_predictions2)

# check for equality
predictions == predictions2
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/botorch/models/ensemble.py:82: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  self.eval()
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/torch/nn/modules/module.py:2932: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  return self.train(False)
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/botorch/models/ensemble.py:82: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  self.eval()
/opt/hostedtoolcache/Python/3.12.15/x64/lib/python3.12/site-packages/torch/nn/modules/module.py:2932: RuntimeWarning: Could not update `train_inputs` with transformed inputs since _RandomForest does not have a `train_inputs` attribute. Make sure that the `input_transform` is applied to both the train inputs and test inputs.
  return self.train(False)
True

MLP Ensemble

Generate the json spec

# we setup the data model, here a Single Task GP
surrogate_data = RegressionMLPEnsemble(
    inputs=input_features,
    outputs=output_features,
    n_estimators=2,
)

# we generate the json spec
jspec = surrogate_data.model_dump_json()

jspec
'{"type":"RegressionMLPEnsemble","inputs":{"type":"Inputs","features":[{"type":"ContinuousInput","key":"x_1","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"x_2","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false}]},"outputs":{"type":"Outputs","features":[{"type":"ContinuousOutput","key":"y","context":null,"unit":null,"objective":{"type":"MinimizeObjective","w":1.0,"bounds":[0.0,1.0]}}]},"dump":null,"categorical_encodings":{},"engineered_features":{"type":"EngineeredFeatures","features":[]},"scaler":null,"output_scaler":"IDENTITY","n_estimators":2,"hidden_layer_sizes":[100],"activation":"relu","dropout":0.0,"batch_size":10,"n_epochs":200,"lr":0.0001,"weight_decay":0.0,"subsample_fraction":1.0,"shuffle":true,"final_activation":"identity"}'
# Load it from the spec
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
# Map it
surrogate = surrogates.map(surrogate_data)
# Fit it
surrogate.fit(experiments=experiments)
# dump it
dump = surrogate.dumps()
# predict with it
df_predictions = surrogate.predict(experiments)
# transform to spec
predictions = surrogate.to_predictions(predictions=df_predictions)
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
surrogate = surrogates.map(surrogate_data)
surrogate.loads(dump)

# predict with it
df_predictions2 = surrogate.predict(experiments)
# transform to spec
predictions2 = surrogate.to_predictions(predictions=df_predictions2)

# check for equality
predictions == predictions2
True

Empirical Surrogate

The empirical model is special as it has per default no fit and you need cloudpickle. There can be empirical models which implement a fit, but for this they also have to inherit from Trainable. The current example is the default without any fit functionality.

from botorch.models.deterministic import DeterministicModel
from torch import Tensor


class HimmelblauModel(DeterministicModel):
    def __init__(self):
        super().__init__()
        self._num_outputs = 1

    def forward(self, X: Tensor) -> Tensor:
        return (
            (X[..., 0] ** 2 + X[..., 1] - 11.0) ** 2
            + (X[..., 0] + X[..., 1] ** 2 - 7.0) ** 2
        ).unsqueeze(-1)
surrogate_data = EmpiricalSurrogate(

    inputs=input_features,
    outputs=output_features,
)

# we generate the json spec
jspec = surrogate_data.model_dump_json()

jspec
'{"type":"EmpiricalSurrogate","inputs":{"type":"Inputs","features":[{"type":"ContinuousInput","key":"x_1","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"x_2","context":null,"unit":null,"descriptors":null,"bounds":[-6.0,6.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false}]},"outputs":{"type":"Outputs","features":[{"type":"ContinuousOutput","key":"y","context":null,"unit":null,"objective":{"type":"MinimizeObjective","w":1.0,"bounds":[0.0,1.0]}}]},"dump":null,"categorical_encodings":{},"engineered_features":{"type":"EngineeredFeatures","features":[]}}'
# Load it from the spec
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
# Map it
surrogate = surrogates.map(surrogate_data)
# attach the actual model to it
surrogate.model = HimmelblauModel()
# dump it
dump = surrogate.dumps()
# predict with it
df_predictions = surrogate.predict(experiments)
# transform to spec
predictions = surrogate.to_predictions(predictions=df_predictions)
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
surrogate = surrogates.map(surrogate_data)
surrogate.loads(dump)

# predict with it
df_predictions2 = surrogate.predict(experiments)
# transform to spec
predictions2 = surrogate.to_predictions(predictions=df_predictions2)

# check for equality
predictions == predictions2
True

Mixed GP

Generate data for a mixed problem.

benchmark = CrossCoupling()
samples = benchmark.domain.inputs.sample(n=50)
experiments = benchmark.f(samples, return_complete=True)

experiments.head(10)
base_eq t_res temperature base catalyst yield cost valid_cost valid_yield
0 1.448644 765.649446 90.101176 TEA tBuXPhos 0.065126 0.248767 1 1
1 1.469334 1031.567505 34.113372 DBU tBuBrettPhos 0.466260 0.279875 1 1
2 1.739449 259.157114 39.391637 TMG AlPhos 0.009498 0.419074 1 1
3 2.006770 758.919464 65.275613 TEA AlPhos 0.088889 0.419741 1 1
4 2.318010 1322.219114 40.732863 DBU tBuBrettPhos 0.524253 0.280802 1 1
5 1.243045 1375.734138 61.286102 BTMG AlPhos 0.947326 0.473307 1 1
6 2.268177 935.276566 44.022700 DBU AlPhos 0.734168 0.421488 1 1
7 1.558656 803.220151 72.257569 TEA tBuXPhos 0.065921 0.248807 1 1
8 1.321682 922.942507 43.693965 BTMG tBuXPhos 0.888931 0.305970 1 1
9 1.236784 1420.104332 85.114795 TMG AlPhos 0.857812 0.419056 1 1

Both categorical inputs of this benchmark carry descriptor data, which a GP uses as continuous columns. To treat the catalyst as a plain categorical instead, we drop its descriptors; the mixed GP then compares catalysts with a Hamming kernel.

catalyst = benchmark.domain.inputs.get_by_key("catalyst")
inputs = Inputs(
    features=[
        feat
        for feat in benchmark.domain.inputs.get()
        if feat.key != "catalyst"
    ]
    + [CategoricalInput(key="catalyst", categories=catalyst.categories)]
)

# we setup the data model, here a mixed GP
surrogate_data = SingleTaskGPSurrogate.mixed(
    inputs, Outputs(features=[benchmark.domain.outputs.features[0]])
)

# we generate the json spec
jspec = surrogate_data.model_dump_json()

jspec
'{"optimizer":{"type":"LBFGSB","maxiter":null,"ftol":null,"gtol":null,"max_attempts":50},"type":"SingleTaskGPSurrogate","inputs":{"type":"Inputs","features":[{"type":"ContinuousInput","key":"base_eq","context":null,"unit":null,"descriptors":null,"bounds":[1.0,2.5],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"t_res","context":null,"unit":null,"descriptors":null,"bounds":[60.0,1800.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"ContinuousInput","key":"temperature","context":null,"unit":null,"descriptors":null,"bounds":[30.0,100.0],"local_relative_bounds":null,"stepsize":null,"allow_zero":false},{"type":"CategoricalInput","key":"base","context":null,"categories":["TEA","TMG","BTMG","DBU"],"descriptors":{"columns":{"area":[162.2992,165.5447,227.3523,192.4693],"M2":[25.8165,81.4847,30.554,59.8367]},"structure":null},"allowed":[true,true,true,true]},{"type":"CategoricalInput","key":"catalyst","context":null,"categories":["tBuXPhos","tBuBrettPhos","AlPhos"],"descriptors":null,"allowed":[true,true,true]}]},"outputs":{"type":"Outputs","features":[{"type":"ContinuousOutput","key":"yield","context":null,"unit":null,"objective":{"type":"MaximizeObjective","w":1.0,"bounds":[0.0,1.0]}}]},"dump":null,"categorical_encodings":{"catalyst":{"type":"OrdinalEncoding"},"base":{"columns":null,"generators":[],"filter_descriptors":false,"correlation_cutoff":0.95,"type":"DescriptorEncoding"}},"engineered_features":{"type":"EngineeredFeatures","features":[]},"scaler":{"type":"Normalize","features":[]},"output_scaler":"STANDARDIZE","kernel":{"type":"AdditiveKernel","kernels":[{"type":"ScaleKernel","base_kernel":{"type":"AdditiveKernel","kernels":[{"type":"RBFKernel","features":["base_eq","t_res","temperature","base"],"lengthscale_prior":{"type":"DimensionalityScaledLogNormalPrior","loc":1.4142135623730951,"loc_scaling":0.5,"scale":1.7320508075688772,"scale_scaling":0.0},"lengthscale_constraint":{"type":"GreaterThan","initial_value":null,"lower_bound":0.025},"ard":true},{"type":"ScaleKernel","base_kernel":{"type":"HammingDistanceKernel","features":["catalyst"],"lengthscale_prior":null,"lengthscale_constraint":null,"ard":true},"outputscale_prior":null,"outputscale_constraint":null}]},"outputscale_prior":null,"outputscale_constraint":null},{"type":"ScaleKernel","base_kernel":{"type":"MultiplicativeKernel","kernels":[{"type":"RBFKernel","features":["base_eq","t_res","temperature","base"],"lengthscale_prior":{"type":"DimensionalityScaledLogNormalPrior","loc":1.4142135623730951,"loc_scaling":0.5,"scale":1.7320508075688772,"scale_scaling":0.0},"lengthscale_constraint":{"type":"GreaterThan","initial_value":null,"lower_bound":0.025},"ard":true},{"type":"HammingDistanceKernel","features":["catalyst"],"lengthscale_prior":null,"lengthscale_constraint":null,"ard":true}]},"outputscale_prior":null,"outputscale_constraint":null}]},"mean":{"type":"ConstantMean","prior":null,"constraint":null},"likelihood":{"type":"GaussianLikelihood","noise_prior":{"type":"LogNormalPrior","loc":-4.0,"scale":1.0},"noise_constraint":{"type":"GreaterThan","initial_value":0.006737946999085467,"lower_bound":0.0001}}}'
# Load it from the spec
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
# Map it
surrogate = surrogates.map(surrogate_data)
# Fit it
surrogate.fit(experiments=experiments)
# dump it
dump = surrogate.dumps()
# predict with it
df_predictions = surrogate.predict(experiments)
# transform to spec
predictions = surrogate.to_predictions(predictions=df_predictions)
surrogate_data = TypeAdapter(AnySurrogate).validate_json(jspec)
surrogate = surrogates.map(surrogate_data)
surrogate.loads(dump)

# predict with it
df_predictions2 = surrogate.predict(experiments)
# transform to spec
predictions2 = surrogate.to_predictions(predictions=df_predictions2)

# check for equality
predictions == predictions2
True