Constraints seem not to work.

[Aiuan]

import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor

class NeuralNetwork(nn.Module):
    def __init__(self, feature_dims1=512, feature_dims2=256, feature_dims3=128):
        super(NeuralNetwork, self).__init__()
        self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(28*28, feature_dims1),
            nn.ReLU(),
            nn.Linear(feature_dims1, feature_dims2),
            nn.ReLU(),
            nn.Linear(feature_dims2, feature_dims3),
            nn.ReLU(),
            nn.Linear(feature_dims3, 10),
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits

def train_loop(dataloader, model, loss_fn, optimizer, device):
    size = len(dataloader.dataset)
    for batch, (X, y) in enumerate(dataloader):
        X = X.to(device)
        y = y.to(device)

        # Compute prediction and loss
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

def test_loop(dataloader, model, loss_fn, device):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    test_loss, correct = 0, 0

    with torch.no_grad():
        for X, y in dataloader:
            X = X.to(device)
            y = y.to(device)

            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()

    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")
    
    return correct

training_data = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor()
)

test_data = datasets.FashionMNIST(
    root="data",
    train=False,
    download=True,
    transform=ToTensor()
)

from openbox import sp

def get_configspace():
    space = sp.Space()
    learning_rate = sp.Real("learning_rate", 1e-3, 0.3, default_value=0.1, log=True)
    batch_size = sp.Int("batch_size", 32, 64)
    feature_dims1 = sp.Int("feature_dims1", 256, 512)
    feature_dims2 = sp.Int("feature_dims2", 256, 512)
    feature_dims3 = sp.Int("feature_dims3", 256, 512)
    space.add_variables([learning_rate, batch_size, feature_dims1, feature_dims2, feature_dims3])    
    return space

def objective_function(config: sp.Configuration):
    params = config.get_dictionary()
    params['epochs'] = 10
    
    train_dataloader = DataLoader(training_data, batch_size=params['batch_size'])
    test_dataloader = DataLoader(test_data, batch_size=params['batch_size'])
    
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")    
    model = NeuralNetwork(
        feature_dims1=params['feature_dims1'], 
        feature_dims2=params['feature_dims2'],
        feature_dims3=params['feature_dims3']
    )
    model = model.to(device)

    loss_fn = nn.CrossEntropyLoss()

    optimizer = torch.optim.SGD(model.parameters(), lr=params['learning_rate'])
    
    for epoch in range(params['epochs']):
        print(f"Epoch {epoch+1}\n-------------------------------")
        train_loop(train_dataloader, model, loss_fn, optimizer, device)
        correct = test_loop(test_dataloader, model, loss_fn, device)
    
    result = dict()
    result['objs'] = [1-correct, ]
    result['constraints'] = [
        params['feature_dims2'] - params['feature_dims1'], 
        params['feature_dims3'] - params['feature_dims2'], 
    ]
    
    return result

from openbox import Optimizer

# Run
opt = Optimizer(
    objective_function,
    get_configspace(),
    num_objs=1,
    num_constraints=2,
    max_runs=10,
    surrogate_type='prf',
    time_limit_per_trial=180,
    task_id='hpo',
)
history = opt.run()

history = opt.get_history()
print(history)

history.plot_convergence()

history.visualize_jupyter()

print(history.get_importance())

I add 2 constrains in results from objective_function, hope "feature_dims1 > feature_dims2 > feature_dims3", but it seems not to work as expect.

[yuixzero]

我也有同样的问题，目前看上去有两个地方会违反约束：

1. 初始点的采集并不会考虑约束条件
2. 优化中会存在一个随机探索的机制，也没有考虑约束，

   open-box/openbox/core/generic_advisor.py

   Lines 337 to 339 in 4fdde46

   	if (not return_list) and self.rng.random() < self.rand_prob:
   	self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob)
   	return self.sample_random_configs(1, history_container)[0]

   
   考虑约束的地方只有采集函数这部分，最简单的解决方法可以自定义符合约束的初始点，然后将rand_prob设置为0（这也许会影响算法的性能）

[jhj0411jhj]

In OpenBox, constraints are considered to be black-box, which means only after the configuration is evaluated, you can observe whether the constraints are violated. In the optimization process, OpenBox tries to find feasible solutions, but does not guarantee the black-box constraints are 100% satisfied.

However, I see that the constraints you set are between input hyperparameters (feature_dims1 > feature_dims2 > feature_dims3). The optimizer should be able to filter invalid suggestions before execution. Unfortunately, the ConfigSpace library that OpenBox depends on does not support such features. So I implement a ConditionedSpace to support complex user-defined conditions (constraints) between hyperparameters (be455c2). Please install the latest version of OpenBox to use this feature (pip install openbox>=0.7.18).

Here is an example:
If sample_condition returns True, the configuration is valid and will be sampled.

from openbox import space as sp

def sample_condition(config):
    # require x1 <= x2 and x1 * x2 < 100
    if config['x1'] > config['x2']:
        return False
    if config['x1'] * config['x2'] >= 100:
        return False
    return True
    # return config['x1'] <= config['x2'] and config['x1'] * config['x2'] < 100

cs = sp.ConditionedSpace()
cs.add_variables([...])
cs.set_sample_condition(sample_condition)  # set the sample condition after all variables are added

You are welcomed to report any bugs encountered.

[jhj0411jhj]

By the way, in the latest version (0.6.0) of ConfigSpace, ForbiddenRelation is added to support direct constraints between two hyperparameters (i.e. a<=b, a==b, a>=b). Due to the limitations of our time, compatibility between this version and OpenBox has not been tested yet. If you want to use this feature, you need to manually install it by running pip install ConfigSpace>=0.6.0 and ignoring the incompatible message.

However, it does not support more complex constraints like a * b < 10.

Here I try to write an example using ForbiddenRelation:

from ConfigSpace import ForbiddenGreaterThanRelation, ForbiddenEqualsRelation, ForbiddenLessThanRelation
from openbox import space as sp

space = sp.Space()
space.add_variables([...])
# we want x1 <= x2, then x1 > x2 is forbidden
fr = ForbiddenGreaterThanRelation(space['x1'], space['x2'])
space.add_forbidden_clause(fr)

Refer to their docs and issues for more details:

• Docs: https://automl.github.io/ConfigSpace/main/guide.html#rd-example-forbidden-clauses
• PR: Add: Add forbidden relations between two hyperparameters automl/ConfigSpace#245

[jhj0411jhj]

我也有同样的问题，目前看上去有两个地方会违反约束：

1. 初始点的采集并不会考虑约束条件

2. 优化中会存在一个随机探索的机制，也没有考虑约束，

   open-box/openbox/core/generic_advisor.py

   Lines 337 to 339 in 4fdde46

   	if (not return_list) and self.rng.random() < self.rand_prob:
   	self.logger.info('Sample random config. rand_prob=%f.' % self.rand_prob)
   	return self.sample_random_configs(1, history_container)[0]

   考虑约束的地方只有采集函数这部分，最简单的解决方法可以自定义符合约束的初始点，然后将rand_prob设置为0（这也许会影响算法的性能）

@yuixzero By using ConditionedSpace, constraints between hyperparameters will be handled in initialization and random sampling.