是否有人为Pytorch LSTM实施了Optuna超参数优化？

Question

我正在尝试针对Pytorch LSTM实现Optuna超参数优化。但是我不知道如何正确定义我的模型。 当我只使用nn.lin.linear Erverything工作正常时，但是当我使用nn.lstmcell时，我会收到以下错误：

AttributeError: 'tuple' object has no attribute 'dim'

由于LSTM返回Tupel而不是tupel，因此会增加错误张量。但是我不知道如何修复它，也找不到具有Optuna优化的Pytorch LSTM的示例。

这里的模型定义：

def build_model_custom(trail):
    
    # Suggest the number of layers of neural network model
    n_layers = trail.suggest_int("n_layers", 1, 3)
    layers = []

    in_features = 20
    
    for i in range(n_layers):
        
        # Suggest the number of units in each layer
        out_features = trail.suggest_int("n_units_l{}".format(i), 4, 18)
        
        layers.append(nn.LSTMCell(in_features, out_features))

        in_features = out_features
        
    layers.append(nn.Linear(in_features, 2))

    return nn.Sequential(*layers)

Answer 1

我以前已经实施了Optuna优化LSTM的示例，希望它能为您提供帮助：

def get_best_parameters(args, Dtr, Val):
    def objective(trial):
        model = TransformerModel(args).to(args.device)
        loss_function = nn.MSELoss().to(args.device)
        optimizer = trial.suggest_categorical('optimizer',
                                              [torch.optim.SGD,
                                               torch.optim.RMSprop,
                                               torch.optim.Adam])(
            model.parameters(), lr=trial.suggest_loguniform('lr', 5e-4, 1e-2))
        print('training...')
        epochs = 10
        val_loss = 0
        for epoch in range(epochs):
            train_loss = []
            for batch_idx, (seq, target) in enumerate(Dtr, 0):
                seq, target = seq.to(args.device), target.to(args.device)
                optimizer.zero_grad()
                y_pred = model(seq)
                loss = loss_function(y_pred, target)
                train_loss.append(loss.item())
                loss.backward()
                optimizer.step()
            # validation
            val_loss = get_val_loss(args, model, Val)

            print('epoch {:03d} train_loss {:.8f} val_loss {:.8f}'.format(epoch, np.mean(train_loss), val_loss))
            model.train()

        return val_loss

    sampler = optuna.samplers.TPESampler()
    study = optuna.create_study(sampler=sampler, direction='minimize')
    study.optimize(func=objective, n_trials=5)
    pruned_trials = study.get_trials(deepcopy=False,
                                     states=tuple([TrialState.PRUNED]))
    complete_trials = study.get_trials(deepcopy=False,
                                       states=tuple([TrialState.COMPLETE]))
    best_trial = study.best_trial
    print('val_loss = ', best_trial.value)
    for key, value in best_trial.params.items():
        print("{}: {}".format(key, value))

Answer 2

我自己实现了解决方案。我不确定它是最柔软的，但它起作用。
欢迎改进建议。

def train_and_evaluate(param, model, trail):
    
    # Load Data
    train_dataloader = torch.utils.data.DataLoader(Train_Dataset, batch_size=batch_size)
    Test_dataloader = torch.utils.data.DataLoader(Test_Dataset, batch_size=batch_size)

    criterion = nn.MSELoss()
    optimizer = getattr(optim, param['optimizer'])(model.parameters(), lr= param['learning_rate'])
    acc = nn.L1Loss()

    # Training Loop
    for epoch_num in range(EPOCHS):

            # Training
            total_loss_train = 0
            for train_input, train_target in train_dataloader:

                output = model.forward(train_input.float())
                batch_loss = criterion(output, train_target.float())
                total_loss_train += batch_loss.item()

                model.zero_grad()
                batch_loss.backward()
                optimizer.step()
            
            # Evaluation
            total_loss_val = 0
            total_mae = 0
            with torch.no_grad():

                for test_input, test_target in Test_dataloader:

                    output = model(test_input.float())

                    batch_loss = criterion(output, test_target)
                    total_loss_val += batch_loss.item()
                    batch_mae = acc(output, test_target)
                    total_mae  += batch_mae.item()
            
            accuracy = total_mae/len(Test_Dataset)

            # Add prune mechanism
            trail.report(accuracy, epoch_num)

            if trail.should_prune():
                raise optuna.exceptions.TrialPruned()

    return accuracy