MNIST 数据集过度拟合

Question

 class NN(nn.Module):
def __init__(self):
    super().__init__()
    self.layers = nn.Sequential(
    nn.Flatten(),
    nn.Linear(784,4096),
    nn.ReLU(),
    nn.Linear(4096,2048),
    nn.ReLU(),
    nn.Linear(2048,1024),
    nn.ReLU(),
    nn.Linear(1024,512),
    nn.ReLU(),
    nn.Linear(512,256),
    nn.ReLU(),
    nn.Linear(256,128),
    nn.ReLU(),
    nn.Linear(128,64),
    nn.ReLU(),
    nn.Linear(64,32),
    nn.ReLU(),
    nn.Linear(32,16),
    nn.ReLU(),
    nn.Linear(16,10))
def forward(self,x):
    return self.layers(x)
def accuracy_and_loss(model, loss_function, dataloader):
    total_correct = 0
    total_loss = 0
    total_examples = 0
    n_batches = 0
    with torch.no_grad():
        for data in testloader:
            images, labels = data
            outputs = model(images)
            batch_loss = loss_function(outputs,labels)
            n_batches += 1
            total_loss += batch_loss.item()
            _, predicted = torch.max(outputs, dim=1)
            total_examples += labels.size(0)
            total_correct += (predicted == labels).sum().item()
    accuracy = total_correct / total_examples
    mean_loss = total_loss / n_batches
    return (accuracy, mean_loss)
def define_and_train(model,dataset_training, dataset_test):
trainloader = torch.utils.data.DataLoader( small_trainset, batch_size=500, shuffle=True)
testloader = torch.utils.data.DataLoader( dataset_test, batch_size=500, shuffle=True)
values = [1e-8,1e-7,1e-6,1e-5]
model = NN()
for params in values:
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay = 1e-7)
    train_acc = []
    val_acc = []
    train_loss = []
    val_loss = []
    for epoch in range(100):
    
        total_loss = 0
        total_correct = 0
        total_examples = 0
        n_mini_batches = 0
    
        for i,mini_batch in enumerate(trainloader,0):
        
            images,labels = mini_batch
            optimizer.zero_grad()
            outputs = model(images)
            loss = loss_function(outputs,labels)
            loss.backward()
            optimizer.step()
            n_mini_batches += 1
            total_loss += loss.item()
            _, predicted = torch.max(outputs, dim=1)
            total_examples += labels.size(0)
            total_correct += (predicted == labels).sum().item()
        
        epoch_training_accuracy = total_correct / total_examples
        epoch_training_loss = total_loss / n_mini_batches
        epoch_val_accuracy, epoch_val_loss = accuracy_and_loss( model, loss_function, testloader )

        print('Params %f Epoch %d loss: %.3f acc: %.3f val_loss: %.3f val_acc: %.3f'
              %(params, epoch+1, epoch_training_loss, epoch_training_accuracy, epoch_val_loss, epoch_val_accuracy))
    
        train_loss.append( epoch_training_loss )
        train_acc.append( epoch_training_accuracy )
        val_loss.append( epoch_val_loss )
        val_acc.append( epoch_val_accuracy )

    history = { 'train_loss': train_loss, 
                'train_acc': train_acc, 
                'val_loss': val_loss,
                'val_acc': val_acc }
    return ( history, model ) 
history1, net1 = define_and_train(model,dataset_training,dataset_test)

我正在尝试过度拟合训练数据，以便稍后我可以应用正则化，然后减少过度拟合，这将使我更好地理解该过程

Answer 1

尽管我不会尝试提供严格的定义，但术语“过度拟合”通常意味着训练损失持续减少，而验证损失则停滞在高于训练损失的位置，或者随着更多迭代而继续增加。

因此，仅根据您的代码很难知道您的网络是否过度拟合。由于密集的、完全连接的网络在没有 dropout 层或其他正则化器的情况下往往很容易过度拟合，因此我的预感是您的网络确实根据您的意图过度拟合。但是，我们必须查看您的张量板日志或损失图来确定模型是否过度拟合。

如果您想让网络与数据集过度拟合，我建议您构建一个具有更多隐藏层的更大模型。当数据集对于模型来说“太简单”并且模型开始记住训练集本身而不学习可应用于验证集的通用模式时，就会发生过度拟合。