Loss Backward Optimizer 损失函数 反向传播 优化器

官方文档 中文 https://www.pytorchtutorial.com/docs/

官方文档 https://pytorch.org/docs/stable/index.html

1 Loss Function

1. Loss Functions

https://pytorch.org/docs/1.8.1/nn.html#loss-functions

2. Some Loss Functions

L1Loss

  • examples
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import torch
from torch.nn import L1Loss

input = torch.tensor([1, 2, 3], dtype=torch.float)
target = torch.tensor([1, 2, 5], dtype=torch.float)

input = torch.reshape(input, (1, 1, 1, 3))
target = torch.reshape(target, (1, 1, 1, 3))

loss_sum = L1Loss(reduction='sum')  # tensor(2.)
loss_mean = L1Loss(reduction='mean')  # tensor(0.6667)
res_sum = loss_sum(input, target)
res_mean = loss_mean(input, target)
print(res_sum)
print(res_mean)
  • input & target
1
2
input 	-> (N, *)
target 	-> (N, *)

MSELoss

  • example
1
2
3
4
5
6
7
8
9
10
11
12
import torch
from torch.nn import MSELoss

input = torch.tensor([1, 2, 3], dtype=torch.float)
target = torch.tensor([1, 2, 5], dtype=torch.float)

input = torch.reshape(input, (1, 1, 1, 3))
target = torch.reshape(target, (1, 1, 1, 3))

loss = MSELoss()
res = loss(input, target)  # tensor(1.3333)
print(res)
  • input & target
1
2
input 	-> (N, *)
target 	-> (N, *)

CrossEntropyLoss

  • example
1
2
3
4
5
6
7
8
9
10
11
import torch
from torch.nn import CrossEntropyLoss

input_predict = torch.tensor([0.1, 0.2, 0.3], dtype=torch.float)  # 模型返回了对3个种类的概率预测
target = torch.tensor([1])   # 实际属于第1+1种，返回1

input_predict = torch.reshape(input_predict, (1, 3))

loss = CrossEntropyLoss()
res = loss(input_predict, target)  # tensor(1.1019)
print(res)
  • input & target
1
2
3
4
5
6
input  -> (N, C)    最后得到的各个种类的概率
target -> (N)	      实际属于哪一种

output -> 返回标量: If reduction is 'none', then the same size as the target: (N) 

# C = number of classes

3. Loss of CIFAR10

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import torch
import torchvision
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter


class Cifar(nn.Module):
    def __init__(self):
        super(Cifar, self).__init__()
        self.model1 = Sequential(
            Conv2d(in_channels=3, out_channels=32, kernel_size=5, stride=1, padding=2),
            MaxPool2d(kernel_size=2),
            Conv2d(in_channels=32, out_channels=32, kernel_size=5, stride=1, padding=2),
            MaxPool2d(kernel_size=2),
            Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2),
            MaxPool2d(kernel_size=2),
            Flatten(),
            Linear(in_features=64 * 4 * 4, out_features=64),
            Linear(in_features=64, out_features=10),
        )

    def forward(self, x):
        x = self.model1(x)
        return x


'''测试'''
dataset = torchvision.datasets.CIFAR10(
    root="./dataset",
    train=False,
    download=False,
    transform=torchvision.transforms.ToTensor()
)

dataloader = DataLoader(dataset, batch_size=64)

my_net = Cifar()
loss = nn.CrossEntropyLoss()
for data in dataloader:
    imgs, targets = data
    outputs = my_net(imgs)
    # print(outputs)  # [p0, p1, ..., p9]  每张图片的类的预测的概率
    # print(targets)  # [t0, t1, ..., t63] 真实每张图片对应的类标号

    res_loss = loss(outputs, targets)
    print(res_loss)   # tensor(2.2994, grad_fn=<NllLossBackward0>) 误差

2 Backward

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
my_net = Cifar()
loss = nn.CrossEntropyLoss()

for data in dataloader:
    ''' network '''
    imgs, targets = data
    outputs = my_net(imgs)
    # print(outputs)  # [p0, p1, ..., p9]  每张图片的类的预测的概率
    # print(targets)  # [t0, t1, ..., t63] 真实每张图片对应的类标号

    ''' loss function '''
    res_loss = loss(outputs, targets)
    # print(res_loss)   # tensor(2.2994, grad_fn=<NllLossBackward0>) 误差

    ''' backward '''
    res_loss.backward()
    print("ok")

3 Optimizer

Pytorch Docs: https://pytorch.org/docs/1.8.1/optim.html

1. Basical Usage

  1. Constructing it
1
2
3
4
''' 参数parameters和学习率lr(learning rate)是必须传入的，其他由特定算法决定 '''
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# or
optimizer = optim.Adam([var1, var2], lr=0.0001)
  1. Taking an optimization step
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
for input, target in dataset:
  	# 清零
    optimizer.zero_grad()
    
    # 模型训练
    output = model(input)
    
    # 损失函数
    loss = loss_fn(output, target)
    
    # 反向传播
    loss.backward()
    
    # 优化器更新参数
    optimizer.step()

4 完整网络代码 Full Example for NN

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import torch.optim
import torchvision
from torch import nn
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader

''' 加载数据 '''
dataset = torchvision.datasets.CIFAR10(
    root="./dataset",
    train=False,
    download=False,
    transform=torchvision.transforms.ToTensor()
)

dataloader = DataLoader(dataset, batch_size=64)


class Cifar(nn.Module):
    ''' 创建网络 '''

    def __init__(self):
        super(Cifar, self).__init__()
        self.model1 = Sequential(
            Conv2d(in_channels=3, out_channels=32, kernel_size=5, stride=1, padding=2),
            MaxPool2d(kernel_size=2),
            Conv2d(in_channels=32, out_channels=32, kernel_size=5, stride=1, padding=2),
            MaxPool2d(kernel_size=2),
            Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2),
            MaxPool2d(kernel_size=2),
            Flatten(),
            Linear(in_features=64 * 4 * 4, out_features=64),
            Linear(in_features=64, out_features=10),
        )

    def forward(self, x):
        x = self.model1(x)
        return x


''' 实例化网络 选择损失函数 优化器 '''
my_net = Cifar()
loss = nn.CrossEntropyLoss()
optim = torch.optim.SGD(my_net.parameters(), lr=0.01)  # 随机梯度下降

''' 开始训练 '''
epochs = 20
for epoch in range(epochs):
    # 每次训练起始损失
    running_loss = 0.0
    for data in dataloader:
        ''' network '''
        imgs, targets = data
        outputs = my_net(imgs)

        ''' loss function '''
        res_loss = loss(outputs, targets)

        ''' zero_grad '''
        optim.zero_grad()

        ''' backward '''
        res_loss.backward()

        ''' optim_step updated '''
        optim.step()

        ''' each epoch loss sum '''
        running_loss += res_loss

    print("every epoch, the loss is: {}".format(running_loss))
1
2
3
4
5
6
7
8
>>> every epoch, the loss is: 360.25927734375
>>> every epoch, the loss is: 354.6569519042969
>>> every epoch, the loss is: 336.74420166015625
>>> every epoch, the loss is: 319.4207458496094
>>> every epoch, the loss is: 310.6980285644531
>>> every epoch, the loss is: 302.37701416015625
>>> every epoch, the loss is: 292.7424011230469
>>> every epoch, the loss is: 284.93902587890625