PyTorch学习笔记（5）–mnist数据集训练

from __future__ import print_function
import os
import struct
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import matplotlib.pyplot as plt
def load_mnist(path,kind='train'):
    labels_path=os.path.join(path,'%s-labels.idx1-ubyte/%s-labels.idx1-ubyte'%(kind,kind))
    images_path = os.path.join(path, '%s-images.idx3-ubyte/%s-images.idx3-ubyte' % (kind, kind))
    # labels_path = os.path.join(path, '%s-labels.idx1-ubyte' % kind)
    # images_path=os.path.join(path,'%s-images.idx3-ubyte'%kind)
    with open(labels_path,'rb') as lbpath:
        magic,n=struct.unpack('>II',lbpath.read(8))
        labels=np.fromfile(lbpath,dtype=np.uint8)
    with open(images_path,'rb') as imgpath:
        magic,num,rows,cols=struct.unpack(">IIII",imgpath.read(16))
        # labels=np.array(labels)
        images=np.fromfile(imgpath,dtype=np.uint8).reshape(len(labels),784)
    return images,labels
X_train,y_train =load_mnist('./data',kind='train')
print('Rows:%d,columns:%d'%(X_train.shape[0],X_train.shape[1]))
X_test,y_test=load_mnist('./data',kind='t10k')
print('Rows:%d,columns:%d'%(X_train.shape[0],X_train.shape[1]))
batch_size=100
num_classes=10
epochs=2
input image dimensions
img_rows,img_cols =28,28
train_losses=[]
test_losses=[]
test_counter=[]
train_counter=[]
x_train=X_train
x_test=X_test
if 'channels_first'=='channels_first':
    x_train=x_train.reshape(x_train.shape[0],1,img_rows,img_cols)#样本，通道，高度，宽度
    x_test=x_test.reshape(x_test.shape[0],1,img_rows,img_cols)
    input_shape=(1,img_rows,img_cols)
else:
    x_train=x_train.reshape(x_train.shape[0],img_rows,img_cols,1)
    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
    input_shape=(img_rows,img_cols,1)
x_train=x_train.astype('float32')
x_test=x_test.astype('float32')
x_train/=255
x_test/=255
print('x_train shape:',x_train.shape)
print(x_train.shape[0],'x_train samples')
print(y_train.shape[0],'y_train samples')
print(x_test.shape[0],'x_test samples')
print(y_test.shape[0],'y_test samples')
num_samples=x_train.shape[0]
print("num_samples:",num_samples)
'''
build torch model
'''
class Net(nn.Module):
    def __init__(self):
        super(Net,self).__init__()
        #(5,5)卷积核
        self.conv1=nn.Conv2d(1,10,kernel_size=5)#将1通道的图像数据的输入卷积成10个神经元，这一个通道会和10个神经元都建立连接
        self.conv2=nn.Conv2d(10,20,kernel_size=5)#将10通道的图像数据的输入卷积成20个神经元，这10个通道会和20个神经元都建立连接
        # self.conv1 = nn.Conv2d(1, 32, kernel_size=5)
        # self.conv2 = nn.Conv2d(32, 64, kernel_size=5)
        self.conv2_drop=nn.Dropout2d()#抛弃部分数据
        #self.fc1=nn.Linear(320,50)
        self.fc1 = nn.Linear(10*6*6, 50)
        self.fc2=nn.Linear(50,10)#两个全连接层
    def forward(self,x):
        x=F.relu(F.max_pool2d(self.conv1(x),2))#池化成最大值，池化核为（2，2）
        x=F.relu(F.max_pool2d(self.conv2_drop(self.conv2_drop(x)),2))
        x=x.view(-1,10*6*6)#动态调整这个维度上的元素个数以保证元素的总数不变
        x=F.relu(self.fc1(x))
        x=F.dropout(x,training=self.training)
        x=self.fc2(x)
        return F.log_softmax(x,dim=1)#图像分类的损失函数
model=Net()
if os.path.exists('mnist_torch.pkl'):
    model=torch.load('mnist_torch.pkl')
print(model)

'''
trainning
'''
optimizer=optim.SGD(model.parameters(),lr=0.01,momentum=0.5)
def train(epoch,x_train,y_train):
    num_batchs=num_samples/batch_size
    model.train()
    num_batchs=int(num_batchs)
    for k in range(num_batchs):
        start,end=k*batch_size,(k+1)*batch_size
        data,target=Variable(x_train[start:end],requires_grad=False),Variable(y_train[start:end])
        optimizer.zero_grad()
        output=model(data)
        loss=F.nll_loss(output,target)
        loss.backward()
        optimizer.step()
        if k%10==0:
            print('Train Epoch:{}[{}/{}({:.0f}%)]\tLoss:{:.6f}'.format(epoch,k*len(data),num_samples,100.*k/num_samples,loss.item()))
            train_losses.append(loss.item())
            train_counter.append((k*64)+((epoch-1)*len(data)))
    torch.save(model,'mnist_torch.pkl')

'''
evaluate
'''
def test(epoch):
    model.eval()
    test_loss=0
    correct=0
    if 2>1:
        with torch.no_grad():
            data,target=Variable(x_test),Variable(y_test)
        #data,target=Variable(x_test,volatile=True),Variable(y_test)
        output=model(data)
        test_loss+=F.nll_loss(output,target).item()
        test_losses.append(test_loss)
        # for i in range(epochs+1):
        #     test_counter.append(i*len(data))
        test_counter.append( len(data))
        # test_counter=[i*len(data) for i in range(epochs+1)]
        # print(test_loss)
        # print(len((x_test)))
        pred=output.data.max(1)[1]
        correct+=pred.eq(target.data).cpu().sum()
    #test_loss/=len(x_test)
    print('\nTest set:Average loss:{:.4f},Accuracy:{}/{}({:.0f}%)\n'.format(test_loss,correct,len(x_test),100.*correct/len(x_test)))
x_train=torch.from_numpy(x_train).float()
x_test=torch.from_numpy(x_test).float()
y_train=torch.from_numpy(y_train).long()
y_test=torch.from_numpy(y_test).long()
for epoch in range(1,epochs):
    train(epoch,x_train,y_train)
    test(epoch)
'''
painting
'''
fig= plt.figure()
plt.plot(train_counter,train_losses,color='blue')
plt.scatter(test_counter,test_losses,color='red')
plt.legend(['Train Loss', 'Test Loss'], loc='upper right')
plt.xlabel('number of training examples seen')
plt.ylabel('negative log likelihood loss')
plt.show()

得到结果：（多次训练后的结果，每次训练得到的结果不同）

Rows:60000,columns:784
Rows:60000,columns:784
x_train shape: (60000, 1, 28, 28)
60000 x_train samples
60000 y_train samples
10000 x_test samples
10000 y_test samples
num_samples: 60000
Net(
(conv1): Conv2d(1, 10, kernel_size=(5, 5), stride=(1, 1))
(conv2): Conv2d(10, 20, kernel_size=(5, 5), stride=(1, 1))
(conv2_drop): Dropout2d(p=0.5, inplace=False)
(fc1): Linear(in_features=360, out_features=50, bias=True)
(fc2): Linear(in_features=50, out_features=10, bias=True)
)
Train Epoch:1[0/60000(0%)] Loss:0.378188
Train Epoch:1[1000/60000(0%)] Loss:0.766636
Train Epoch:1[2000/60000(0%)] Loss:0.662513
Train Epoch:1[3000/60000(0%)] Loss:0.674844
Train Epoch:1[4000/60000(0%)] Loss:0.374800
Train Epoch:1[5000/60000(0%)] Loss:0.466920
Train Epoch:1[6000/60000(0%)] Loss:0.474571
Train Epoch:1[7000/60000(0%)] Loss:0.589114
Train Epoch:1[8000/60000(0%)] Loss:0.557918
Train Epoch:1[9000/60000(0%)] Loss:0.316381
Train Epoch:1[10000/60000(0%)] Loss:0.515177
Train Epoch:1[11000/60000(0%)] Loss:0.336269
Train Epoch:1[12000/60000(0%)] Loss:0.458744
Train Epoch:1[13000/60000(0%)] Loss:0.630309
Train Epoch:1[14000/60000(0%)] Loss:0.698837
Train Epoch:1[15000/60000(0%)] Loss:0.519987
Train Epoch:1[16000/60000(0%)] Loss:0.631619
Train Epoch:1[17000/60000(0%)] Loss:0.646146
Train Epoch:1[18000/60000(0%)] Loss:0.692150
Train Epoch:1[19000/60000(0%)] Loss:0.635705
Train Epoch:1[20000/60000(0%)] Loss:0.465035
Train Epoch:1[21000/60000(0%)] Loss:0.415029
Train Epoch:1[22000/60000(0%)] Loss:0.412988
Train Epoch:1[23000/60000(0%)] Loss:0.583587
Train Epoch:1[24000/60000(0%)] Loss:0.593450
Train Epoch:1[25000/60000(0%)] Loss:0.530810
Train Epoch:1[26000/60000(0%)] Loss:0.570838
Train Epoch:1[27000/60000(0%)] Loss:0.474587
Train Epoch:1[28000/60000(0%)] Loss:0.456399
Train Epoch:1[29000/60000(0%)] Loss:0.559481
Train Epoch:1[30000/60000(0%)] Loss:0.441676
Train Epoch:1[31000/60000(1%)] Loss:0.546501
Train Epoch:1[32000/60000(1%)] Loss:0.662970
Train Epoch:1[33000/60000(1%)] Loss:0.598461
Train Epoch:1[34000/60000(1%)] Loss:0.529693
Train Epoch:1[35000/60000(1%)] Loss:0.492719
Train Epoch:1[36000/60000(1%)] Loss:0.541546
Train Epoch:1[37000/60000(1%)] Loss:0.877233
Train Epoch:1[38000/60000(1%)] Loss:0.443502
Train Epoch:1[39000/60000(1%)] Loss:0.315365
Train Epoch:1[40000/60000(1%)] Loss:0.523321
Train Epoch:1[41000/60000(1%)] Loss:0.494766
Train Epoch:1[42000/60000(1%)] Loss:0.587744
Train Epoch:1[43000/60000(1%)] Loss:0.607159
Train Epoch:1[44000/60000(1%)] Loss:0.575720
Train Epoch:1[45000/60000(1%)] Loss:0.761669
Train Epoch:1[46000/60000(1%)] Loss:0.535713
Train Epoch:1[47000/60000(1%)] Loss:0.402516
Train Epoch:1[48000/60000(1%)] Loss:0.527466
Train Epoch:1[49000/60000(1%)] Loss:0.652991
Train Epoch:1[50000/60000(1%)] Loss:0.438617
Train Epoch:1[51000/60000(1%)] Loss:0.314596
Train Epoch:1[52000/60000(1%)] Loss:0.697034
Train Epoch:1[53000/60000(1%)] Loss:0.533702
Train Epoch:1[54000/60000(1%)] Loss:0.569537
Train Epoch:1[55000/60000(1%)] Loss:0.375768
Train Epoch:1[56000/60000(1%)] Loss:0.473715
Train Epoch:1[57000/60000(1%)] Loss:0.612956
Train Epoch:1[58000/60000(1%)] Loss:0.484374
Train Epoch:1[59000/60000(1%)] Loss:0.285547
Test set:Average loss:0.1313,Accuracy:9600/10000(96%)

得到train_loss曲线和test_loss点：

读者自己跑代码时若遇到问题：

PermissionError: [Errno 13] Permission denied: ‘./data\train-labels.idx1-ubyte’

请读者移步至笔者所写另一篇博客：(1条消息) 解决PermissionError: [Errno 13] Permission denied: ‘./data\train-labels.idx1-ubyte’_别管我啦就是说的博客-CSDN博客https://blog.csdn.net/m0_61385981/article/details/124309779?spm=1001.2014.3001.5502 ;

若读者需要查看mnist数据集内容：

import cv2
for i in range(60000):
    image1 = X_train[i]
    image1 = image1.astype('float32')
    image1 = image1.reshape(28, 28, 1)
    cv2.imwrite('./img/'+str(i)+".jpg", image1

mnist数据集下载网址：

http://yann.lecun.com/exdb/mnist/

参考书籍：《PyTorch深度学习实战》

Original: https://blog.csdn.net/m0_61385981/article/details/124335227
Author: 别管我啦就是说
Title: PyTorch学习笔记（5）–mnist数据集训练