Kaggle Competition - PyTorch Dataset and DataLoader

Preparation

Required Python Packages

We FIRST make sure 2 Python packages - PyTorch and TorchVision have been successfully installed.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
➜  ~ pip show torch
Name: torch
Version: 1.1.0a0+b6a8c45
Summary: Tensors and Dynamic neural networks in Python with strong GPU acceleration
Home-page: UNKNOWN
Author: UNKNOWN
Author-email: UNKNOWN
License: UNKNOWN
Location: /home/jiapei/.local/lib/python3.6/site-packages
Requires:
Required-by: torchvision, torchtext, torchgan, pytorch-pretrained-bert, pyro-ppl, flair, autokeras
➜ ~ pip show torchvision
Name: torchvision
Version: 0.2.1
Summary: image and video datasets and models for torch deep learning
Home-page: https://github.com/pytorch/vision
Author: PyTorch Core Team
Author-email: soumith@pytorch.org
License: BSD
Location: /home/jiapei/.local/lib/python3.6/site-packages
Requires: numpy, six, torch, pillow
Required-by: torchgan, torchfusion, autokeras

As you can see, I actually built my own PyTorch from scratch, so that I can also use caffe2 sometimes. Please refer to the following:

1
2
3
4
5
6
7
8
9
10
11
12
13
➜  ~ pip show caffe2
➜ ~ python
Python 3.6.7 (default, Oct 22 2018, 11:32:17)
[GCC 8.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import caffe2
>>> caffe2.__version__
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: module 'caffe2' has no attribute '__version__'
>>> caffe2.__file__
'/home/jiapei/.local/lib/python3.6/site-packages/caffe2/__init__.py'
>>>

Test

The Code

Trivial modifications have been done upon the code on PyTorch Dataset and DataLoader, as follows:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
#get_ipython().run_line_magic('matplotlib', 'inline')

import torch
from torch.utils.data import DataLoader, Dataset
import torchvision
from torchvision import transforms

print(torch.__version__)

class DatasetMNIST(Dataset):

def __init__(self, file_path, transform=None):
self.data = pd.read_csv(file_path)
self.transform = transform

def __len__(self):
return len(self.data)

def __getitem__(self, index):
# load image as ndarray type (Height * Width * Channels)
# be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]
# in this example, i don't use ToTensor() method of torchvision.transforms
# so you can convert numpy ndarray shape to tensor in PyTorch (H, W, C) --> (C, H, W)
image = self.data.iloc[index, 1:].values.astype(np.uint8).reshape((1, 28, 28))
label = self.data.iloc[index, 0]

if self.transform is not None:
image = self.transform(image)

return image, label


train_dataset = DatasetMNIST('./input/train.csv', transform=None)

# we can access and get data with index by __getitem__(index)
img, lab = train_dataset.__getitem__(0)


# we now didn't convert numpy array.
print(img.shape)
print(type(img))


# ### 3.3 take a look at the dataset
#
# you have to use data loader in PyTorch that will accutually read the data within batch size and put into memory.

train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)


# we can use dataloader as iterator by using iter() function.
train_iter = iter(train_loader)
print(type(train_iter))


# we can look at images and labels of batch size by extracting data .next() method.
images, labels = train_iter.next()

print('images shape on batch size = {}'.format(images.size()))
print('labels shape on batch size = {}'.format(labels.size()))


# make grid takes tensor as arg
# tensor : (batchsize, channels, height, width)
grid = torchvision.utils.make_grid(images)

plt.imshow(grid.numpy().transpose((1, 2, 0)))
plt.axis('off')
plt.title(labels.numpy());
plt.savefig('1.jpg')

# ### 3.4 transform is ToTensor()**
class DatasetMNIST2(Dataset):

def __init__(self, file_path, transform=None):
self.data = pd.read_csv(file_path)
self.transform = transform

def __len__(self):
return len(self.data)

def __getitem__(self, index):
# load image as ndarray type (Height * Width * Channels)
# be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]
# in this example, we use ToTensor(), so we define the numpy array like (H, W, C)
image = self.data.iloc[index, 1:].values.astype(np.uint8).reshape((28, 28, 1))
label = self.data.iloc[index, 0]

if self.transform is not None:
image = self.transform(image)

return image, label

train_dataset2 = DatasetMNIST2('./input/train.csv', transform=torchvision.transforms.ToTensor())

img, lab = train_dataset2.__getitem__(0)

print('image shape at the first row : {}'.format(img.size()))


train_loader2 = DataLoader(train_dataset2, batch_size=8, shuffle=True)

train_iter2 = iter(train_loader2)
print(type(train_iter2))

images, labels = train_iter2.next()

print('images shape on batch size = {}'.format(images.size()))
print('labels shape on batch size = {}'.format(labels.size()))


grid = torchvision.utils.make_grid(images)

plt.imshow(grid.numpy().transpose((1, 2, 0)))
plt.axis('off')
plt.title(labels.numpy());
plt.savefig('2.jpg')


transform = transforms.Compose([
transforms.ToPILImage(), # because the input dtype is numpy.ndarray
transforms.RandomHorizontalFlip(0.5), # because this method is used for PIL Image dtype
transforms.ToTensor(), # because inpus dtype is PIL Image
])

train_dataset3 = DatasetMNIST2('./input/train.csv', transform=transform)
train_loader3 = DataLoader(train_dataset3, batch_size=8, shuffle=True)

train_iter3 = iter(train_loader3)
print(type(train_iter3))

images, labels = train_iter3.next()

print('images shape on batch size = {}'.format(images.size()))
print('labels shape on batch size = {}'.format(labels.size()))


grid = torchvision.utils.make_grid(images)

plt.imshow(grid.numpy().transpose((1, 2, 0)))
plt.axis('off')
plt.title(labels.numpy());
plt.savefig('3.jpg')

Outcome

MNIST 1st Load
MNIST 2nd Load
MNIST 3rd Load