pytutorial/pytorch/MNIST

Class

{:.no_toc}

* TOC {:toc}

The goal

Class has a very important job as a core container type in Python. It is really hard to find a good overview how to use them in a good practice manner.

Questions to David Rotermund

I will use Linux. You will need a replacement for gzip under Windows.

Download the files

We need to download the MNIST database files

t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz A source for that is for example https://deepai.org/dataset/mnist

Unpack the gz files

In a terminal:

gzip -d *.gz

Convert the data into numpy files

Label file structure

[offset] [type] [value] [description]

0000 32 bit integer 0x00000801(2049) magic number (MSB first)

0004 32 bit integer 60000 number of items

0008 unsigned byte ?? label

0009 unsigned byte ?? label

........

xxxx unsigned byte ?? label

The labels values are 0 to 9.

Pattern file structure

[offset] [type] [value] [description]

0000 32 bit integer 0x00000803(2051) magic number

0004 32 bit integer 60000 number of images

0008 32 bit integer 28 number of rows

0012 32 bit integer 28 number of columns

0016 unsigned byte ?? pixel

0017 unsigned byte ?? pixel

........

xxxx unsigned byte ?? pixel

Pixels are organized row-wise.

Pixel values are 0 to 255. 0 means background (white),

255 means foreground (black).

My source code for that task: convert.py

%%

import numpy as np

[offset] [type] [value] [description]

0000 32 bit integer 0x00000801(2049) magic number (MSB first)

0004 32 bit integer 60000 number of items

0008 unsigned byte ?? label

0009 unsigned byte ?? label

........

xxxx unsigned byte ?? label

The labels values are 0 to 9.

class ReadLabel: """Class for reading the labels from an MNIST label file"""

def __init__(self, filename: str) -> None:
    self.filename: str = filename
    self.data = self.read_from_file(filename)

def read_from_file(self, filename: str) -> np.ndarray:

    int_32bit_data = np.dtype(np.uint32)
    int_32bit_data = int_32bit_data.newbyteorder(">")

    with open(filename, "rb") as file:

        magic_flag: np.uint32 = np.frombuffer(file.read(4), int_32bit_data)[0]

        if magic_flag != 2049:
            data: np.ndarray = np.zeros(0)
            number_of_elements: int = 0
        else:
            number_of_elements = np.frombuffer(file.read(4), int_32bit_data)[0]

        if number_of_elements < 1:
            data = np.zeros(0)
        else:
            data = np.frombuffer(file.read(number_of_elements), dtype=np.uint8)

    return data

[offset] [type] [value] [description]

0000 32 bit integer 0x00000803(2051) magic number

0004 32 bit integer 60000 number of images

0008 32 bit integer 28 number of rows

0012 32 bit integer 28 number of columns

0016 unsigned byte ?? pixel

0017 unsigned byte ?? pixel

........

xxxx unsigned byte ?? pixel

Pixels are organized row-wise.

Pixel values are 0 to 255. 0 means background (white),

255 means foreground (black).

class ReadPicture: """Class for reading the images from an MNIST image file"""

def __init__(self, filename: str) -> None:
    self.filename: str = filename
    self.Data = self.read_from_file(filename)

def read_from_file(self, filename: str) -> np.ndarray:

    int_32bit_data = np.dtype(np.uint32)
    int_32bit_data = int_32bit_data.newbyteorder(">")

    with open(filename, "rb") as file:

        magic_flag = np.frombuffer(file.read(4), int_32bit_data)[0]

        if magic_flag != 2051:
            data = np.zeros(0)
            number_of_elements: int = 0
        else:
            number_of_elements = np.frombuffer(file.read(4), int_32bit_data)[0]

        if number_of_elements < 1:
            data = np.zeros(0)
            number_of_rows: int = 0
        else:
            number_of_rows = np.frombuffer(file.read(4), int_32bit_data)[0]

        if number_of_rows != 28:
            data = np.zeros(0)
            number_of_columns: int = 0
        else:
            number_of_columns = np.frombuffer(file.read(4), int_32bit_data)[0]

        if number_of_columns != 28:
            data = np.zeros(0)
        else:
            data = np.frombuffer(
                file.read(number_of_elements * number_of_rows * number_of_columns),
                dtype=np.uint8,
            )
            data = data.reshape(
                number_of_elements, number_of_columns, number_of_rows
            )

    return data

def proprocess_dataset(testdata_mode: bool) -> None:

if testdata_mode is True:
    filename_out_pattern: str = "test_pattern_storage.npy"
    filename_out_label: str = "test_label_storage.npy"
    filename_in_image: str = "t10k-images-idx3-ubyte"
    filename_in_label: str = "t10k-labels-idx1-ubyte"
else:
    filename_out_pattern = "train_pattern_storage.npy"
    filename_out_label = "train_label_storage.npy"
    filename_in_image = "train-images-idx3-ubyte"
    filename_in_label = "train-labels-idx1-ubyte"

pictures = ReadPicture(filename_in_image)
labels = ReadLabel(filename_in_label)

# Down to 0 ... 1.0
max_value = np.max(pictures.Data.astype(np.float32))
pattern_storage = np.float32(pictures.Data.astype(np.float32) / max_value).astype(
    np.float32
)

label_storage = np.uint64(labels.data)

np.save(filename_out_pattern, pattern_storage)
np.save(filename_out_label, label_storage)

proprocess_dataset(testdata_mode=True) proprocess_dataset(testdata_mode=False)

Now we have the files:

test_label_storage.npy test_pattern_storage.npy train_label_storage.npy train_pattern_storage.npy