TFRecord读写简介+Demo 基于Ubuntu18.04+Tensorflow1.12 无WARNING
- TFRecord是TensorFlow官方推荐使用的数据格式化存储工具。
- 它规范了数据的读写方式。
- 只要生成一次TFRecord，之后的数据读取和加工处理的效率都会得到提高。
将图片转换成TFRecord
本例，将fashion-MNIST数据转换成TFRecord，需要先下载fashion数据集到当前目录下，参考：https://github.com/zalandoresearch/fashion-mnist/tree/master/data/fashion
import numpy as np
import tensorflow as tf
import gzip
import os
fashion_mnist_directory = './data/fashion/'
def load_mnist(path, kind='train'):
labels_path = os.path.join(path, '%s-labels-idx1-ubyte.gz' % kind)
images_path = os.path.join(path, '%s-images-idx3-ubyte.gz' % kind)
with gzip.open(labels_path, 'rb') as lbpath:
labels = np.frombuffer(lbpath.read(), dtype=np.uint8, offset=8)
with gzip.open(images_path, 'rb') as imgpath:
images = np.frombuffer(imgpath.read(), dtype=np.uint8, offset=16).reshape(-1, 784)
print(labels_path, "shape =", labels.shape)
print(images_path, "shape =", images.shape)
return images, labels
def make_example(image, label):
return tf.train.Example(features=tf.train.Features(feature={
'image_raw' : tf.train.Feature(bytes_list=tf.train.BytesList(value=[image.tobytes()])),
'label' : tf.train.Feature(int64_list=tf.train.Int64List(value=[int(label) ])) }))
def write_tfrecord(images, labels, filename):
writer = tf.python_io.TFRecordWriter(filename)
for image, label, k in zip(images, labels, range(labels.shape[0])):
exam = make_example(image, label)
writer.write(exam.SerializeToString())
if (k%100 == 0):
print("\rwriting", filename, "%6.2f%% complited." %(100.0*(k+1)/labels.shape[0]), end='')
print("\rwriting", filename, "%6.2f%% complited." %(100.0))
writer.close()
def main():
train_images, train_labels = load_mnist(fashion_mnist_directory, 'train')
test_images, test_labels = load_mnist(fashion_mnist_directory, 't10k')
write_tfrecord(train_images, train_labels, 'fashion_mnist_train.tfrecords')
write_tfrecord(test_images, test_labels, 'fashion_mnist_test.tfrecords')
if __name__ == '__main__':
main()读取TFRecord数据来训练
以下代码读取TFRecord数据用于训练，改代码改编自官方例程：https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/how_tos/reading_data
原始代码运行时报错，已修复。
注意：在这个例子中，_, loss_value = sess.run([train_op, loss])，只执行一次Batch Input，无论[]中是什么，有多少个操作。
import argparse
import os.path
import sys
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import mnist
FLAGS = None
TRAIN_FILE = 'fashion_mnist_train.tfrecords'
VALIDATION_FILE = 'fashion_mnist_test.tfrecords'
def decode(serialized_example):
features = tf.parse_single_example(serialized_example,
features={'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64)})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image.set_shape((mnist.IMAGE_PIXELS))
label = tf.cast(features['label'], tf.int32)
return image, label
def augment(image, label):
"""Placeholder for data augmentation."""
# OPTIONAL: Could reshape into a 28x28 image and apply distortions here.
return image, label
def normalize(image, label):
"""Convert `image` from [0, 255] -> [-0.5, 0.5] floats."""
image = tf.cast(image, tf.float32) * (1. / 255) - 0.5
return image, label
def inputs(train, batch_size, num_epochs):
"""Reads input data"""
if not num_epochs:
num_epochs = None
filename = os.path.join(FLAGS.train_dir, TRAIN_FILE if train else VALIDATION_FILE)
with tf.name_scope('input'):
dataset = tf.data.TFRecordDataset(filename)
dataset = dataset.map(decode)
dataset = dataset.map(augment)
dataset = dataset.map(normalize)
dataset = dataset.shuffle(1000 + 3 * batch_size)
dataset = dataset.repeat(num_epochs)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
return iterator.get_next()
def run_training():
with tf.Graph().as_default():
image_batch, label_batch = inputs(train=True,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logits = mnist.inference(image_batch, FLAGS.hidden1, FLAGS.hidden2)
loss = mnist.loss(logits, label_batch)
train_op = mnist.training(loss, FLAGS.learning_rate)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
try:
step = 0
while True: # Train until OutOfRangeError
start_time = time.time()
_, loss_value = sess.run([train_op, loss])
duration = time.time() - start_time
if step % 100 == 0:
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
step += 1
except tf.errors.OutOfRangeError:
print('Done training for %d epochs, %d steps.' % (FLAGS.num_epochs, step))
def main(_):
run_training()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--learning_rate', type=float, default=0.01, help='Initial learning rate.')
parser.add_argument('--num_epochs', type=int, default=2, help='Number of epochs to run trainer.')
parser.add_argument('--hidden1', type=int, default=128, help='Number of units in hidden layer 1.')
parser.add_argument('--hidden2', type=int, default=32, help='Number of units in hidden layer 2.')
parser.add_argument('--batch_size', type=int, default=100, help='Batch size.')
parser.add_argument('--train_dir', type=str, default='./', help='Directory with the training data.')
FLAGS, unparsed = parser.parse_known_args()
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)参考了：
- https://blog.csdn.net/gg_18826075157/article/details/78449104
- https://github.com/zalandoresearch/fashion-mnist/blob/master/utils/mnist_reader.py