tf-CNN（二）Tensorflow製作自己的數據集並訓練

參考：https://www.jianshu.com/p/15e3f74180fc

https://blog.csdn.net/wiinter_fdd/article/details/72835939

對於數據量較小而言，可能一般選擇直接將數據加載進內存，然後再分batch輸入網絡進行訓練。但是，如果數據量較大，這樣的方法就不適用了，因爲太耗內存。在這兒我介紹一種比較通用，高效的讀取方法，即使用tensorflow內定標準格式——TFRecord.TFRecords其實是一種二進制文件，雖然它不如其他格式好理解，但是它能更好的利用內存，更方便複製和移動，並且不需要單獨的標籤文件。TFRecord會根據你輸入的文件的類，自動給每一類打上同樣的標籤。

以訓練貓狗分類器爲例。收集貓狗的圖片各150張，其中訓練集各100張，測試集各50張，按順序編號。如圖，data與代碼在同一路徑下。

1、生成tfrecords文件

 make_tfrecords.py

"""
#用自己的圖片製作tfrecords數據集
"""

import tensorflow as tf
import os
from PIL import Image
#import matplotlib.pyplot as plt
import numpy as np

#訓練集和測試集需要各自生成一個tfcords文件
#cwd = './data/train/'
cwd = './data/test/'

#自己定義類別
classes = {'dog', 'cat'}

#生成的records文件名
#writer = tf.compat.v1.python_io.TFRecordWriter("dog_and_cat_train.tfrecords")    
##tensorflow2.0版的多加了compat.v1
writer = tf.compat.v1.python_io.TFRecordWriter("dog_and_cat_test.tfrecords")

for index,name in enumerate(classes):
    class_path = cwd + name + '/'
    for img_name in os.listdir(class_path):
        # 每張圖片的地址
        img_path = class_path + img_name

        img = Image.open(img_path)
        img = img.resize((128,128))
        print(np.shape(img))

        # 圖片轉化爲二進制格式
        img_raw = img.tobytes()

        # example對象對label和image數據進行封裝
        example = tf.train.Example(features = tf.train.Features(feature={
            "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[index])),
            "img_raw":tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
        }))

        #序列化爲字符串
        writer.write(example.SerializeToString())

writer.close()

因爲我的tensorflow是2。0版本，所以將

tf.python_io.TFRecordWriter

寫成

tf.compat.v1.python_io.TFRecordWriter

運行之後就在代碼所在路徑下生成兩個文件，一個是訓練集，一個是測試集

2、讀取tfrecords文件

read_tfrecords.py

"""
###讀出tfrecords中的圖片和標籤
"""

import numpy as np
#import tensorflow as tf
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

def read_and_decode(filename):

    #生成一個queue隊列
    filename_queue = tf.train.string_input_producer([filename], shuffle = True)

    reader = tf.TFRecordReader()
    #返回文件名和文件
    _, serialized_example = reader.read(filename_queue)
    #將image和label取出來
    features = tf.parse_single_example(serialized_example,
                                       features={'label':tf.FixedLenFeature([],tf.int64),
                                                 'img_raw':tf.FixedLenFeature([],tf.string)})

    img = tf.decode_raw(features['img_raw'], tf.uint8)

    # reshape後圖片的大小及通道
    img = tf.reshape(img, [128,128,3])

    #在流中拋出img張量   和   label張量
    img = tf.cast(img, tf.float32) * (1./255) -0.5
    label = tf.cast(features['label'], tf.int32)

    return img, label

3、訓練

dog_and_cat_train.py

"""
###訓練貓狗分類模型
"""

#import tensorflow as tf
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
import numpy as np
import read_tfrecords

epoch = 15
batch_size = 20

def one_hot(labels, Label_class):
    one_hot_label = np.array([[int(i==int(labels[j])) for i in range(Label_class)] for j in range(len(labels))])
    return one_hot_label

#初始化權值
def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev = 0.02)
    return tf.Variable(initial)

#初始化偏置
def bias_variable(shape):
    initial = tf.constant(0.0 , shape=shape)
    return tf.Variable(initial)

#卷積層
def conv2d(x,W):
    return tf.nn.conv2d(x,W,strides = [1,1,1,1],padding = 'SAME')

#池化層
def max_pool_4x4(x):
    return tf.nn.max_pool(x,ksize=[1,4,4,1],strides=[1,4,4,1],padding='SAME')

x = tf.placeholder(tf.float32, [batch_size, 128,128,3])
y_ = tf.placeholder(tf.float32, [batch_size, 2])

#1 卷積1和池化1
W_conv1 = weight_variable([5,5,3,32])
b_conv1 = bias_variable([32])
h_conv1 = tf.nn.relu(conv2d(x,W_conv1)+b_conv1)
h_pool1 = max_pool_4x4(h_conv1)

#2 卷積2和池化2
W_conv2 = weight_variable([5,5,32,64])
b_conv2 = bias_variable([64])
h_conv2 = tf.nn.relu(conv2d(h_pool1,W_conv2)+b_conv2)
h_pool2 = max_pool_4x4(h_conv2)

#全連接，用1個MLP處理
reshape = tf.reshape(h_pool2,[batch_size, -1])
dim = reshape.get_shape()[1].value
W_fc1 = weight_variable([dim, 1024])
b_fc1 = bias_variable([1024])
h_fc1 = tf.nn.relu(tf.matmul(reshape, W_fc1)+b_fc1)

#dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

W_fc2 = weight_variable([1024,2])
b_fc2 = bias_variable([2])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2)+b_fc2)

#損失函數及優化算法
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv),reduction_indices = [1]))
train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)

correct_prediction = tf.equal(tf.argmax(y_conv,1),tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))

img,label = read_tfrecords.read_and_decode("dog_and_cat_train.tfrecords")
img_test, label_test = read_tfrecords.read_and_decode (("dog_and_cat_test.tfrecords"))

#使用shuffle_batch可以隨機打亂輸入
img_batch, label_batch = tf.train.shuffle_batch([img,label],
                                                batch_size=batch_size,capacity=2000,
                                                min_after_dequeue=1000)
img_test,label_test = tf.train.shuffle_batch([img_test,label_test],
                                             batch_size=batch_size,capacity=2000,
                                             min_after_dequeue=1000)

init = tf.initialize_all_variables()
t_vars = tf.trainable_variables()
print(t_vars)

with tf.Session() as sess:
    sess.run(init)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess,coord=coord)
    batch_idxs = int(2314/batch_size)
    for i in range(epoch):
        for j in range(batch_idxs):
            val,l = sess.run([img_batch, label_batch])
            l = one_hot(l,2)
            _, acc = sess.run([train_step, accuracy], feed_dict={x: val, y_: l, keep_prob: 0.5})
            print("Epoch:[%4d] [%4d/%4d], accuracy:[%.8f]" % (i, j, batch_idxs, acc))

    val,l = sess.run([img_test,label_test])
    l=one_hot(l,2)
    print(l)
    y,acc = sess.run([y_conv,accuracy],feed_dict={x:val,y_:l,keep_prob:1})
    print(y)
    print("test accuracy:[%.8f]" % (acc))

    coord.request_stop()
    coord.join(threads)

這一部分不多說，就是卷積層、池化層、全連接層的CNN網絡。

開頭的

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

代替

import tensorflow as tf

主要還是因爲我的tensorflow版本問題，如果直接import會出現“module 'tensorflow' has no attribute 'placeholder'”的錯誤。

而由於3要調用2的代碼，所以read_tfrecords.py中，也需要這麼寫，否則也會出現錯誤。

訓練結果：

最終我的accuracy只有0.4499，可能是數據太少，欠擬合了。