Tensorflow技術點整理

 我們用tensorflow2來看一個服飾數據集的識別。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    print(x_valid.shape, y_valid.shape)
    print(x_train.shape, y_train.shape)
    print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    show_single_image(x_train[1])

運行結果

(5000, 28, 28) (5000,)
(55000, 28, 28) (55000,)
(10000, 28, 28) (10000,)

這裏我們可以看到，這些圖像數據集都是28*28的單通道圖片。我們將這些數據集分成了55000個訓練數據集，5000個驗證數據集以及10000個測試數據集。現在我們再來看一下這些數據集的分類類型

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers

if __name__ == "__main__":

    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    show_imgs(3, 5, x_train, y_train, class_names)

運行結果

這裏我們可以看到這些服飾總共有10種類型。現在我們就使用神經網絡來對這些數據進行訓練和驗證

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵損失函數，隨機梯度下降法進行訓練
    # 這裏需要說明的是如果是categorical_crossentropy
    # 則需要將標籤數據集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=10, validation_data=(x_valid, y_valid))
    print(model.summary())
    print(history.history)

    def plot_learning_curves(history):
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
55000/55000 [==============================] - 6s 107us/sample - loss: 2.2443 - accuracy: 0.7192 - val_loss: 0.6124 - val_accuracy: 0.7928
Epoch 2/10
55000/55000 [==============================] - 5s 95us/sample - loss: 0.5678 - accuracy: 0.7980 - val_loss: 0.5467 - val_accuracy: 0.8018
Epoch 3/10
55000/55000 [==============================] - 5s 92us/sample - loss: 0.4991 - accuracy: 0.8188 - val_loss: 0.4968 - val_accuracy: 0.8358
Epoch 4/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.4646 - accuracy: 0.8316 - val_loss: 0.4910 - val_accuracy: 0.8332
Epoch 5/10
55000/55000 [==============================] - 7s 119us/sample - loss: 0.4371 - accuracy: 0.8393 - val_loss: 0.4768 - val_accuracy: 0.8374
Epoch 6/10
55000/55000 [==============================] - 7s 123us/sample - loss: 0.4211 - accuracy: 0.8455 - val_loss: 0.4608 - val_accuracy: 0.8444
Epoch 7/10
55000/55000 [==============================] - 6s 103us/sample - loss: 0.4046 - accuracy: 0.8504 - val_loss: 0.4506 - val_accuracy: 0.8464
Epoch 8/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.3902 - accuracy: 0.8550 - val_loss: 0.4427 - val_accuracy: 0.8480
Epoch 9/10
55000/55000 [==============================] - 5s 97us/sample - loss: 0.3796 - accuracy: 0.8581 - val_loss: 0.4814 - val_accuracy: 0.8312
Epoch 10/10
55000/55000 [==============================] - 5s 99us/sample - loss: 0.3714 - accuracy: 0.8608 - val_loss: 0.4665 - val_accuracy: 0.8440
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 266,610
Trainable params: 266,610
Non-trainable params: 0
_________________________________________________________________
None
{'loss': [2.2442730520161716, 0.5677844335772775, 0.49907195046598263, 0.46455069565122775, 0.43706936968456617, 0.4210625493981621, 0.4045901944290508, 0.3901773897236044, 0.3795636681730097, 0.37135513420971955], 'accuracy': [0.71916366, 0.7979818, 0.8187636, 0.83156365, 0.8393091, 0.84549093, 0.8503636, 0.8550182, 0.8580545, 0.8608364], 'val_loss': [0.6123501856803895, 0.5466709545612335, 0.4968059474229813, 0.49100303683280944, 0.4768493502855301, 0.46080493590831756, 0.4505628205776215, 0.44266556935310364, 0.4814212405562401, 0.4664571787118912], 'val_accuracy': [0.7928, 0.8018, 0.8358, 0.8332, 0.8374, 0.8444, 0.8464, 0.848, 0.8312, 0.844]}

這裏我們使用了三個全連接層來構建神經網絡，對驗證數據集的準確率達到了85.92%。這裏我們使用的是隨機梯度下降法，關於隨機梯度下降法的內容請參考機器學習算法整理 中的隨機梯度下降法。下面這個報告是告訴我們在神經網絡中每一個全連接層的維度以及參數量。由於我們的圖像是28*28，所以扁平化處理後是28*28=784個維度。經過全連接層之後，我們要把第一個全連接層的輸出變成樣本數*300個維度的矩陣，我們需要將矩陣[樣本數，784]•[784, 300]的矩陣W(有關矩陣乘法的內容請參考線性代數整理 中矩陣和矩陣的乘法)，一般我們會加上一個b，就是一個長度爲300的向量。這樣就得到了一個[樣本數, 300]的輸出矩陣。而參數量就等於784*300+300=235500。同理，第二個全連接層的輸出爲樣本數*100的矩陣，我們需要將第一層的輸出矩陣[樣本數, 300]•[300, 100]+b，就得到[樣本數, 100]的輸出，參數量即爲300*100+100=30100。而圖中也顯示出損失函數值是在穩固下降的，而識別準確率是在穩固上升的。現在我們來給訓練數據加上歸一化處理。我們先來看一下歸一化前後訓練數據集的最大值和最小值

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片數據轉成二維數據[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片數據,這裏fit_transform會記錄訓練數據集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裏驗證數據集會採用訓練數據集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    print(np.max(x_train), np.min(x_train))

運行結果

255 0
2.0231433 -0.8105136

這裏我們可以看到，訓練數據集歸一化前後它們的最大值最小值是不同的。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片數據轉成二維數據[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片數據,這裏fit_transform會記錄訓練數據集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裏驗證數據集會採用訓練數據集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵損失函數，隨機梯度下降法進行訓練
    # 這裏需要說明的是如果是categorical_crossentropy
    # 則需要將標籤數據集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=10, validation_data=(x_valid, y_valid))
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    print(history.history)

    def plot_learning_curves(history):
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
55000/55000 [==============================] - 5s 95us/sample - loss: 0.9122 - accuracy: 0.7003 - val_loss: 0.6157 - val_accuracy: 0.7920
Epoch 2/10
55000/55000 [==============================] - 5s 83us/sample - loss: 0.5771 - accuracy: 0.8010 - val_loss: 0.5165 - val_accuracy: 0.8182
Epoch 3/10
55000/55000 [==============================] - 6s 104us/sample - loss: 0.5075 - accuracy: 0.8235 - val_loss: 0.4732 - val_accuracy: 0.8378
Epoch 4/10
55000/55000 [==============================] - 7s 127us/sample - loss: 0.4716 - accuracy: 0.8343 - val_loss: 0.4473 - val_accuracy: 0.8480
Epoch 5/10
55000/55000 [==============================] - 6s 117us/sample - loss: 0.4477 - accuracy: 0.8421 - val_loss: 0.4323 - val_accuracy: 0.8530
Epoch 6/10
55000/55000 [==============================] - 6s 110us/sample - loss: 0.4306 - accuracy: 0.8483 - val_loss: 0.4175 - val_accuracy: 0.8580
Epoch 7/10
55000/55000 [==============================] - 5s 90us/sample - loss: 0.4167 - accuracy: 0.8521 - val_loss: 0.4101 - val_accuracy: 0.8564
Epoch 8/10
55000/55000 [==============================] - 5s 90us/sample - loss: 0.4059 - accuracy: 0.8562 - val_loss: 0.4022 - val_accuracy: 0.8626
Epoch 9/10
55000/55000 [==============================] - 5s 92us/sample - loss: 0.3964 - accuracy: 0.8599 - val_loss: 0.3944 - val_accuracy: 0.8634
Epoch 10/10
55000/55000 [==============================] - 5s 93us/sample - loss: 0.3880 - accuracy: 0.8630 - val_loss: 0.3889 - val_accuracy: 0.8606
10000/1 ==============================================] - 1s 51us/sample - loss: 0.2991 - accuracy: 0.8472
[0.43079993045330045, 0.8472]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 266,610
Trainable params: 266,610
Non-trainable params: 0
_________________________________________________________________
None
{'loss': [0.912204550890489, 0.5771390253067017, 0.507524390927228, 0.4716112705404108, 0.4477458443338221, 0.4306134765581651, 0.416733508162065, 0.4059152506828308, 0.3963754140810533, 0.38801397878473454], 'accuracy': [0.70034546, 0.801, 0.82345456, 0.8343091, 0.84214544, 0.84827274, 0.85214543, 0.85623634, 0.8598727, 0.8629818], 'val_loss': [0.615698953294754, 0.516535887670517, 0.47322214045524597, 0.4472690547943115, 0.43233813087940215, 0.41751169788837433, 0.41005783474445345, 0.40220267939567567, 0.3943846334218979, 0.38885269825458524], 'val_accuracy': [0.792, 0.8182, 0.8378, 0.848, 0.853, 0.858, 0.8564, 0.8626, 0.8634, 0.8606]}

這裏我們對測試數據集進行了測試，它的分類準確率達到了84.72%。

回調函數

回調函數是當我們在訓練模型的時候，中間可能要做一些事情。比如說模型訓練中，當損失函數值loss不再下降的時候，我們可以提前停止訓練。又比如在訓練的過程中，每隔段時間把模型參數給保存下來。又或者在模型訓練過程中，就對一些模型指標進行圖形化輸出等等。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片數據轉成二維數據[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片數據,這裏fit_transform會記錄訓練數據集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裏驗證數據集會採用訓練數據集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
    ])
    # 使用交叉熵損失函數，隨機梯度下降法進行訓練
    # 這裏需要說明的是如果是categorical_crossentropy
    # 則需要將標籤數據集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我們這裏使用三個回調函數Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以保存我們在訓練中的指標，比如損失函數值以及識別準確率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是保存模型參數
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前終止訓練
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=10,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

當我們訓練完成之後，可以到callbacks上一層文件夾中，展開callbacks文件夾，我們可以看到這樣一些文件

在callbacks上一層文件夾中輸入命令

(base) -bash-3.2$ tensorboard --logdir=callbacks

會得到這樣一些輸出

Serving TensorBoard on localhost; to expose to the network, use a proxy or pass --bind_all
TensorBoard 2.0.0 at http://localhost:6006/ (Press CTRL+C to quit)

此時打開瀏覽器訪問http://localhost:6006，會看到我們訓練中的指標數據圖

以及神經網絡的層級圖

而EarlyStopping的參數意義如下：

1. min_delta：增大或減小的閾值，只有大於這個部分纔算作improvement(改善)。這個值的大小取決於monitor，也反映了你的容忍程度。例如monitor(監控)是’acc’，同時其變化範圍在70%-90%之間，所以對於小於0.01%的變化不關心。加上觀察到訓練過程中存在抖動的情況（即先下降後上升），所以適當增大容忍程度，最終設爲0.001%。
2. patience：能夠容忍多少個epoch內都沒有improvement。這個設置其實是在抖動和真正的準確率下降之間做tradeoff(權衡)。如果patience設的大，那麼最終得到的準確率要略低於模型可以達到的最高準確率。如果patience設的小，那麼模型很可能在前期抖動，還在全圖搜索的階段就停止了，準確率一般很差。patience的大小和learning rate直接相關。在learning rate設定的情況下，前期先訓練幾次觀察抖動的epoch number，比其稍大些設置patience。在learning rate變化的情況下，建議要略小於最大的抖動epoch number。筆者在引入EarlyStopping之前就已經得到可以接受的結果了，EarlyStopping算是錦上添花，所以patience設的比較高，設爲抖動epoch number的最大值。
3. min_delta和patience都和“避免模型停止在抖動過程中”有關係，所以調節的時候需要互相協調。通常情況下，min_delta降低，那麼patience可以適當減少；min_delta增加，那麼patience需要適當延長；反之亦然。

歸一化與批歸一化

• 歸一化

1. Min-max歸一化：x*=(x-min)/(max-min)
2. Z-score歸一化：x*=(x-μ)/δ，這裏μ表示均值，δ表示方差，是我們經常使用的歸一化

• 批歸一化

批歸一化是指把歸一化這種思路從輸入數據上給擴展到網絡的每層的激活值上，每層的激活值都做歸一化。我們知道對於神經網絡來說是一種層級結構，每一層的輸出就是下一層的輸入，我們在每一層的輸入上都去做歸一化，那麼就會使得整個的網絡的效果會更好。

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片數據轉成二維數據[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片數據,這裏fit_transform會記錄訓練數據集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裏驗證數據集會採用訓練數據集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        # 批歸一化
        keras.layers.BatchNormalization(),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.BatchNormalization(),
        keras.layers.Dense(10, activation='softmax')
        # 上面是將批歸一化放在激活函數之後，也可以放在激活函數之前
        # keras.layers.Dense(300),
        # keras.layers.BatchNormalization(),
        # keras.layers.Activation('relu'),
    ])
    # 使用交叉熵損失函數，隨機梯度下降法進行訓練
    # 這裏需要說明的是如果是categorical_crossentropy
    # 則需要將標籤數據集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我們這裏使用三個回調函數Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以保存我們在訓練中的指標，比如損失函數值以及識別準確率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是保存模型參數
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前終止訓練
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=10,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

運行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/10
   32/55000 [..............................] - ETA: 44:06 - loss: 3.0657 - accuracy: 0.06252021-10-20 23:21:46.915226: I tensorflow/core/profiler/lib/profiler_session.cc:184] Profiler session started.
55000/55000 [==============================] - 12s 223us/sample - loss: 0.8149 - accuracy: 0.7291 - val_loss: 0.5425 - val_accuracy: 0.8172
Epoch 2/10
55000/55000 [==============================] - 11s 195us/sample - loss: 0.5600 - accuracy: 0.8085 - val_loss: 0.4776 - val_accuracy: 0.8398
Epoch 3/10
55000/55000 [==============================] - 13s 229us/sample - loss: 0.5023 - accuracy: 0.8254 - val_loss: 0.4405 - val_accuracy: 0.8514
Epoch 4/10
55000/55000 [==============================] - 12s 216us/sample - loss: 0.4684 - accuracy: 0.8367 - val_loss: 0.4181 - val_accuracy: 0.8600
Epoch 5/10
55000/55000 [==============================] - 11s 200us/sample - loss: 0.4447 - accuracy: 0.8459 - val_loss: 0.4045 - val_accuracy: 0.8634
Epoch 6/10
55000/55000 [==============================] - 11s 200us/sample - loss: 0.4250 - accuracy: 0.8526 - val_loss: 0.3916 - val_accuracy: 0.8688
Epoch 7/10
55000/55000 [==============================] - 11s 204us/sample - loss: 0.4093 - accuracy: 0.8561 - val_loss: 0.3838 - val_accuracy: 0.8692
Epoch 8/10
55000/55000 [==============================] - 11s 207us/sample - loss: 0.3983 - accuracy: 0.8600 - val_loss: 0.3772 - val_accuracy: 0.8718
Epoch 9/10
55000/55000 [==============================] - 11s 207us/sample - loss: 0.3886 - accuracy: 0.8645 - val_loss: 0.3687 - val_accuracy: 0.8732
Epoch 10/10
55000/55000 [==============================] - 12s 212us/sample - loss: 0.3777 - accuracy: 0.8681 - val_loss: 0.3646 - val_accuracy: 0.8742
10000/1 [==================================================] - 1s 99us/sample - loss: 0.3293 - accuracy: 0.8609
[0.39349488220214845, 0.8609]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
batch_normalization (BatchNo (None, 300)               1200      
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
batch_normalization_1 (Batch (None, 100)               400       
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 268,210
Trainable params: 267,410
Non-trainable params: 800
_________________________________________________________________
None

通過批過一化後，我們可以看到它的識別準確率比之前都得到了提升。

dropout

有關dropout的詳細內容請參考Tensorflow深度學習算法整理

import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
import pandas as pd
import os
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
    fashion_mnist = keras.datasets.fashion_mnist
    (x_train_all, y_train_all), (x_test, y_test) = fashion_mnist.load_data()
    x_valid, x_train = x_train_all[:5000], x_train_all[5000:]
    y_valid, y_train = y_train_all[:5000], y_train_all[5000:]
    # print(x_valid.shape, y_valid.shape)
    # print(x_train.shape, y_train.shape)
    # print(x_test.shape, y_test.shape)
    # print(np.max(x_train), np.min(x_train))

    scaler = StandardScaler()
    # 歸一化過程中，需要將三維圖片數據轉成二維數據[None, 28, 28] -> [None, 784]
    # 歸一化結束之後再轉回三維圖片數據,這裏fit_transform會記錄訓練數據集的均值和方差
    x_train = scaler.fit_transform(x_train.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # 這裏驗證數據集會採用訓練數據集的均值和方差
    x_valid = scaler.transform(x_valid.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    x_test = scaler.transform(x_test.astype(np.float32).reshape(-1, 1)).reshape(-1, 28, 28)
    # print(np.max(x_train), np.min(x_train))

    def show_single_image(img_arr):
        plt.imshow(img_arr, cmap='binary')
        plt.show()

    # show_single_image(x_train[1])

    def show_imgs(n_rows, n_cols, x, y, class_names):
        assert len(x) == len(y)
        assert n_rows * n_cols < len(x)
        plt.figure(figsize=(n_cols * 1.4, n_rows * 1.6))
        for row in range(n_rows):
            for col in range(n_cols):
                index = n_cols * row + col
                plt.subplot(n_rows, n_cols, index + 1)
                plt.imshow(x[index], cmap='binary', interpolation='nearest')
                plt.axis('off')
                plt.title(class_names[y[index]])
        plt.show()
    #
    class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
                   'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
    # show_imgs(3, 5, x_train, y_train, class_names)
    model = keras.models.Sequential([
        keras.layers.Flatten(input_shape=[28, 28]),
        keras.layers.Dense(300, activation='relu'),
        # 批歸一化
        keras.layers.BatchNormalization(),
        keras.layers.Dense(100, activation='relu'),
        keras.layers.BatchNormalization(),
        # AlphaDropout，1均值和方差不變，2激活值歸一化性質不變
        keras.layers.AlphaDropout(rate=0.5),
        # keras.layers.Dropout(rate=0.5),
        keras.layers.Dense(10, activation='softmax')
        # 上面是將批歸一化放在激活函數之後，也可以放在激活函數之前
        # keras.layers.Dense(300),
        # keras.layers.BatchNormalization(),
        # keras.layers.Activation('relu'),
    ])
    # 使用交叉熵損失函數，隨機梯度下降法進行訓練
    # 這裏需要說明的是如果是categorical_crossentropy
    # 則需要將標籤數據集進行one_hot處理，如果是sparse_categorical_crossentropy，
    # 則不需要
    model.compile(loss=tf.losses.sparse_categorical_crossentropy,
                  optimizer=optimizers.SGD(learning_rate=0.001),
                  metrics=['accuracy'])
    # 我們這裏使用三個回調函數Tensorboard, earlystopping, ModelCheckpoint
    logdir = "./callbacks"
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir, "fashion_mnist_model.h5")
    callbacks = [
        # TensorBoard可以保存我們在訓練中的指標，比如損失函數值以及識別準確率
        keras.callbacks.TensorBoard(logdir),
        # ModelCheckpoint是保存模型參數
        keras.callbacks.ModelCheckpoint(output_model_file, save_best_only=True),
        # EarlyStopping是提前終止訓練
        keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)
    ]
    history = model.fit(x_train, y_train, epochs=20,
                        validation_data=(x_valid, y_valid),
                        callbacks=callbacks)
    print(model.evaluate(x_test, y_test))
    print(model.summary())
    # print(history.history)

    # def plot_learning_curves(history):
    #     pd.DataFrame(history.history).plot(figsize=(8, 5))
    #     plt.grid(True)
    #     plt.gca().set_ylim(0, 1)
    #     plt.show()
    #
    # plot_learning_curves(history)

運行結果

Train on 55000 samples, validate on 5000 samples
Epoch 1/20
   32/55000 [..............................] - ETA: 32:30 - loss: 2.7970 - accuracy: 0.18752021-10-20 23:56:36.685505: I tensorflow/core/profiler/lib/profiler_session.cc:184] Profiler session started.
55000/55000 [==============================] - 14s 252us/sample - loss: 1.6393 - accuracy: 0.4541 - val_loss: 0.7376 - val_accuracy: 0.7648
Epoch 2/20
55000/55000 [==============================] - 12s 222us/sample - loss: 1.1364 - accuracy: 0.6211 - val_loss: 0.6595 - val_accuracy: 0.7934
Epoch 3/20
55000/55000 [==============================] - 15s 275us/sample - loss: 0.9986 - accuracy: 0.6656 - val_loss: 0.6102 - val_accuracy: 0.8088
Epoch 4/20
55000/55000 [==============================] - 14s 257us/sample - loss: 0.9116 - accuracy: 0.6943 - val_loss: 0.5842 - val_accuracy: 0.8180
Epoch 5/20
55000/55000 [==============================] - 13s 232us/sample - loss: 0.8560 - accuracy: 0.7124 - val_loss: 0.5690 - val_accuracy: 0.8248
Epoch 6/20
55000/55000 [==============================] - 12s 227us/sample - loss: 0.8100 - accuracy: 0.7288 - val_loss: 0.5518 - val_accuracy: 0.8328
Epoch 7/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.7779 - accuracy: 0.7401 - val_loss: 0.5428 - val_accuracy: 0.8372
Epoch 8/20
55000/55000 [==============================] - 13s 229us/sample - loss: 0.7481 - accuracy: 0.7501 - val_loss: 0.5353 - val_accuracy: 0.8414
Epoch 9/20
55000/55000 [==============================] - 12s 225us/sample - loss: 0.7238 - accuracy: 0.7579 - val_loss: 0.5326 - val_accuracy: 0.8420
Epoch 10/20
55000/55000 [==============================] - 12s 222us/sample - loss: 0.6993 - accuracy: 0.7648 - val_loss: 0.5207 - val_accuracy: 0.8458
Epoch 11/20
55000/55000 [==============================] - 12s 222us/sample - loss: 0.6878 - accuracy: 0.7677 - val_loss: 0.5111 - val_accuracy: 0.8478
Epoch 12/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.6640 - accuracy: 0.7769 - val_loss: 0.5103 - val_accuracy: 0.8474
Epoch 13/20
55000/55000 [==============================] - 13s 243us/sample - loss: 0.6516 - accuracy: 0.7794 - val_loss: 0.5073 - val_accuracy: 0.8494
Epoch 14/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.6374 - accuracy: 0.7845 - val_loss: 0.5106 - val_accuracy: 0.8528
Epoch 15/20
55000/55000 [==============================] - 13s 234us/sample - loss: 0.6228 - accuracy: 0.7924 - val_loss: 0.5013 - val_accuracy: 0.8534
Epoch 16/20
55000/55000 [==============================] - 13s 231us/sample - loss: 0.6058 - accuracy: 0.7943 - val_loss: 0.4987 - val_accuracy: 0.8558
Epoch 17/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.5989 - accuracy: 0.7980 - val_loss: 0.4985 - val_accuracy: 0.8556
Epoch 18/20
55000/55000 [==============================] - 13s 232us/sample - loss: 0.5835 - accuracy: 0.8036 - val_loss: 0.4997 - val_accuracy: 0.8566
Epoch 19/20
55000/55000 [==============================] - 13s 237us/sample - loss: 0.5795 - accuracy: 0.8058 - val_loss: 0.4907 - val_accuracy: 0.8596
Epoch 20/20
55000/55000 [==============================] - 13s 230us/sample - loss: 0.5730 - accuracy: 0.8054 - val_loss: 0.4887 - val_accuracy: 0.8606
10000/1 [================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 1s 81us/sample - loss: 0.4401 - accuracy: 0.8478
[0.564703483581543, 0.8478]
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
dense (Dense)                (None, 300)               235500    
_________________________________________________________________
batch_normalization (BatchNo (None, 300)               1200      
_________________________________________________________________
dense_1 (Dense)              (None, 100)               30100     
_________________________________________________________________
batch_normalization_1 (Batch (None, 100)               400       
_________________________________________________________________
alpha_dropout (AlphaDropout) (None, 100)               0         
_________________________________________________________________
dense_2 (Dense)              (None, 10)                1010      
=================================================================
Total params: 268,210
Trainable params: 267,410
Non-trainable params: 800
_________________________________________________________________
None

通過對dropout的使用，我們發現這個數據集的過擬合現象並不嚴重，它的識別準確率比之前反而有所降低。

wide_deep模型

google16年發佈，可用於分類和迴歸

稀疏特徵

• 離散值特徵，比如說一個人的性別信息，只能從兩個值裏面去進行選擇。即類別類型的特徵。
• 可以使用One-hot表示。

比如：專業={計算機,人文,其他}。人文=[0, 1, 0]

比如:  詞表={人工智能,你,他,網絡,...}，他=[0, 0, 1, 0, ...]

叉乘={(計算機,人工智能),(計算機,你),...}，這裏表示"專業"中的每個詞跟"詞表"中的每個詞進行組合。叉乘是可以用來刻畫一個樣本的。

比如說一個物體的所有特徵，我們都把它表示成離散特徵，我們就可以用叉乘來精確的刻畫一個樣本，叉乘可以精確的刻畫到所有樣本的所有可能性。

• 叉乘之後
  • 稀疏特徵做叉乘獲取共現信息
  • 實現記憶的效果

如果一個物體可以用離散特徵去表達它的全部信息，那麼這個物體的總量也是確定的，那麼叉乘之後的總的集合包含了所有物體的所有可能性。設想我們有一種非常大的模型，這個模型可以把所有樣本都記住。當我們遇到新的樣本的時候，那麼這個樣本肯定也是這個大集合中的一個。所以如果我們的模型能夠記住大集合中的每一個樣本，那麼對於任何新的樣本它都可以100%的去進行正確的預測。

稀疏特徵——優缺點

• 優點
  • 有效，廣泛用於工業界
• 缺點
  • 需要人工設計
  • 可能過擬合，所有特徵都叉乘，相當於記住每一個樣本
  • 泛化能力差，沒出現過就不會起效果

密集特徵

• 向量表達

比如：詞表={人工智能,你,他,網絡,...}，他=[0.3, 0.2, 0.6, (n維向量)]，我們可以用向量之間的距離來表示這兩個詞對應的距離。

• Word2vec工具

可以參考Tensorflow深度學習算法整理(二) 中的內容

密集特徵的優缺點

• 優點
  • 帶有語義信息，不同向量之間有相關性
  • 兼容沒有出現過的特徵組合
  • 更少人工參與
• 缺點
  • 過度泛化，推薦不怎麼相關的產品

Wide & Deep模型結構

Wide & Deep Vs Wide

在左邊的Wide模型中，我們可以看到它只有一層，輸入就是稀疏特徵，也就是一個One-hot表達。One-hot表達的每個分量都直接連接到輸出單元上去。在右邊的Wide & Deep模型，它的左半部分也就是一個Wide模型，它的右半部分是一個Deep模型，這個Deep模型就是有多層的神經網絡，在這裏，我們將輸入的數據表現爲一個密集的向量表達，在密集表達之上有一個多層的神經網絡，在經過多層神經網絡之後再連接到輸出的神經單元上去。

Wide & Deep Vs Deep

在這裏，右圖是一個Deep模型，它跟Wide & Deep模型的區別就在於它沒有連接稀疏特徵的那一塊。

這是google商店中一個推薦算法的模型圖，它就是一個典型的Wide & Deep模型，它有很多的特徵，其中用戶已經安裝的軟件和展示的軟件做了叉乘，作爲Wide模型的輸入。而其他的特徵，有些做了Embedding，有些沒有做Embedding，把它們拼接起來，作爲一個總的特徵(密集表示)，再經過多層的神經網絡，形成一個Deep的部分，然後再使用一個邏輯迴歸的損失函數把Wide和Deep給組合起來。現在我們先來看一個加利福尼亞的一個房價的迴歸問題

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split

if __name__ == "__main__":

    housing = fetch_california_housing()
    print(housing.DESCR)
    print(housing.data.shape)
    print(housing.target.shape)
    pprint.pprint(housing.data[0:5])
    pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    print(X_train.shape, y_train.shape)
    print(X_valid.shape, y_valid.shape)
    print(X_test.shape, y_test.shape)

運行結果

.. _california_housing_dataset:

California Housing dataset
--------------------------

**Data Set Characteristics:**

    :Number of Instances: 20640

    :Number of Attributes: 8 numeric, predictive attributes and the target

    :Attribute Information:
        - MedInc        median income in block
        - HouseAge      median house age in block
        - AveRooms      average number of rooms
        - AveBedrms     average number of bedrooms
        - Population    block population
        - AveOccup      average house occupancy
        - Latitude      house block latitude
        - Longitude     house block longitude

    :Missing Attribute Values: None

This dataset was obtained from the StatLib repository.
http://lib.stat.cmu.edu/datasets/

The target variable is the median house value for California districts.

This dataset was derived from the 1990 U.S. census, using one row per census
block group. A block group is the smallest geographical unit for which the U.S.
Census Bureau publishes sample data (a block group typically has a population
of 600 to 3,000 people).

It can be downloaded/loaded using the
:func:`sklearn.datasets.fetch_california_housing` function.

.. topic:: References

    - Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,
      Statistics and Probability Letters, 33 (1997) 291-297

(20640, 8)
(20640,)
array([[ 8.32520000e+00,  4.10000000e+01,  6.98412698e+00,
         1.02380952e+00,  3.22000000e+02,  2.55555556e+00,
         3.78800000e+01, -1.22230000e+02],
       [ 8.30140000e+00,  2.10000000e+01,  6.23813708e+00,
         9.71880492e-01,  2.40100000e+03,  2.10984183e+00,
         3.78600000e+01, -1.22220000e+02],
       [ 7.25740000e+00,  5.20000000e+01,  8.28813559e+00,
         1.07344633e+00,  4.96000000e+02,  2.80225989e+00,
         3.78500000e+01, -1.22240000e+02],
       [ 5.64310000e+00,  5.20000000e+01,  5.81735160e+00,
         1.07305936e+00,  5.58000000e+02,  2.54794521e+00,
         3.78500000e+01, -1.22250000e+02],
       [ 3.84620000e+00,  5.20000000e+01,  6.28185328e+00,
         1.08108108e+00,  5.65000000e+02,  2.18146718e+00,
         3.78500000e+01, -1.22250000e+02]])
array([4.526, 3.585, 3.521, 3.413, 3.422])
(11610, 8) (11610,)
(3870, 8) (3870,)
(5160, 8) (5160,)

我們可以看到這裏有20640條樣本數據，維度爲8，說明影響房價的有8個特徵。其中，我們打印了5條樣本數據和標籤數據。並將其拆分爲訓練數據集，驗證數據集和測試數據集。現在我們來進行模型訓練和打印學習曲線

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    model = keras.models.Sequential([
        layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
        layers.Dense(1)
    ])
    print(model.summary())
    # 使用均方誤差損失函數和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.01))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 30)                270       
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 31        
=================================================================
Total params: 301
Trainable params: 301
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 88us/sample - loss: 1.3027 - val_loss: 2.2434
Epoch 2/100
11610/11610 [==============================] - 1s 45us/sample - loss: 4.6891 - val_loss: 0.4434
Epoch 3/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.4092 - val_loss: 0.3887
Epoch 4/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3811 - val_loss: 0.3797
Epoch 5/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3718 - val_loss: 0.3839
Epoch 6/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3652 - val_loss: 0.3714
Epoch 7/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3620 - val_loss: 0.3662
Epoch 8/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3587 - val_loss: 0.3715
Epoch 9/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3555 - val_loss: 0.3632
Epoch 10/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3525 - val_loss: 0.3620
Epoch 11/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3518 - val_loss: 0.3566
Epoch 12/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3486 - val_loss: 0.3544
Epoch 13/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3466 - val_loss: 0.3494
Epoch 14/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3468 - val_loss: 0.3458
Epoch 15/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3417 - val_loss: 0.3487
Epoch 16/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3412 - val_loss: 0.3449
Epoch 17/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3386 - val_loss: 0.3592
Epoch 18/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3372 - val_loss: 0.3412
Epoch 19/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3364 - val_loss: 0.3426
Epoch 20/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3336 - val_loss: 0.3462
Epoch 21/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3317 - val_loss: 0.3388
Epoch 22/100
11610/11610 [==============================] - 0s 43us/sample - loss: 0.3300 - val_loss: 0.3461
Epoch 23/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3292 - val_loss: 0.3355
Epoch 24/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3281 - val_loss: 0.3344
Epoch 25/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3266 - val_loss: 0.3374
Epoch 26/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3283 - val_loss: 0.3348
Epoch 27/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3242 - val_loss: 0.3395
Epoch 28/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3247 - val_loss: 0.3322
Epoch 29/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3233 - val_loss: 0.3288
Epoch 30/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3216 - val_loss: 0.3353
Epoch 31/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3216 - val_loss: 0.3293
Epoch 32/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3214 - val_loss: 0.3312
Epoch 33/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3201 - val_loss: 0.3290
Epoch 34/100
11610/11610 [==============================] - 0s 42us/sample - loss: 0.3189 - val_loss: 0.3319
5160/1 [==================================================] - 0s 19us/sample - loss: 0.4063
0.342487285580746

這裏我們發現，我們設置了100次的迭代，但是隻到了34次並沒有到100次就停止了，再根據學習曲線的情況，我們發現是EarlyStopping的回調函數起到了作用，是梯度下降已經到了瓶頸，趨於平緩。而測試數據集的損失函數值爲0.342。

現在我們來將這個簡單的迴歸模型改成Wide & Deep模型

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函數式API Deep模型
    input = layers.Input(shape=X_train.shape[1:])
    hidden1 = layers.Dense(30, activation='relu')(input)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # input是Wide模型的輸入，hidden2是Deep模型的輸出
    concat = layers.concatenate([input, hidden2])
    output = layers.Dense(1)(concat)
    model = models.Model(inputs=[input], outputs=[output])
    print(model.summary())
    # 使用均方誤差損失函數和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 8)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           270         input_1[0][0]                    
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 38)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            39          concatenate[0][0]                
==================================================================================================
Total params: 1,239
Trainable params: 1,239
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 107us/sample - loss: 1.7756 - val_loss: 0.8341
Epoch 2/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.7057 - val_loss: 0.7208
Epoch 3/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.6495 - val_loss: 0.6740
Epoch 4/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.6138 - val_loss: 0.6451
Epoch 5/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5884 - val_loss: 0.6189
Epoch 6/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.5682 - val_loss: 0.6013
Epoch 7/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5515 - val_loss: 0.5818
Epoch 8/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.5387 - val_loss: 0.5675
Epoch 9/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5272 - val_loss: 0.5547
Epoch 10/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5180 - val_loss: 0.5441
Epoch 11/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5085 - val_loss: 0.5336
Epoch 12/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.5012 - val_loss: 0.5285
Epoch 13/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4931 - val_loss: 0.5167
Epoch 14/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4881 - val_loss: 0.5128
Epoch 15/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4813 - val_loss: 0.5043
Epoch 16/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4760 - val_loss: 0.4983
Epoch 17/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4710 - val_loss: 0.4928
Epoch 18/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4655 - val_loss: 0.4917
Epoch 19/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4621 - val_loss: 0.4818
Epoch 20/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4571 - val_loss: 0.4776
Epoch 21/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4531 - val_loss: 0.4779
Epoch 22/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4501 - val_loss: 0.4696
Epoch 23/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4455 - val_loss: 0.4650
Epoch 24/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4425 - val_loss: 0.4612
Epoch 25/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4404 - val_loss: 0.4592
Epoch 26/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4367 - val_loss: 0.4542
Epoch 27/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4336 - val_loss: 0.4507
Epoch 28/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4310 - val_loss: 0.4492
Epoch 29/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4278 - val_loss: 0.4470
Epoch 30/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4257 - val_loss: 0.4429
Epoch 31/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4227 - val_loss: 0.4398
Epoch 32/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4208 - val_loss: 0.4375
Epoch 33/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4183 - val_loss: 0.4346
Epoch 34/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4162 - val_loss: 0.4327
Epoch 35/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4136 - val_loss: 0.4306
Epoch 36/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4116 - val_loss: 0.4275
Epoch 37/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4096 - val_loss: 0.4258
Epoch 38/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4075 - val_loss: 0.4236
Epoch 39/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4053 - val_loss: 0.4213
Epoch 40/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4037 - val_loss: 0.4191
Epoch 41/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4016 - val_loss: 0.4180
Epoch 42/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.4003 - val_loss: 0.4157
Epoch 43/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3981 - val_loss: 0.4129
Epoch 44/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3962 - val_loss: 0.4130
Epoch 45/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3949 - val_loss: 0.4094
Epoch 46/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3929 - val_loss: 0.4080
Epoch 47/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3915 - val_loss: 0.4067
Epoch 48/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3898 - val_loss: 0.4044
Epoch 49/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3885 - val_loss: 0.4032
Epoch 50/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3868 - val_loss: 0.4019
Epoch 51/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3850 - val_loss: 0.3995
Epoch 52/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3839 - val_loss: 0.3993
Epoch 53/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3825 - val_loss: 0.3971
Epoch 54/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3809 - val_loss: 0.3951
Epoch 55/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3798 - val_loss: 0.3940
Epoch 56/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3788 - val_loss: 0.3929
Epoch 57/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3775 - val_loss: 0.3916
Epoch 58/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3760 - val_loss: 0.3906
Epoch 59/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3749 - val_loss: 0.3900
Epoch 60/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3742 - val_loss: 0.3880
Epoch 61/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3726 - val_loss: 0.3866
Epoch 62/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3718 - val_loss: 0.3863
Epoch 63/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3706 - val_loss: 0.3853
Epoch 64/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.3698 - val_loss: 0.3834
Epoch 65/100
11610/11610 [==============================] - 1s 71us/sample - loss: 0.3687 - val_loss: 0.3834
Epoch 66/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3679 - val_loss: 0.3818
Epoch 67/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3668 - val_loss: 0.3811
Epoch 68/100
11610/11610 [==============================] - 1s 78us/sample - loss: 0.3658 - val_loss: 0.3801
Epoch 69/100
11610/11610 [==============================] - 1s 76us/sample - loss: 0.3649 - val_loss: 0.3787
Epoch 70/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.3639 - val_loss: 0.3781
Epoch 71/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3634 - val_loss: 0.3769
Epoch 72/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3621 - val_loss: 0.3762
Epoch 73/100
11610/11610 [==============================] - 1s 66us/sample - loss: 0.3615 - val_loss: 0.3762
Epoch 74/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3606 - val_loss: 0.3746
Epoch 75/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3596 - val_loss: 0.3749
Epoch 76/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3590 - val_loss: 0.3736
Epoch 77/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3580 - val_loss: 0.3740
Epoch 78/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3576 - val_loss: 0.3721
Epoch 79/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3562 - val_loss: 0.3712
Epoch 80/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3558 - val_loss: 0.3710
Epoch 81/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3549 - val_loss: 0.3710
Epoch 82/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3541 - val_loss: 0.3707
Epoch 83/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3537 - val_loss: 0.3690
Epoch 84/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3530 - val_loss: 0.3691
Epoch 85/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3520 - val_loss: 0.3679
Epoch 86/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3514 - val_loss: 0.3673
Epoch 87/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3508 - val_loss: 0.3671
Epoch 88/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3500 - val_loss: 0.3662
Epoch 89/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3496 - val_loss: 0.3655
Epoch 90/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3487 - val_loss: 0.3661
Epoch 91/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3484 - val_loss: 0.3658
Epoch 92/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3479 - val_loss: 0.3641
Epoch 93/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3468 - val_loss: 0.3637
Epoch 94/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3462 - val_loss: 0.3640
Epoch 95/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3456 - val_loss: 0.3629
Epoch 96/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3449 - val_loss: 0.3621
Epoch 97/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3444 - val_loss: 0.3621
Epoch 98/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3431 - val_loss: 0.3622
Epoch 99/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3434 - val_loss: 0.3606
Epoch 100/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3430 - val_loss: 0.3604
5160/1 [========================================================] - 0s 24us/sample - loss: 0.4557
0.36289026386053985

在這裏，我們使用的是函數式編程來實現的Wide & Deep模型，下面我們來使用子類API來實現這個模型

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函數式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的輸入，hidden2是Deep模型的輸出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子類API，父類爲keras.models.Model
    class WideDeepModel(models.Model):

        def __init__(self):
            super(WideDeepModel, self).__init__()
            # 定義模型的層次
            self.hidden1_layer = layers.Dense(30, activation='relu')
            self.hidden2_layer = layers.Dense(30, activation='relu')
            self.output_layer = layers.Dense(1)


        def call(self, input):
            # 完成模型的正向計算
            hidden1 = self.hidden1_layer(input)
            hidden2 = self.hidden2_layer(hidden1)
            concat = layers.concatenate([input, hidden2])
            output = self.output_layer(concat)
            return output

    model = WideDeepModel()
    # 也可以使用層堆疊
    # model = models.Sequential([
    #     WideDeepModel()
    # ])
    model.build(input_shape=(None, 8))
    print(model.summary())
    # 使用均方誤差損失函數和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    history = model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate(X_test, y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Model: "wide_deep_model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                multiple                  270       
_________________________________________________________________
dense_1 (Dense)              multiple                  930       
_________________________________________________________________
dense_2 (Dense)              multiple                  39        
=================================================================
Total params: 1,239
Trainable params: 1,239
Non-trainable params: 0
_________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 1s 107us/sample - loss: 2.0999 - val_loss: 0.8217
Epoch 2/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.6996 - val_loss: 0.7236
Epoch 3/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.6376 - val_loss: 0.6704
Epoch 4/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.6003 - val_loss: 0.6322
Epoch 5/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5720 - val_loss: 0.6104
Epoch 6/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5499 - val_loss: 0.5869
Epoch 7/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5324 - val_loss: 0.5653
Epoch 8/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.5188 - val_loss: 0.5499
Epoch 9/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.5075 - val_loss: 0.5401
Epoch 10/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4970 - val_loss: 0.5268
Epoch 11/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4902 - val_loss: 0.5204
Epoch 12/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.4819 - val_loss: 0.5103
Epoch 13/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.4759 - val_loss: 0.5056
Epoch 14/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.4697 - val_loss: 0.4977
Epoch 15/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.4654 - val_loss: 0.4931
Epoch 16/100
11610/11610 [==============================] - 1s 83us/sample - loss: 0.4597 - val_loss: 0.4928
Epoch 17/100
11610/11610 [==============================] - 1s 78us/sample - loss: 0.4538 - val_loss: 0.4821
Epoch 18/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.4506 - val_loss: 0.4765
Epoch 19/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.4465 - val_loss: 0.4744
Epoch 20/100
11610/11610 [==============================] - 1s 58us/sample - loss: 0.4430 - val_loss: 0.4762
Epoch 21/100
11610/11610 [==============================] - 1s 99us/sample - loss: 0.4391 - val_loss: 0.4668
Epoch 22/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.4361 - val_loss: 0.4641
Epoch 23/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.4328 - val_loss: 0.4587
Epoch 24/100
11610/11610 [==============================] - 1s 66us/sample - loss: 0.4293 - val_loss: 0.4565
Epoch 25/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4266 - val_loss: 0.4525
Epoch 26/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4237 - val_loss: 0.4481
Epoch 27/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4210 - val_loss: 0.4452
Epoch 28/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4178 - val_loss: 0.4426
Epoch 29/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.4163 - val_loss: 0.4410
Epoch 30/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.4135 - val_loss: 0.4372
Epoch 31/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.4111 - val_loss: 0.4350
Epoch 32/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.4096 - val_loss: 0.4329
Epoch 33/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4075 - val_loss: 0.4298
Epoch 34/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4053 - val_loss: 0.4293
Epoch 35/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.4035 - val_loss: 0.4263
Epoch 36/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.4018 - val_loss: 0.4236
Epoch 37/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3997 - val_loss: 0.4221
Epoch 38/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3977 - val_loss: 0.4207
Epoch 39/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3962 - val_loss: 0.4175
Epoch 40/100
11610/11610 [==============================] - 1s 48us/sample - loss: 0.3947 - val_loss: 0.4152
Epoch 41/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3927 - val_loss: 0.4140
Epoch 42/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3914 - val_loss: 0.4131
Epoch 43/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3897 - val_loss: 0.4102
Epoch 44/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3883 - val_loss: 0.4088
Epoch 45/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3869 - val_loss: 0.4063
Epoch 46/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3858 - val_loss: 0.4053
Epoch 47/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3841 - val_loss: 0.4041
Epoch 48/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3832 - val_loss: 0.4029
Epoch 49/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3818 - val_loss: 0.4002
Epoch 50/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.3807 - val_loss: 0.3987
Epoch 51/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3791 - val_loss: 0.3970
Epoch 52/100
11610/11610 [==============================] - 1s 73us/sample - loss: 0.3779 - val_loss: 0.3970
Epoch 53/100
11610/11610 [==============================] - 1s 98us/sample - loss: 0.3768 - val_loss: 0.3964
Epoch 54/100
11610/11610 [==============================] - 1s 92us/sample - loss: 0.3756 - val_loss: 0.3941
Epoch 55/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3746 - val_loss: 0.3926
Epoch 56/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3739 - val_loss: 0.3914
Epoch 57/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3727 - val_loss: 0.3896
Epoch 58/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3718 - val_loss: 0.3879
Epoch 59/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3701 - val_loss: 0.3900
Epoch 60/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3696 - val_loss: 0.3865
Epoch 61/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3683 - val_loss: 0.3855
Epoch 62/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3674 - val_loss: 0.3860
Epoch 63/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3669 - val_loss: 0.3831
Epoch 64/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3652 - val_loss: 0.3820
Epoch 65/100
11610/11610 [==============================] - 1s 76us/sample - loss: 0.3645 - val_loss: 0.3818
Epoch 66/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.3641 - val_loss: 0.3794
Epoch 67/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.3623 - val_loss: 0.3799
Epoch 68/100
11610/11610 [==============================] - 1s 57us/sample - loss: 0.3623 - val_loss: 0.3782
Epoch 69/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.3606 - val_loss: 0.3766
Epoch 70/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.3599 - val_loss: 0.3766
Epoch 71/100
11610/11610 [==============================] - 1s 87us/sample - loss: 0.3589 - val_loss: 0.3748
Epoch 72/100
11610/11610 [==============================] - 1s 100us/sample - loss: 0.3582 - val_loss: 0.3743
Epoch 73/100
11610/11610 [==============================] - 1s 60us/sample - loss: 0.3573 - val_loss: 0.3728
Epoch 74/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3564 - val_loss: 0.3735
Epoch 75/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.3562 - val_loss: 0.3718
Epoch 76/100
11610/11610 [==============================] - 1s 75us/sample - loss: 0.3557 - val_loss: 0.3714
Epoch 77/100
11610/11610 [==============================] - 1s 85us/sample - loss: 0.3548 - val_loss: 0.3703
Epoch 78/100
11610/11610 [==============================] - 1s 93us/sample - loss: 0.3540 - val_loss: 0.3700
Epoch 79/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3528 - val_loss: 0.3685
Epoch 80/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3526 - val_loss: 0.3678
Epoch 81/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3512 - val_loss: 0.3678
Epoch 82/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3506 - val_loss: 0.3672
Epoch 83/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3508 - val_loss: 0.3657
Epoch 84/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3495 - val_loss: 0.3669
Epoch 85/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3491 - val_loss: 0.3670
Epoch 86/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3490 - val_loss: 0.3642
Epoch 87/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3486 - val_loss: 0.3642
Epoch 88/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3478 - val_loss: 0.3643
Epoch 89/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3474 - val_loss: 0.3631
Epoch 90/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3467 - val_loss: 0.3614
Epoch 91/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3451 - val_loss: 0.3604
Epoch 92/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3444 - val_loss: 0.3606
Epoch 93/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3436 - val_loss: 0.3592
Epoch 94/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3435 - val_loss: 0.3588
Epoch 95/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3585
Epoch 96/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3576
Epoch 97/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3424 - val_loss: 0.3578
Epoch 98/100
11610/11610 [==============================] - 1s 50us/sample - loss: 0.3419 - val_loss: 0.3569
Epoch 99/100
11610/11610 [==============================] - 1s 49us/sample - loss: 0.3412 - val_loss: 0.3572
Epoch 100/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3404 - val_loss: 0.3551
5160/1 [===========================================================] - 0s 45us/sample - loss: 0.4603
0.36134041002554484

由於Wide & Deep模型可以分別挑選特徵來進行輸入，現在我們來改成多輸入的模式

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函數式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的輸入，hidden2是Deep模型的輸出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子類API，父類爲keras.models.Model
    # class WideDeepModel(models.Model):
    #
    #     def __init__(self):
    #         super(WideDeepModel, self).__init__()
    #         # 定義模型的層次
    #         self.hidden1_layer = layers.Dense(30, activation='relu')
    #         self.hidden2_layer = layers.Dense(30, activation='relu')
    #         self.output_layer = layers.Dense(1)
    #
    #
    #     def call(self, input):
    #         # 完成模型的正向計算
    #         hidden1 = self.hidden1_layer(input)
    #         hidden2 = self.hidden2_layer(hidden1)
    #         concat = layers.concatenate([input, hidden2])
    #         output = self.output_layer(concat)
    #         return output
    #
    # model = WideDeepModel()
    # # 也可以使用層堆疊
    # # model = models.Sequential([
    # #     WideDeepModel()
    # # ])
    # model.build(input_shape=(None, 8))
    # 多輸入 wide模型輸入前5個特徵，deep模型輸入後6個特徵
    input_wide = layers.Input(shape=[5])
    input_deep = layers.Input(shape=[6])
    hidden1 = layers.Dense(30, activation='relu')(input_deep)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    concat = layers.concatenate([input_wide, hidden2])
    output = layers.Dense(1)(concat)
    model = models.Model(inputs=[input_wide, input_deep], outputs=[output])
    print(model.summary())
    # 使用均方誤差損失函數和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    X_train_wide = X_train[:, :5]
    X_train_deep = X_train[:, 2:]
    X_valid_wide = X_valid[:, :5]
    X_valid_deep = X_valid[:, 2:]
    X_test_wide = X_test[:, :5]
    X_test_deep = X_test[:, 2:]
    history = model.fit([X_train_wide, X_train_deep], y_train,
                        validation_data=([X_valid_wide, X_valid_deep], y_valid),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate([X_test_wide, X_test_deep], y_test))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 6)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           210         input_2[0][0]                    
__________________________________________________________________________________________________
input_1 (InputLayer)            [(None, 5)]          0                                            
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 35)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            36          concatenate[0][0]                
==================================================================================================
Total params: 1,176
Trainable params: 1,176
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 2s 138us/sample - loss: 1.9978 - val_loss: 0.9837
Epoch 2/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.8110 - val_loss: 0.7862
Epoch 3/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.7007 - val_loss: 0.7168
Epoch 4/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.6489 - val_loss: 0.6742
Epoch 5/100
11610/11610 [==============================] - 1s 54us/sample - loss: 0.6118 - val_loss: 0.6378
Epoch 6/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5831 - val_loss: 0.6127
Epoch 7/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5607 - val_loss: 0.5910
Epoch 8/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5427 - val_loss: 0.5736
Epoch 9/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.5275 - val_loss: 0.5589
Epoch 10/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5144 - val_loss: 0.5455
Epoch 11/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.5033 - val_loss: 0.5353
Epoch 12/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4939 - val_loss: 0.5258
Epoch 13/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4853 - val_loss: 0.5156
Epoch 14/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4784 - val_loss: 0.5084
Epoch 15/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4718 - val_loss: 0.5035
Epoch 16/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4657 - val_loss: 0.4952
Epoch 17/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4606 - val_loss: 0.4887
Epoch 18/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4560 - val_loss: 0.4840
Epoch 19/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4517 - val_loss: 0.4805
Epoch 20/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4477 - val_loss: 0.4773
Epoch 21/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4451 - val_loss: 0.4727
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4423 - val_loss: 0.4697
Epoch 23/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4397 - val_loss: 0.4708
Epoch 24/100
11610/11610 [==============================] - 1s 72us/sample - loss: 0.4375 - val_loss: 0.4650
Epoch 25/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4350 - val_loss: 0.4633
Epoch 26/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.4329 - val_loss: 0.4593
Epoch 27/100
11610/11610 [==============================] - 1s 88us/sample - loss: 0.4310 - val_loss: 0.4569
Epoch 28/100
11610/11610 [==============================] - 1s 99us/sample - loss: 0.4294 - val_loss: 0.4558
Epoch 29/100
11610/11610 [==============================] - 1s 97us/sample - loss: 0.4276 - val_loss: 0.4521
Epoch 30/100
11610/11610 [==============================] - 1s 120us/sample - loss: 0.4254 - val_loss: 0.4494
Epoch 31/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4239 - val_loss: 0.4484
Epoch 32/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.4219 - val_loss: 0.4470
Epoch 33/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4209 - val_loss: 0.4456
Epoch 34/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4194 - val_loss: 0.4433
Epoch 35/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4179 - val_loss: 0.4427
Epoch 36/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4163 - val_loss: 0.4399
Epoch 37/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4150 - val_loss: 0.4399
Epoch 38/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4137 - val_loss: 0.4370
Epoch 39/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4122 - val_loss: 0.4364
Epoch 40/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4112 - val_loss: 0.4369
Epoch 41/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4098 - val_loss: 0.4329
Epoch 42/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4088 - val_loss: 0.4326
Epoch 43/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.4077 - val_loss: 0.4299
Epoch 44/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4066 - val_loss: 0.4304
Epoch 45/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.4054 - val_loss: 0.4284
Epoch 46/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.4043 - val_loss: 0.4286
Epoch 47/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.4036 - val_loss: 0.4263
Epoch 48/100
11610/11610 [==============================] - 1s 88us/sample - loss: 0.4026 - val_loss: 0.4251
Epoch 49/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.4014 - val_loss: 0.4240
Epoch 50/100
11610/11610 [==============================] - 1s 96us/sample - loss: 0.4007 - val_loss: 0.4231
Epoch 51/100
11610/11610 [==============================] - 1s 56us/sample - loss: 0.3998 - val_loss: 0.4216
Epoch 52/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3990 - val_loss: 0.4235
Epoch 53/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3981 - val_loss: 0.4211
Epoch 54/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3976 - val_loss: 0.4210
Epoch 55/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3965 - val_loss: 0.4217
Epoch 56/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3959 - val_loss: 0.4174
Epoch 57/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3953 - val_loss: 0.4175
Epoch 58/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.3943 - val_loss: 0.4163
Epoch 59/100
11610/11610 [==============================] - 1s 74us/sample - loss: 0.3935 - val_loss: 0.4156
Epoch 60/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3928 - val_loss: 0.4150
Epoch 61/100
11610/11610 [==============================] - 1s 70us/sample - loss: 0.3922 - val_loss: 0.4147
Epoch 62/100
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3913 - val_loss: 0.4137
Epoch 63/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3906 - val_loss: 0.4128
Epoch 64/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3898 - val_loss: 0.4113
Epoch 65/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3894 - val_loss: 0.4121
Epoch 66/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3884 - val_loss: 0.4110
Epoch 67/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3878 - val_loss: 0.4129
Epoch 68/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3873 - val_loss: 0.4109
Epoch 69/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3864 - val_loss: 0.4101
Epoch 70/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3858 - val_loss: 0.4092
Epoch 71/100
11610/11610 [==============================] - 1s 51us/sample - loss: 0.3853 - val_loss: 0.4073
Epoch 72/100
11610/11610 [==============================] - 1s 94us/sample - loss: 0.3847 - val_loss: 0.4076
Epoch 73/100
11610/11610 [==============================] - 1s 98us/sample - loss: 0.3842 - val_loss: 0.4059
Epoch 74/100
11610/11610 [==============================] - 1s 80us/sample - loss: 0.3834 - val_loss: 0.4047
Epoch 75/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3826 - val_loss: 0.4056
Epoch 76/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.3822 - val_loss: 0.4031
Epoch 77/100
11610/11610 [==============================] - 1s 67us/sample - loss: 0.3814 - val_loss: 0.4049
Epoch 78/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3812 - val_loss: 0.4029
Epoch 79/100
11610/11610 [==============================] - 1s 59us/sample - loss: 0.3800 - val_loss: 0.4022
Epoch 80/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3794 - val_loss: 0.4004
Epoch 81/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3793 - val_loss: 0.4004
Epoch 82/100
11610/11610 [==============================] - 1s 79us/sample - loss: 0.3784 - val_loss: 0.3998
Epoch 83/100
11610/11610 [==============================] - 1s 77us/sample - loss: 0.3778 - val_loss: 0.3988
Epoch 84/100
11610/11610 [==============================] - 1s 65us/sample - loss: 0.3770 - val_loss: 0.3981
Epoch 85/100
11610/11610 [==============================] - 1s 69us/sample - loss: 0.3764 - val_loss: 0.3987
Epoch 86/100
11610/11610 [==============================] - 1s 83us/sample - loss: 0.3756 - val_loss: 0.3970
Epoch 87/100
11610/11610 [==============================] - 1s 89us/sample - loss: 0.3751 - val_loss: 0.3969
Epoch 88/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.3747 - val_loss: 0.3950
Epoch 89/100
11610/11610 [==============================] - 1s 90us/sample - loss: 0.3740 - val_loss: 0.3951
Epoch 90/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.3733 - val_loss: 0.3945
Epoch 91/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3727 - val_loss: 0.3937
Epoch 92/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3721 - val_loss: 0.3937
Epoch 93/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3717 - val_loss: 0.3932
Epoch 94/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3709 - val_loss: 0.3930
Epoch 95/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3703 - val_loss: 0.3919
Epoch 96/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3695 - val_loss: 0.3921
Epoch 97/100
11610/11610 [==============================] - 1s 52us/sample - loss: 0.3692 - val_loss: 0.3909
Epoch 98/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3687 - val_loss: 0.3892
Epoch 99/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3676 - val_loss: 0.3902
Epoch 100/100
11610/11610 [==============================] - 1s 53us/sample - loss: 0.3677 - val_loss: 0.3888
5160/1 [=======================================================] - 0s 24us/sample - loss: 0.4975
0.3788368522658829

有了多輸入就會有多輸出

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, optimizers, models
from sklearn.datasets import fetch_california_housing
import pprint
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":

    housing = fetch_california_housing()
    # print(housing.DESCR)
    # print(housing.data.shape)
    # print(housing.target.shape)
    # pprint.pprint(housing.data[0:5])
    # pprint.pprint(housing.target[0:5])
    X_train_all, X_test, y_train_all, y_test = train_test_split(housing.data, housing.target, random_state=7)
    X_train, X_valid, y_train, y_valid = train_test_split(X_train_all, y_train_all, random_state=11)
    # print(X_train.shape, y_train.shape)
    # print(X_valid.shape, y_valid.shape)
    # print(X_test.shape, y_test.shape)
    scaler = StandardScaler()
    # 歸一化
    X_train = scaler.fit_transform(X_train)
    X_valid = scaler.transform(X_valid)
    X_test = scaler.transform(X_test)
    # model = keras.models.Sequential([
    #     layers.Dense(30, activation='relu', input_shape=X_train.shape[1:]),
    #     layers.Dense(1)
    # ])
    # 函數式API Deep模型
    # input = layers.Input(shape=X_train.shape[1:])
    # hidden1 = layers.Dense(30, activation='relu')(input)
    # hidden2 = layers.Dense(30, activation='relu')(hidden1)
    # # input是Wide模型的輸入，hidden2是Deep模型的輸出
    # concat = layers.concatenate([input, hidden2])
    # output = layers.Dense(1)(concat)
    # model = models.Model(inputs=[input], outputs=[output])
    # 子類API，父類爲keras.models.Model
    # class WideDeepModel(models.Model):
    #
    #     def __init__(self):
    #         super(WideDeepModel, self).__init__()
    #         # 定義模型的層次
    #         self.hidden1_layer = layers.Dense(30, activation='relu')
    #         self.hidden2_layer = layers.Dense(30, activation='relu')
    #         self.output_layer = layers.Dense(1)
    #
    #
    #     def call(self, input):
    #         # 完成模型的正向計算
    #         hidden1 = self.hidden1_layer(input)
    #         hidden2 = self.hidden2_layer(hidden1)
    #         concat = layers.concatenate([input, hidden2])
    #         output = self.output_layer(concat)
    #         return output
    #
    # model = WideDeepModel()
    # # 也可以使用層堆疊
    # # model = models.Sequential([
    # #     WideDeepModel()
    # # ])
    # model.build(input_shape=(None, 8))
    # 多輸入 wide模型輸入前5個特徵，deep模型輸入後6個特徵
    input_wide = layers.Input(shape=[5])
    input_deep = layers.Input(shape=[6])
    hidden1 = layers.Dense(30, activation='relu')(input_deep)
    hidden2 = layers.Dense(30, activation='relu')(hidden1)
    concat = layers.concatenate([input_wide, hidden2])
    # 多輸出
    output1 = layers.Dense(1)(concat)
    output2 = layers.Dense(1)(hidden2)
    model = models.Model(inputs=[input_wide, input_deep], outputs=[output1, output2])
    print(model.summary())
    # 使用均方誤差損失函數和隨機梯度下降法
    model.compile(loss='mean_squared_error', optimizer=optimizers.SGD(learning_rate=0.001))
    callbacks = [keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3)]
    X_train_wide = X_train[:, :5]
    X_train_deep = X_train[:, 2:]
    X_valid_wide = X_valid[:, :5]
    X_valid_deep = X_valid[:, 2:]
    X_test_wide = X_test[:, :5]
    X_test_deep = X_test[:, 2:]
    history = model.fit([X_train_wide, X_train_deep], [y_train, y_train],
                        validation_data=([X_valid_wide, X_valid_deep], [y_valid, y_valid]),
                        epochs=100, callbacks=callbacks)
    print(model.evaluate([X_test_wide, X_test_deep], [y_test, y_test]))

    def plot_learning_curves(history):
        # 學習曲線
        pd.DataFrame(history.history).plot(figsize=(8, 5))
        plt.grid(True)
        plt.gca().set_ylim(0, 1)
        plt.show()

    plot_learning_curves(history)

運行結果

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 6)]          0                                            
__________________________________________________________________________________________________
dense (Dense)                   (None, 30)           210         input_2[0][0]                    
__________________________________________________________________________________________________
input_1 (InputLayer)            [(None, 5)]          0                                            
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 30)           930         dense[0][0]                      
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 35)           0           input_1[0][0]                    
                                                                 dense_1[0][0]                    
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 1)            36          concatenate[0][0]                
__________________________________________________________________________________________________
dense_3 (Dense)                 (None, 1)            31          dense_1[0][0]                    
==================================================================================================
Total params: 1,207
Trainable params: 1,207
Non-trainable params: 0
__________________________________________________________________________________________________
None
Train on 11610 samples, validate on 3870 samples
Epoch 1/100
11610/11610 [==============================] - 2s 151us/sample - loss: 4.6904 - dense_2_loss: 2.2887 - dense_3_loss: 2.4007 - val_loss: 2.2365 - val_dense_2_loss: 0.8763 - val_dense_3_loss: 1.3602
Epoch 2/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.8804 - dense_2_loss: 0.7132 - dense_3_loss: 1.1668 - val_loss: 1.8578 - val_dense_2_loss: 0.6893 - val_dense_3_loss: 1.1685
Epoch 3/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.6375 - dense_2_loss: 0.6051 - dense_3_loss: 1.0320 - val_loss: 1.6503 - val_dense_2_loss: 0.6125 - val_dense_3_loss: 1.0377
Epoch 4/100
11610/11610 [==============================] - 1s 63us/sample - loss: 1.4545 - dense_2_loss: 0.5502 - dense_3_loss: 0.9039 - val_loss: 1.4679 - val_dense_2_loss: 0.5670 - val_dense_3_loss: 0.9009
Epoch 5/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.3185 - dense_2_loss: 0.5166 - dense_3_loss: 0.8021 - val_loss: 1.3543 - val_dense_2_loss: 0.5365 - val_dense_3_loss: 0.8177
Epoch 6/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.2300 - dense_2_loss: 0.4951 - dense_3_loss: 0.7349 - val_loss: 1.2700 - val_dense_2_loss: 0.5162 - val_dense_3_loss: 0.7536
Epoch 7/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.1740 - dense_2_loss: 0.4805 - dense_3_loss: 0.6936 - val_loss: 1.2194 - val_dense_2_loss: 0.5032 - val_dense_3_loss: 0.7161
Epoch 8/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.1349 - dense_2_loss: 0.4695 - dense_3_loss: 0.6651 - val_loss: 1.1812 - val_dense_2_loss: 0.4888 - val_dense_3_loss: 0.6923
Epoch 9/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.1066 - dense_2_loss: 0.4612 - dense_3_loss: 0.6452 - val_loss: 1.1580 - val_dense_2_loss: 0.4876 - val_dense_3_loss: 0.6703
Epoch 10/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0847 - dense_2_loss: 0.4549 - dense_3_loss: 0.6300 - val_loss: 1.1370 - val_dense_2_loss: 0.4784 - val_dense_3_loss: 0.6585
Epoch 11/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0667 - dense_2_loss: 0.4483 - dense_3_loss: 0.6183 - val_loss: 1.1159 - val_dense_2_loss: 0.4717 - val_dense_3_loss: 0.6441
Epoch 12/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0524 - dense_2_loss: 0.4440 - dense_3_loss: 0.6086 - val_loss: 1.1014 - val_dense_2_loss: 0.4640 - val_dense_3_loss: 0.6373
Epoch 13/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0388 - dense_2_loss: 0.4397 - dense_3_loss: 0.5995 - val_loss: 1.0919 - val_dense_2_loss: 0.4677 - val_dense_3_loss: 0.6241
Epoch 14/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0277 - dense_2_loss: 0.4357 - dense_3_loss: 0.5921 - val_loss: 1.0758 - val_dense_2_loss: 0.4573 - val_dense_3_loss: 0.6184
Epoch 15/100
11610/11610 [==============================] - 1s 61us/sample - loss: 1.0185 - dense_2_loss: 0.4323 - dense_3_loss: 0.5861 - val_loss: 1.0668 - val_dense_2_loss: 0.4580 - val_dense_3_loss: 0.6087
Epoch 16/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0102 - dense_2_loss: 0.4295 - dense_3_loss: 0.5807 - val_loss: 1.0546 - val_dense_2_loss: 0.4518 - val_dense_3_loss: 0.6027
Epoch 17/100
11610/11610 [==============================] - 1s 62us/sample - loss: 1.0019 - dense_2_loss: 0.4267 - dense_3_loss: 0.5751 - val_loss: 1.0487 - val_dense_2_loss: 0.4512 - val_dense_3_loss: 0.5974
Epoch 18/100
11610/11610 [==============================] - 1s 84us/sample - loss: 0.9950 - dense_2_loss: 0.4239 - dense_3_loss: 0.5714 - val_loss: 1.0379 - val_dense_2_loss: 0.4465 - val_dense_3_loss: 0.5913
Epoch 19/100
11610/11610 [==============================] - 1s 109us/sample - loss: 0.9884 - dense_2_loss: 0.4213 - dense_3_loss: 0.5668 - val_loss: 1.0312 - val_dense_2_loss: 0.4447 - val_dense_3_loss: 0.5864
Epoch 20/100
11610/11610 [==============================] - 1s 82us/sample - loss: 0.9820 - dense_2_loss: 0.4188 - dense_3_loss: 0.5630 - val_loss: 1.0274 - val_dense_2_loss: 0.4428 - val_dense_3_loss: 0.5844
Epoch 21/100
11610/11610 [==============================] - 1s 68us/sample - loss: 0.9757 - dense_2_loss: 0.4165 - dense_3_loss: 0.5590 - val_loss: 1.0197 - val_dense_2_loss: 0.4413 - val_dense_3_loss: 0.5783
Epoch 22/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.9690 - dense_2_loss: 0.4140 - dense_3_loss: 0.5547 - val_loss: 1.0125 - val_dense_2_loss: 0.4372 - val_dense_3_loss: 0.5752
Epoch 23/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9633 - dense_2_loss: 0.4119 - dense_3_loss: 0.5514 - val_loss: 1.0073 - val_dense_2_loss: 0.4348 - val_dense_3_loss: 0.5723
Epoch 24/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9581 - dense_2_loss: 0.4105 - dense_3_loss: 0.5483 - val_loss: 1.0053 - val_dense_2_loss: 0.4386 - val_dense_3_loss: 0.5666
Epoch 25/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9543 - dense_2_loss: 0.4085 - dense_3_loss: 0.5457 - val_loss: 0.9971 - val_dense_2_loss: 0.4320 - val_dense_3_loss: 0.5650
Epoch 26/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9473 - dense_2_loss: 0.4054 - dense_3_loss: 0.5417 - val_loss: 0.9948 - val_dense_2_loss: 0.4335 - val_dense_3_loss: 0.5612
Epoch 27/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9427 - dense_2_loss: 0.4044 - dense_3_loss: 0.5383 - val_loss: 0.9884 - val_dense_2_loss: 0.4279 - val_dense_3_loss: 0.5604
Epoch 28/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9392 - dense_2_loss: 0.4028 - dense_3_loss: 0.5362 - val_loss: 0.9836 - val_dense_2_loss: 0.4280 - val_dense_3_loss: 0.5555
Epoch 29/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9333 - dense_2_loss: 0.4003 - dense_3_loss: 0.5329 - val_loss: 0.9803 - val_dense_2_loss: 0.4257 - val_dense_3_loss: 0.5544
Epoch 30/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9297 - dense_2_loss: 0.3990 - dense_3_loss: 0.5304 - val_loss: 0.9735 - val_dense_2_loss: 0.4229 - val_dense_3_loss: 0.5505
Epoch 31/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.9257 - dense_2_loss: 0.3978 - dense_3_loss: 0.5278 - val_loss: 0.9735 - val_dense_2_loss: 0.4216 - val_dense_3_loss: 0.5518
Epoch 32/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9202 - dense_2_loss: 0.3956 - dense_3_loss: 0.5247 - val_loss: 0.9665 - val_dense_2_loss: 0.4209 - val_dense_3_loss: 0.5455
Epoch 33/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9178 - dense_2_loss: 0.3946 - dense_3_loss: 0.5229 - val_loss: 0.9594 - val_dense_2_loss: 0.4180 - val_dense_3_loss: 0.5413
Epoch 34/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9121 - dense_2_loss: 0.3925 - dense_3_loss: 0.5197 - val_loss: 0.9559 - val_dense_2_loss: 0.4147 - val_dense_3_loss: 0.5411
Epoch 35/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9080 - dense_2_loss: 0.3910 - dense_3_loss: 0.5168 - val_loss: 0.9536 - val_dense_2_loss: 0.4119 - val_dense_3_loss: 0.5415
Epoch 36/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9046 - dense_2_loss: 0.3900 - dense_3_loss: 0.5146 - val_loss: 0.9511 - val_dense_2_loss: 0.4139 - val_dense_3_loss: 0.5371
Epoch 37/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.9000 - dense_2_loss: 0.3882 - dense_3_loss: 0.5118 - val_loss: 0.9507 - val_dense_2_loss: 0.4199 - val_dense_3_loss: 0.5307
Epoch 38/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8990 - dense_2_loss: 0.3884 - dense_3_loss: 0.5110 - val_loss: 0.9416 - val_dense_2_loss: 0.4145 - val_dense_3_loss: 0.5270
Epoch 39/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8928 - dense_2_loss: 0.3859 - dense_3_loss: 0.5071 - val_loss: 0.9357 - val_dense_2_loss: 0.4092 - val_dense_3_loss: 0.5264
Epoch 40/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8882 - dense_2_loss: 0.3838 - dense_3_loss: 0.5044 - val_loss: 0.9325 - val_dense_2_loss: 0.4092 - val_dense_3_loss: 0.5233
Epoch 41/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8846 - dense_2_loss: 0.3831 - dense_3_loss: 0.5016 - val_loss: 0.9277 - val_dense_2_loss: 0.4055 - val_dense_3_loss: 0.5221
Epoch 42/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8831 - dense_2_loss: 0.3827 - dense_3_loss: 0.5002 - val_loss: 0.9259 - val_dense_2_loss: 0.4053 - val_dense_3_loss: 0.5205
Epoch 43/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8804 - dense_2_loss: 0.3817 - dense_3_loss: 0.4987 - val_loss: 0.9199 - val_dense_2_loss: 0.4035 - val_dense_3_loss: 0.5163
Epoch 44/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8737 - dense_2_loss: 0.3792 - dense_3_loss: 0.4946 - val_loss: 0.9165 - val_dense_2_loss: 0.4009 - val_dense_3_loss: 0.5154
Epoch 45/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8698 - dense_2_loss: 0.3776 - dense_3_loss: 0.4920 - val_loss: 0.9099 - val_dense_2_loss: 0.3978 - val_dense_3_loss: 0.5120
Epoch 46/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8689 - dense_2_loss: 0.3779 - dense_3_loss: 0.4910 - val_loss: 0.9068 - val_dense_2_loss: 0.3983 - val_dense_3_loss: 0.5084
Epoch 47/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8622 - dense_2_loss: 0.3751 - dense_3_loss: 0.4870 - val_loss: 0.9008 - val_dense_2_loss: 0.3938 - val_dense_3_loss: 0.5069
Epoch 48/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8588 - dense_2_loss: 0.3739 - dense_3_loss: 0.4848 - val_loss: 0.8974 - val_dense_2_loss: 0.3932 - val_dense_3_loss: 0.5041
Epoch 49/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8582 - dense_2_loss: 0.3740 - dense_3_loss: 0.4840 - val_loss: 0.8923 - val_dense_2_loss: 0.3919 - val_dense_3_loss: 0.5003
Epoch 50/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8522 - dense_2_loss: 0.3718 - dense_3_loss: 0.4803 - val_loss: 0.8924 - val_dense_2_loss: 0.3922 - val_dense_3_loss: 0.5000
Epoch 51/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.8503 - dense_2_loss: 0.3715 - dense_3_loss: 0.4789 - val_loss: 0.8888 - val_dense_2_loss: 0.3900 - val_dense_3_loss: 0.4986
Epoch 52/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.8487 - dense_2_loss: 0.3714 - dense_3_loss: 0.4772 - val_loss: 0.8834 - val_dense_2_loss: 0.3910 - val_dense_3_loss: 0.4923
Epoch 53/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8443 - dense_2_loss: 0.3696 - dense_3_loss: 0.4746 - val_loss: 0.8799 - val_dense_2_loss: 0.3882 - val_dense_3_loss: 0.4915
Epoch 54/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8423 - dense_2_loss: 0.3691 - dense_3_loss: 0.4731 - val_loss: 0.8745 - val_dense_2_loss: 0.3870 - val_dense_3_loss: 0.4874
Epoch 55/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8385 - dense_2_loss: 0.3679 - dense_3_loss: 0.4705 - val_loss: 0.8729 - val_dense_2_loss: 0.3859 - val_dense_3_loss: 0.4870
Epoch 56/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8307 - dense_2_loss: 0.3647 - dense_3_loss: 0.4659 - val_loss: 0.8685 - val_dense_2_loss: 0.3823 - val_dense_3_loss: 0.4861
Epoch 57/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.8316 - dense_2_loss: 0.3654 - dense_3_loss: 0.4663 - val_loss: 0.8642 - val_dense_2_loss: 0.3818 - val_dense_3_loss: 0.4823
Epoch 58/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8277 - dense_2_loss: 0.3640 - dense_3_loss: 0.4636 - val_loss: 0.8592 - val_dense_2_loss: 0.3810 - val_dense_3_loss: 0.4781
Epoch 59/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8323 - dense_2_loss: 0.3661 - dense_3_loss: 0.4661 - val_loss: 0.8578 - val_dense_2_loss: 0.3809 - val_dense_3_loss: 0.4768
Epoch 60/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8185 - dense_2_loss: 0.3608 - dense_3_loss: 0.4580 - val_loss: 0.8512 - val_dense_2_loss: 0.3777 - val_dense_3_loss: 0.4734
Epoch 61/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8188 - dense_2_loss: 0.3609 - dense_3_loss: 0.4577 - val_loss: 0.8514 - val_dense_2_loss: 0.3779 - val_dense_3_loss: 0.4734
Epoch 62/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8165 - dense_2_loss: 0.3608 - dense_3_loss: 0.4556 - val_loss: 0.8442 - val_dense_2_loss: 0.3746 - val_dense_3_loss: 0.4695
Epoch 63/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.8118 - dense_2_loss: 0.3589 - dense_3_loss: 0.4530 - val_loss: 0.8407 - val_dense_2_loss: 0.3724 - val_dense_3_loss: 0.4682
Epoch 64/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8097 - dense_2_loss: 0.3587 - dense_3_loss: 0.4512 - val_loss: 0.8400 - val_dense_2_loss: 0.3758 - val_dense_3_loss: 0.4640
Epoch 65/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8025 - dense_2_loss: 0.3552 - dense_3_loss: 0.4471 - val_loss: 0.8353 - val_dense_2_loss: 0.3721 - val_dense_3_loss: 0.4631
Epoch 66/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7997 - dense_2_loss: 0.3542 - dense_3_loss: 0.4456 - val_loss: 0.8299 - val_dense_2_loss: 0.3707 - val_dense_3_loss: 0.4591
Epoch 67/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.8013 - dense_2_loss: 0.3553 - dense_3_loss: 0.4458 - val_loss: 0.8289 - val_dense_2_loss: 0.3691 - val_dense_3_loss: 0.4597
Epoch 68/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7985 - dense_2_loss: 0.3548 - dense_3_loss: 0.4436 - val_loss: 0.8209 - val_dense_2_loss: 0.3677 - val_dense_3_loss: 0.4532
Epoch 69/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7968 - dense_2_loss: 0.3538 - dense_3_loss: 0.4428 - val_loss: 0.8211 - val_dense_2_loss: 0.3682 - val_dense_3_loss: 0.4527
Epoch 70/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7889 - dense_2_loss: 0.3510 - dense_3_loss: 0.4378 - val_loss: 0.8174 - val_dense_2_loss: 0.3667 - val_dense_3_loss: 0.4506
Epoch 71/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7877 - dense_2_loss: 0.3506 - dense_3_loss: 0.4373 - val_loss: 0.8115 - val_dense_2_loss: 0.3632 - val_dense_3_loss: 0.4482
Epoch 72/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7845 - dense_2_loss: 0.3496 - dense_3_loss: 0.4354 - val_loss: 0.8157 - val_dense_2_loss: 0.3665 - val_dense_3_loss: 0.4490
Epoch 73/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7818 - dense_2_loss: 0.3487 - dense_3_loss: 0.4332 - val_loss: 0.8084 - val_dense_2_loss: 0.3622 - val_dense_3_loss: 0.4461
Epoch 74/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.7843 - dense_2_loss: 0.3501 - dense_3_loss: 0.4348 - val_loss: 0.8084 - val_dense_2_loss: 0.3620 - val_dense_3_loss: 0.4463
Epoch 75/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7785 - dense_2_loss: 0.3474 - dense_3_loss: 0.4311 - val_loss: 0.8079 - val_dense_2_loss: 0.3622 - val_dense_3_loss: 0.4457
Epoch 76/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7765 - dense_2_loss: 0.3470 - dense_3_loss: 0.4295 - val_loss: 0.8049 - val_dense_2_loss: 0.3639 - val_dense_3_loss: 0.4409
Epoch 77/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7735 - dense_2_loss: 0.3460 - dense_3_loss: 0.4276 - val_loss: 0.8012 - val_dense_2_loss: 0.3612 - val_dense_3_loss: 0.4399
Epoch 78/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7730 - dense_2_loss: 0.3459 - dense_3_loss: 0.4273 - val_loss: 0.8023 - val_dense_2_loss: 0.3603 - val_dense_3_loss: 0.4419
Epoch 79/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7756 - dense_2_loss: 0.3472 - dense_3_loss: 0.4284 - val_loss: 0.7986 - val_dense_2_loss: 0.3576 - val_dense_3_loss: 0.4409
Epoch 80/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7761 - dense_2_loss: 0.3475 - dense_3_loss: 0.4286 - val_loss: 0.7979 - val_dense_2_loss: 0.3598 - val_dense_3_loss: 0.4380
Epoch 81/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7733 - dense_2_loss: 0.3466 - dense_3_loss: 0.4267 - val_loss: 0.7936 - val_dense_2_loss: 0.3593 - val_dense_3_loss: 0.4341
Epoch 82/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7655 - dense_2_loss: 0.3430 - dense_3_loss: 0.4224 - val_loss: 0.7945 - val_dense_2_loss: 0.3580 - val_dense_3_loss: 0.4363
Epoch 83/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7669 - dense_2_loss: 0.3442 - dense_3_loss: 0.4224 - val_loss: 0.7913 - val_dense_2_loss: 0.3572 - val_dense_3_loss: 0.4340
Epoch 84/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7617 - dense_2_loss: 0.3424 - dense_3_loss: 0.4194 - val_loss: 0.7889 - val_dense_2_loss: 0.3561 - val_dense_3_loss: 0.4326
Epoch 85/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7601 - dense_2_loss: 0.3414 - dense_3_loss: 0.4186 - val_loss: 0.7866 - val_dense_2_loss: 0.3560 - val_dense_3_loss: 0.4304
Epoch 86/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7578 - dense_2_loss: 0.3408 - dense_3_loss: 0.4169 - val_loss: 0.7873 - val_dense_2_loss: 0.3562 - val_dense_3_loss: 0.4309
Epoch 87/100
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7631 - dense_2_loss: 0.3432 - dense_3_loss: 0.4198 - val_loss: 0.7842 - val_dense_2_loss: 0.3540 - val_dense_3_loss: 0.4302
Epoch 88/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7603 - dense_2_loss: 0.3422 - dense_3_loss: 0.4181 - val_loss: 0.7832 - val_dense_2_loss: 0.3549 - val_dense_3_loss: 0.4282
Epoch 89/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7542 - dense_2_loss: 0.3391 - dense_3_loss: 0.4148 - val_loss: 0.7893 - val_dense_2_loss: 0.3571 - val_dense_3_loss: 0.4320
Epoch 90/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7528 - dense_2_loss: 0.3391 - dense_3_loss: 0.4139 - val_loss: 0.7808 - val_dense_2_loss: 0.3533 - val_dense_3_loss: 0.4274
Epoch 91/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7523 - dense_2_loss: 0.3385 - dense_3_loss: 0.4137 - val_loss: 0.7778 - val_dense_2_loss: 0.3532 - val_dense_3_loss: 0.4245
Epoch 92/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7514 - dense_2_loss: 0.3390 - dense_3_loss: 0.4130 - val_loss: 0.7748 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4241
Epoch 93/100
11610/11610 [==============================] - 1s 63us/sample - loss: 0.7479 - dense_2_loss: 0.3371 - dense_3_loss: 0.4106 - val_loss: 0.7771 - val_dense_2_loss: 0.3529 - val_dense_3_loss: 0.4241
Epoch 94/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7477 - dense_2_loss: 0.3371 - dense_3_loss: 0.4105 - val_loss: 0.7711 - val_dense_2_loss: 0.3500 - val_dense_3_loss: 0.4210
Epoch 95/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7519 - dense_2_loss: 0.3391 - dense_3_loss: 0.4126 - val_loss: 0.7720 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4212
Epoch 96/100
11610/11610 [==============================] - 1s 62us/sample - loss: 0.7544 - dense_2_loss: 0.3404 - dense_3_loss: 0.4140 - val_loss: 0.7853 - val_dense_2_loss: 0.3559 - val_dense_3_loss: 0.4293
Epoch 97/100
11610/11610 [==============================] - 1s 81us/sample - loss: 0.7543 - dense_2_loss: 0.3404 - dense_3_loss: 0.4140 - val_loss: 0.7712 - val_dense_2_loss: 0.3506 - val_dense_3_loss: 0.4205
Epoch 98/100
11610/11610 [==============================] - 1s 95us/sample - loss: 0.7540 - dense_2_loss: 0.3405 - dense_3_loss: 0.4140 - val_loss: 0.7728 - val_dense_2_loss: 0.3520 - val_dense_3_loss: 0.4207
Epoch 99/100
11610/11610 [==============================] - 1s 64us/sample - loss: 0.7424 - dense_2_loss: 0.3352 - dense_3_loss: 0.4070 - val_loss: 0.7710 - val_dense_2_loss: 0.3507 - val_dense_3_loss: 0.4203
5160/1 [====================================================] - 0s 29us/sample - loss: 0.8831 - dense_2_loss: 0.3513 - dense_3_loss: 0.4160
[0.7661929805149403, 0.35130394, 0.41597202]

這裏會有三組線，一個是總的，一個是第一個輸出，一個是第二個輸出。

基礎API

我們現在來看一下tensorflow中最基本的常量操作

import tensorflow as tf
import numpy as np

if __name__ == "__main__":

    # 定義矩陣常量
    t = tf.constant([[1., 2., 3.], [4., 5., 6.]])
    print(t)
    print(t[:, 1:])
    print(t[:, 1])
    # 算子
    print(t + 10)
    # 計算平方
    print(tf.square(t))
    # 計算它本身和它轉置的點乘
    print(t @ tf.transpose(t))
    # 獲取numpy的值
    print(t.numpy())
    # 作爲numpy的輸入
    print(np.square(t))
    # 將numpy對象轉成tensorflow對象
    np_t = np.array([[1., 2., 3.], [4., 5., 6.]])
    print(tf.constant(np_t))
    # 標量
    num = tf.constant(2.718)
    print(num)
    print(num.numpy())
    print(num.shape)
    # 字符串
    t = tf.constant("coffee")
    print(t)
    print(tf.strings.length(t))
    print(tf.strings.length(t, unit='UTF8_CHAR'))
    # 解碼成utf8編碼
    print(tf.strings.unicode_decode(t, 'UTF8'))
    # 字符串數組
    t = tf.constant(['cafe', 'coffee', '咖啡'])
    print(t)
    print(tf.strings.length(t, unit='UTF8_CHAR'))
    print(tf.strings.unicode_decode(t, 'UTF8'))
    # ragged tensor不等長的數組
    r = tf.ragged.constant([[11, 12], [21, 22, 23], [], [41]])
    print(r)
    print(r[1])
    print(r[1:3])
    # ragged tensor算子
    r2 = tf.ragged.constant([[51, 52], [], [71], [42, 43]])
    # 在第0個維度上進行拼接
    print(tf.concat([r, r2], axis=0))
    # 在第1個維度上進行拼接,如果第一個維度的數量不相等，會報錯
    print(tf.concat([r, r2], axis=1))
    # 將ragged tensor轉化爲普通tensor，空缺位置補0
    print(r.to_tensor())

運行結果

tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[2. 3.]
 [5. 6.]], shape=(2, 2), dtype=float32)
tf.Tensor([2. 5.], shape=(2,), dtype=float32)
tf.Tensor(
[[11. 12. 13.]
 [14. 15. 16.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[ 1.  4.  9.]
 [16. 25. 36.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[14. 32.]
 [32. 77.]], shape=(2, 2), dtype=float32)
[[1. 2. 3.]
 [4. 5. 6.]]
[[ 1.  4.  9.]
 [16. 25. 36.]]
tf.Tensor(
[[1. 2. 3.]
 [4. 5. 6.]], shape=(2, 3), dtype=float64)
tf.Tensor(2.718, shape=(), dtype=float32)
2.718
()
tf.Tensor(b'coffee', shape=(), dtype=string)
tf.Tensor(6, shape=(), dtype=int32)
tf.Tensor(6, shape=(), dtype=int32)
tf.Tensor([ 99 111 102 102 101 101], shape=(6,), dtype=int32)
tf.Tensor([b'cafe' b'coffee' b'\xe5\x92\x96\xe5\x95\xa1'], shape=(3,), dtype=string)
tf.Tensor([4 6 2], shape=(3,), dtype=int32)
<tf.RaggedTensor [[99, 97, 102, 101], [99, 111, 102, 102, 101, 101], [21654, 21857]]>
<tf.RaggedTensor [[11, 12], [21, 22, 23], [], [41]]>
tf.Tensor([21 22 23], shape=(3,), dtype=int32)
<tf.RaggedTensor [[21, 22, 23], []]>
<tf.RaggedTensor [[11, 12], [21, 22, 23], [], [41], [51, 52], [], [71], [42, 43]]>
<tf.RaggedTensor [[11, 12, 51, 52], [21, 22, 23], [71], [41, 42, 43]]>
tf.Tensor(
[[11 12  0]
 [21 22 23]
 [ 0  0  0]
 [41  0  0]], shape=(4, 3), dtype=int32)