011-lab-002. mnist 99% with cnn
<![CDATA[
# @
# lab-11-1-mnist_cnn.py
import tensorflow as tf
import random
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(777)
mnist=input_data.read_data_sets("MNIST_data/",one_hot=True)
learning_rate=0.001
training_epochs=15
batch_size=100
# n number of images
# [n,784]
X_placeholder_node=tf.placeholder(tf.float32,[None,784])
# To input X_placeholder_node as image,
# you need to reshape it
# n number of images, 28x28, 1 color
reshaped_X_img_node=tf.reshape(X_placeholder_node,[-1,28,28,1])
# Final output shape [n,10]
Y_placeholder_node=tf.placeholder(tf.float32,[None,10])
# shape of image is [-1,28,28,1]
# You can define filter size whatever you want
# You will define filter size as 3*3
# color information has 1
# 32 filters which will create 32 ouputs from one convolution layer
# [3,3,1,32]=[fiter size,filter size,color,number of filters]
first_filter_W_variable_node=tf.Variable(tf.random_normal([3,3,1,32],stddev=0.01))
# After convolution layer,
# output tensor shape will be (-1,28,28,32)
output_after_convolution_layer_node\
=tf.nn.conv2d(reshaped_X_img_node,first_filter_W_variable_node,strides=[1,1,1,1],padding='SAME')
output_after_convolution_layer_after_relu_node=tf.nn.relu(output_after_convolution_layer_node)
# You will perform max pooling
# pooling filter size is 2*2
# strider of pooling filter is 2*2
# padding is same => 14*14*32images
output_after_max_pooling_node\
=tf.nn.max_pool(\
output_after_convolution_layer_after_relu_node\
,ksize=[1,2,2,1]\
,strides=[1,2,2,1]\
,padding='SAME')
# < Tensor("Conv2D:0",shape=(?,28,28,32),dtype=float32)
# < Tensor("Relu:0",shape=(?,28,28,32),dtype=float32)
# < Tensor("MaxPool:0",shape=(?,14,14,32),dtype=float32)
# < final tensor shape=(14,14,32)
# You will perform 2nd convolution layer
# Input data's shape=(?,14,14,32)
# Filter size will be 3*3
# You will use 64 filters
filter_W_of_convolution_layer2_node=tf.Variable(tf.random_normal([3,3,32,64],stddev=0.01))
convolution_layer2_node=tf.nn.conv2d(output_after_max_pooling_node,filter_W_of_convolution_layer2_node,strides=[1,1,1,1],padding='SAME')
convolution_layer2_after_relu_node=tf.nn.relu(convolution_layer2_node)
convolution_layer2_after_maxpool_node\
=tf.nn.max_pool(convolution_layer2_after_relu_node,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')
# Now, you will have 3d shape
# You should flatten 3d shape,
# to input output data into fully connected layer
# You will create shape [7*7*64]
flatten_data_in_layer2_before_fullyconnectedlayer_node\
=tf.reshape(convolution_layer2_after_maxpool_node,[-1,7*7*64])
# Tensor("Conv2D_1:0",shape=(?,14,14,64),dtype=float32)
# Tensor("Relu_1:0",shape=(?,14,14,64),dtype=float32)
# Tensor("MaxPool_1:0",shape=(?,7,7,64),dtype=float32)
# Tensor("Reshape_1:0",shape=(?,3136),dtype=float32)
# input data's shape is [m,7*7*64]
# output data's shape is [n,10]
W3_variable_in_fullyconnectedlayer_node\
=tf.get_variable(\
"W3_variable_in_fullyconnectedlayer_node"\
,shape=[7*7*64,10]\
,initializer=tf.contrib.layers.xavier_initializer())
b_variable_in_fullyconnectedlayer_node=tf.Variable(tf.random_normal([10]))
logits_node=tf.matmul(flatten_data_in_layer2_before_fullyconnectedlayer_node,W3_variable_in_fullyconnectedlayer_node) + b_variable_in_fullyconnectedlayer_node
cost_function_node\
=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits_node,labels=Y_placeholder_node))
adam_optimizer_node=tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost_function)
sess_object=tf.Session()
sess_object.run(tf.global_variables_initializer())
print('Learning started. It takes sometime.')
for epoch in range(training_epochs):
average_cost_value=0
total_batch=int(mnist.train.num_examples/batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
feed_dict={X_placeholder_node:batch_xs,Y_placeholder_node:batch_ys}
c,_=sess_object.run([cost_function,adam_optimizer_node],feed_dict=feed_dict)
average_cost_value+=c/total_batch
print('Epoch:','%04d' % (epoch + 1),'cost_function =','{:.9f}'.format(average_cost_value))
print('Learning Finished!')
# Test model and check accuracy
correct_prediction=tf.equal(tf.argmax(logits,1),tf.argmax(Y_placeholder_node,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
print('Accuracy:',sess_object.run(accuracy,feed_dict={X_placeholder_node:mnist.test.images, Y_placeholder_node:mnist.test.labels}))
r=random.randint(0,mnist.test.num_examples - 1)
print("Label:",sess_object.run(tf.argmax(mnist.test.labels[r:r+1],1)))
print("Prediction:",sess_object.run(tf.argmax(logits,1),feed_dict={X_placeholder_node:mnist.test.images[r:r + 1]}))
# plt.imshow(mnist.test.images[r:r + 1].
# reshape(28,28),cmap='Greys',interpolation='nearest')
# plt.show()
# Epoch:0001 cost_function=0.340291267
# Epoch:0002 cost_function=0.090731326
# Epoch:0003 cost_function=0.064477619
# Epoch:0004 cost_function=0.050683064
# Epoch:0005 cost_function=0.041864835
# Epoch:0006 cost_function=0.035760704
# Epoch:0007 cost_function=0.030572132
# Epoch:0008 cost_function=0.026207981
# Epoch:0009 cost_function=0.022622454
# Epoch:0010 cost_function=0.019055919
# Epoch:0011 cost_function=0.017758641
# Epoch:0012 cost_function=0.014156652
# Epoch:0013 cost_function=0.012397016
# Epoch:0014 cost_function=0.010693789
# Epoch:0015 cost_function=0.009469977
# Learning Finished!
# Accuracy:0.9885
DeepLearningZeroToAll/lab-11-2-mnist_deep_cnn.py
import tensorflow as tf
import random
# import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(777)
mnist=input_data.read_data_sets("MNIST_data/",one_hot=True)
learning_rate=0.001
training_epochs=15
batch_size=100
# dropout rate: 0.7~0.5 on training,
# but should be 1 for testing
dropout_placeholder_node=tf.placeholder(tf.float32)
# [n,784]
X_placeholder_node=tf.placeholder(tf.float32,[None,784])
# img 28x28x1 (black/white)
reshaped_X_img_node=tf.reshape(X_placeholder_node,[-1,28,28,1])
# [n,10]
Y_placeholder_node=tf.placeholder(tf.float32,[None,10])
# [3,3,1,32]
# filter size is 3*3*1(coler)
# You will use 32 filters
first_filter_W_variable_node=tf.Variable(tf.random_normal([3,3,1,32],stddev=0.01))
# shape of image after convolution layer will be (?,28,28,32)
# shape of image after convolution layer after pooling will be (?,14,14,32)
output_after_convolution_layer_node\
=tf.nn.conv2d(reshaped_X_img_node,first_filter_W_variable_node,strides=[1,1,1,1],padding='SAME')
output_after_relu_node=tf.nn.relu(output_after_convolution_layer_node)
output_after_pooling_node\
=tf.nn.max_pool(\
output_after_relu_node\
,ksize=[1,2,2,1]\
,strides=[1,2,2,1]\
,padding='SAME')
output_after_dropout_node=tf.nn.dropout(output_after_pooling_node,keep_prob=dropout_placeholder_node)
'''
Tensor("Conv2D:0",shape=(?,28,28,32),dtype=float32)
Tensor("Relu:0",shape=(?,28,28,32),dtype=float32)
Tensor("MaxPool:0",shape=(?,14,14,32),dtype=float32)
Tensor("dropout/mul:0",shape=(?,14,14,32),dtype=float32)
'''
# In convolution layer2, shape of input image is (?,14,14,32)
filter_W_of_convolution_layer2_node=tf.Variable(tf.random_normal([3,3,32,64],stddev=0.01))
# Conv ->(?,14,14,64)
# Pool ->(?,7,7,64)
output_after_convolution_layer2_node=tf.nn.conv2d(\
output_after_dropout_node\
,filter_W_of_convolution_layer2_node\
,strides=[1,1,1,1]\
,padding='SAME')
output_after_relu_node=tf.nn.relu(output_after_convolution_layer2_node)
output_after_pooling_node\
=tf.nn.max_pool(\
output_after_relu_node\
,ksize=[1,2,2,1]
,strides=[1,2,2,1]\
,padding='SAME')
output_after_dropout_node\
=tf.nn.dropout(output_after_pooling_node,keep_prob=dropout_placeholder_node)
'''
Tensor("Conv2D_1:0",shape=(?,14,14,64),dtype=float32)
Tensor("Relu_1:0",shape=(?,14,14,64),dtype=float32)
Tensor("MaxPool_1:0",shape=(?,7,7,64),dtype=float32)
Tensor("dropout_1/mul:0",shape=(?,7,7,64),dtype=float32)
'''
# In layer 3, shape of input image is (n,7,7,64)
filter_W_of_layer3_node=tf.Variable(tf.random_normal([3,3,64,128],stddev=0.01))
# Conv ->(?,7,7,128)
# Pool ->(?,4,4,128)
# Reshape ->(?,4 * 4 * 128) # Flatten them for FC
output_after_filter_sliding_in_layer3_node\
=tf.nn.conv2d(output_after_dropout_node,filter_W_of_layer3_node,strides=[1,1,1,1],padding='SAME')
output_after_relu_in_layer3_node=tf.nn.relu(output_after_filter_sliding_in_layer3_node)
output_after_pooling_in_layer3_node=tf.nn.max_pool(\
output_after_relu_in_layer3_node\
,ksize=[1,2,2,1]\
,strides=[1,2,2,1]\
,padding='SAME')
output_after_dropout_in_layer3_node\
=tf.nn.dropout(output_after_pooling_in_layer3_node,keep_prob=dropout_placeholder_node)
reshaped_output_in_layer3_node\
=tf.reshape(output_after_dropout_in_layer3_node,[-1,128*4*4])
'''
Tensor("Conv2D_2:0",shape=(?,7,7,128),dtype=float32)
Tensor("Relu_2:0",shape=(?,7,7,128),dtype=float32)
Tensor("MaxPool_2:0",shape=(?,4,4,128),dtype=float32)
Tensor("dropout_2/mul:0",shape=(?,4,4,128),dtype=float32)
Tensor("Reshape_1:0",shape=(?,2048),dtype=float32)
'''
# Layer4 is 1st fully connected layer
# In this layer, input data will be [n,625]
# You will define output data will be [n,625]
# [n,625][?,?]=[n,625]
# [?,?]=[625,625]
W4_variable_in_f_c_layer4_node\
=tf.get_variable(\
"W4_variable_in_f_c_layer4_node"\
,shape=[128*4*4,625]\
,initializer=tf.contrib.layers.xavier_initializer())
# [625] is from [n,625]
b4_variable_in_f_c_layer4_node=tf.Variable(tf.random_normal([625]))
hypothesis_f_after_relu_in_layer4_node=tf.nn.relu(tf.matmul(reshaped_output_in_layer3_node,W4_variable_in_f_c_layer4_node)+b4_variable_in_f_c_layer4_node)
hypothesis_f_after_dropout_in_layer4_node\
=tf.nn.dropout(\
hypothesis_f_after_relu_in_layer4_node\
,keep_prob=dropout_placeholder_node)
'''
Tensor("Relu_3:0",shape=(?,625),dtype=float32)
Tensor("dropout_3/mul:0",shape=(?,625),dtype=float32)
'''
# L5 Final FC 625 inputs -> 10 outputs
# Layer5 is 2nd fully connected layer
# In this layer, input data will be [n,625]
# You will define output data will be [n,10]
# [n,625][?,?]=[n,10]
# [?,?]=[625,10]
W5_variable_in_f_c_layer4_node\
=tf.get_variable(\
"W5_variable_in_f_c_layer4_node"\
,shape=[625,10]\
,initializer=tf.contrib.layers.xavier_initializer())
b5_variable_in_f_c_layer4_node\
=tf.Variable(tf.random_normal([10]))
logits_in_layer5_node=tf.matmul(hypothesis_f_after_dropout_in_layer4_node,W5_variable_in_f_c_layer4_node)+b5_variable_in_f_c_layer4_node
'''
Tensor("add_1:0",shape=(?,10),dtype=float32)
'''
cost_function_in_layer5_node\
=tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits(logits=logits_in_layer5_node,labels=Y_placeholder_node))
adam_optimizer_in_layer5_node=tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost_function_in_layer5_node)
sess_object=tf.Session()
sess_object.run(tf.global_variables_initializer())
# You train your model
print('Learning started. It takes sometime.')
for epoch in range(training_epochs):
average_cost_value=0
total_batch=int(mnist.train.num_examples/batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
feed_dict={X_placeholder_node:batch_xs,Y_placeholder_node:batch_ys,dropout_placeholder_node:0.7}
c,_=sess_object.run([cost_function_in_layer5_node,adam_optimizer_in_layer5_node],feed_dict=feed_dict)
average_cost_value+=c/total_batch
print('Epoch:','%04d' % (epoch + 1),'cost_function =','{:.9f}'.format(average_cost_value))
print('Learning Finished!')
# Test model and check accuracy
# if you have a OOM error,please refer to lab-11-X_placeholder_node-mnist_deep_cnn_low_memory.py
correct_prediction=tf.equal(tf.argmax(logits_in_layer5_node,1),tf.argmax(Y_placeholder_node,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
print('Accuracy:',sess_object.run(accuracy,feed_dict={
X_placeholder_node:mnist.test.images,Y_placeholder_node:mnist.test.labels,dropout_placeholder_node:1}))
# Get one and predict
r=random.randint(0,mnist.test.num_examples - 1)
print("Label:",sess_object.run(tf.argmax(mnist.test.labels[r:r + 1],1)))
print("Prediction:",sess_object.run(
tf.argmax(logits_in_layer5_node,1),feed_dict={X_placeholder_node:mnist.test.images[r:r + 1],dropout_placeholder_node:1}))
# plt.imshow(mnist.test.images[r:r + 1].
# reshape(28,28),cmap='Greys',interpolation='nearest')
# plt.show()
'''
Learning stared. It takes sometime.
Epoch:0001 cost_function=0.385748474
Epoch:0002 cost_function=0.092017397
Epoch:0003 cost_function=0.065854684
Epoch:0004 cost_function=0.055604566
Epoch:0005 cost_function=0.045996377
Epoch:0006 cost_function=0.040913645
Epoch:0007 cost_function=0.036924479
Epoch:0008 cost_function=0.032808939
Epoch:0009 cost_function=0.031791007
Epoch:0010 cost_function=0.030224456
Epoch:0011 cost_function=0.026849916
Epoch:0012 cost_function=0.026826763
Epoch:0013 cost_function=0.027188021
Epoch:0014 cost_function=0.023604777
Epoch:0015 cost_function=0.024607201
Learning Finished!
Accuracy:0.9938
'''
]]>