011-lab-003. build cnn class,use tf.layers,implement ensemble
<![CDATA[
# @
# lab-11-3-mnist_cnn_class.py
import tensorflow as tf
# import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(777)
mnist=input_data.read_data_sets("MNIST_data/",one_hot=True)
learning_rate=0.001
training_epochs=15
batch_size=100
class Model:
# This class' constructor takes session object
def __init__(self,sess,name):
self.sess=sess
self.name=name
self._build_net()
# This method build neural network
def _build_net(self):
with tf.variable_scope(self.name):
# dropout rate: 0.7~0.5 on training,
# but should be 1 for testing
self.dropout_placeholder_node=tf.placeholder(tf.float32)
# [n,784]
self.X_placeholder_node=tf.placeholder(tf.float32,[None,784])
# You reshape input data into 28x28x1(black/white)
reshaped_X_img_node=tf.reshape(self.X_placeholder_node,[-1,28,28,1])
# [n,10]
self.Y_placeholder_node=tf.placeholder(tf.float32,[None,10])
# In layer 1, input image's shape=(?,28,28,1)
# Filter size will be 3*3*1(color)
# 32 filters
filter_W_variable_in_layer1_node=tf.Variable(tf.random_normal([3,3,1,32],stddev=0.01))
# After 1st convolution layer, image will be (?,28,28,32)
# After 1st convolution layer and pooling, image will be (?,14,14,32)
convolved_image_in_layer1_node\
=tf.nn.conv2d(\
reshaped_X_img_node\
,filter_W_variable_in_layer1_node\
,strides=[1,1,1,1]\
,padding='SAME')
convolved_image_after_relu_in_layer1_node\
=tf.nn.relu(convolved_image_in_layer1_node)
convolved_image_after_pooling_in_layer1_node\
=tf.nn.max_pool(\
convolved_image_after_relu_in_layer1_node\
,ksize=[1,2,2,1]\
,strides=[1,2,2,1],padding='SAME')
convolved_image_after_dropout_in_layer1_node\
=tf.nn.dropout(\
convolved_image_after_pooling_in_layer1_node\
,keep_prob=self.dropout_placeholder_node)
'''
Tensor("Conv2D:0",shape=(?,28,28,32),dtype=float32)
Tensor("Relu:0",shape=(?,28,28,32),dtype=float32)
Tensor("MaxPool:0",shape=(?,14,14,32),dtype=float32)
Tensor("dropout/mul:0",shape=(?,14,14,32),dtype=float32)
'''
# In layer 2, input image's shape=(?,14,14,32)
filter_W_variable_in_layer2_node\
=tf.Variable(tf.random_normal([3,3,32,64],stddev=0.01))
# After convolution layer 2, image will be (?,14,14,64)
# After convolution layer 2 and pooling, image will be (?,7,7,64)
convolved_image_in_layer2_node\
=tf.nn.conv2d(\
convolved_image_after_dropout_in_layer1_node\
,filter_W_variable_in_layer2_node\
,strides=[1,1,1,1]\
,padding='SAME')
convolved_image_after_relu_in_layer2_node\
=tf.nn.relu(convolved_image_in_layer2_node)
convolved_image_after_pooling_in_layer2_node\
=tf.nn.max_pool(\
convolved_image_after_relu_in_layer2_node\
,ksize=[1,2,2,1]
,strides=[1,2,2,1]\
,padding='SAME')
convolved_image_after_dropout_in_layer2_node\
=tf.nn.dropout(\
convolved_image_after_pooling_in_layer2_node\
,keep_prob=self.dropout_placeholder_node)
'''
Tensor("Conv2D_1:0",shape=(?,14,14,64),dtype=float32)
Tensor("Relu_1:0",shape=(?,14,14,64),dtype=float32)
Tensor("MaxPool_1:0",shape=(?,7,7,64),dtype=float32)
Tensor("dropout_1/mul:0",shape=(?,7,7,64),dtype=float32)
'''
# In layer 3, input image's shape=(?,7,7,64)
filter_W_variable_in_layer3_node\
=tf.Variable(tf.random_normal([3,3,64,128],stddev=0.01))
# After convolution layer 3, image will be (?,7,7,128)
# After convolution layer 3 and pooling, image will be (?,4,4,128)
# After convolution layer 3 and pooling and reshape, image will be (?,4 * 4 * 128)
# You reshape them to input them into fully connected layer
convolved_image_in_layer3_node\
=tf.nn.conv2d(\
convolved_image_after_dropout_in_layer2_node\
,filter_W_variable_in_layer3_node\
,strides=[1,1,1,1]\
,padding='SAME')
convolved_image_after_relu_in_layer3_node\
=tf.nn.relu(convolved_image_in_layer3_node)
convolved_image_after_pooling_in_layer3_node\
=tf.nn.max_pool(\
convolved_image_after_relu_in_layer3_node\
,ksize=[1,2,2,1]\
,strides=[1,2,2,1]\
,padding='SAME')
convolved_image_after_dropout_in_layer3_node\
=tf.nn.dropout(\
convolved_image_after_pooling_in_layer3_node\
,keep_prob=self.dropout_placeholder_node)
convolved_image_after_flatten_in_layer3_node\
=tf.reshape(convolved_image_after_dropout_in_layer3_node,[-1,128*4*4])
'''
Tensor("Conv2D_2:0",shape=(?,7,7,128),dtype=float32)
Tensor("Relu_2:0",shape=(?,7,7,128),dtype=float32)
Tensor("MaxPool_2:0",shape=(?,4,4,128),dtype=float32)
Tensor("dropout_2/mul:0",shape=(?,4,4,128),dtype=float32)
Tensor("Reshape_1:0",shape=(?,2048),dtype=float32)
'''
# In layer 4(1st fully connected layer), input image's shape=(n,128*4*4)
# [n,128*4*4][?,?]=[n,625]
# [?,?]=[128*4*4,625]
W_variable_in_layer4_node\
=tf.get_variable(\
"W_variable_in_layer4_node"\
,shape=[128*4*4,625]\
,initializer=tf.contrib.layers.xavier_initializer())
b_variable_in_layer4_node=tf.Variable(tf.random_normal([625]))
hypothesis_f_after_relu_in_layer4_node\
=tf.nn.relu(tf.matmul(\
convolved_image_after_flatten_in_layer3_node\
,W_variable_in_layer4_node)+b_variable_in_layer4_node)
hypothesis_f_after_dropout_in_layer4_node\
=tf.nn.dropout(\
hypothesis_f_after_relu_in_layer4_node\
,keep_prob=self.dropout_placeholder_node)
'''
Tensor("Relu_3:0",shape=(?,625),dtype=float32)
Tensor("dropout_3/mul:0",shape=(?,625),dtype=float32)
'''
# In layer 5(2nd fully connected layer), input image's shape=(n,625)
# [n,625][?,?]=[n,10]
# [?,?]=[625,10]
W_variable_in_layer5_node\
=tf.get_variable(\
"W_variable_in_layer5_node"\
,shape=[625,10]\
,initializer=tf.contrib.layers.xavier_initializer())
b_variable_in_layer5_node\
=tf.Variable(tf.random_normal([10]))
self.logits=tf.matmul(\
hypothesis_f_after_dropout_in_layer4_node\
,W_variable_in_layer5_node)+b_variable_in_layer5_node
'''
Tensor("add_1:0",shape=(?,10),dtype=float32)
'''
self.cost_function_in_layer5_node\
=tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits(\
logits=self.logits\
,labels=self.Y_placeholder_node))
self.adap_optimizer_node\
=tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.cost_function_in_layer5_node)
correct_prediction=tf.equal(
tf.argmax(self.logits,1),tf.argmax(self.Y_placeholder_node,1))
self.accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
# This helper method predicts
def predict(self,x_test,keep_prop=1.0):
return self.sess.run(self.logits,feed_dict={self.X_placeholder_node: x_test,self.keep_prob: keep_prop})
def get_accuracy(self,x_test,y_test,keep_prop=1.0):
return self.sess.run(self.accuracy,feed_dict={self.X_placeholder_node: x_test,self.Y_placeholder_node: y_test,self.keep_prob: keep_prop})
def train(self,x_data,y_data,keep_prop=0.7):
return self.sess.run([self.cost_function_in_layer5_node,self.adap_optimizer_node],feed_dict={
self.X_placeholder_node: x_data,self.Y_placeholder_node: y_data,self.keep_prob: keep_prop})
# initialize
sess=tf.Session()
# I create instance of Model
m1=Model(sess,"m1")
sess.run(tf.global_variables_initializer())
print('Learning Started!')
# train my model
for epoch in range(training_epochs):
avg_cost=0
total_batch=int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
# I invoke train method
c,_=m1.train(batch_xs,batch_ys)
avg_cost += c / total_batch
print('Epoch:','%04d' % (epoch + 1),'cost_function_in_layer5_node =','{:.9f}'.format(avg_cost))
print('Learning Finished!')
# Test model and check accuracy
print('Accuracy:',m1.get_accuracy(mnist.test.images,mnist.test.labels))
# @
# lab-11-4-mnist_cnn_layers.py
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(777)
mnist=input_data.read_data_sets("MNIST_data/",one_hot=True)
learning_rate=0.001
training_epochs=15
batch_size=100
class Model:
def __init__(self,sess,name):
self.sess=sess
self.name=name
self._build_net()
def _build_net(self):
with tf.variable_scope(self.name):
# dropout (keep_prob) rate 0.7~0.5 on training,but should be 1
# for testing
self.training=tf.placeholder(tf.bool)
# input place holders
self.X_placeholder_node=tf.placeholder(tf.float32,[None,784])
# img 28x28x1 (black/white),Input Layer
reshaped_X_img_node=tf.reshape(self.X_placeholder_node,[-1,28,28,1])
self.Y_placeholder_node=tf.placeholder(tf.float32,[None,10])
# I use tf.layers
# Convolutional layer 1
conv1=tf.layers.conv2d(inputs=reshaped_X_img_node,filters=32,kernel_size=[3,3],
padding="SAME",activation=tf.nn.relu)
# Pooling layer 1
pool1=tf.layers.max_pooling2d(inputs=conv1,pool_size=[2,2],
padding="SAME",strides=2)
dropout1=tf.layers.dropout(inputs=pool1,
rate=0.7,training=self.training)
# Convolutional layer 2
conv2=tf.layers.conv2d(inputs=dropout1,filters=64,kernel_size=[3,3],
padding="SAME",activation=tf.nn.relu)
# Pooling layer 2
pool2=tf.layers.max_pooling2d(inputs=conv2,pool_size=[2,2],
padding="SAME",strides=2)
dropout2=tf.layers.dropout(inputs=pool2,
rate=0.7,training=self.training)
# Convolutional layer 3
# Pooling layer 3
conv3=tf.layers.conv2d(inputs=dropout2,filters=128,kernel_size=[3,3],
padding="same",activation=tf.nn.relu)
pool3=tf.layers.max_pooling2d(inputs=conv3,pool_size=[2,2],
padding="same",strides=2)
dropout3=tf.layers.dropout(inputs=pool3,
rate=0.7,training=self.training)
# Now,we need to use fully connected layer
# Dense layer with relu
flat=tf.reshape(dropout3,[-1,128 * 4 * 4])
# inputs=flat: autumatically control
# units=625: number of output
dense4=tf.layers.dense(inputs=flat,
units=625,
activation=tf.nn.relu)
dropout4=tf.layers.dropout(inputs=dense4,
rate=0.5,
training=self.training)
# Logits layer(no activation): layer5 Final FC 625 inputs -> 10 outputs
self.logits=tf.layers.dense(inputs=dropout4,units=10)
# define cost_function_in_layer5_node/loss & adap_optimizer_node
self.cost_function_in_layer5_node=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=self.logits,labels=self.Y_placeholder_node))
self.adap_optimizer_node=tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(self.cost_function_in_layer5_node)
correct_prediction=tf.equal(
tf.argmax(self.logits,1),tf.argmax(self.Y_placeholder_node,1))
self.accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
def predict(self,x_test,training=False):
return self.sess.run(self.logits,
feed_dict={self.X_placeholder_node: x_test,self.training: training})
def get_accuracy(self,x_test,y_test,training=False):
return self.sess.run(self.accuracy,
feed_dict={self.X_placeholder_node: x_test,
self.Y_placeholder_node: y_test,self.training: training})
def train(self,x_data,y_data,training=True):
return self.sess.run([self.cost_function_in_layer5_node,self.adap_optimizer_node],feed_dict={
self.X_placeholder_node: x_data,self.Y_placeholder_node: y_data,self.training: training})
# initialize
sess=tf.Session()
m1=Model(sess,"m1")
sess.run(tf.global_variables_initializer())
print('Learning Started!')
# train my model
for epoch in range(training_epochs):
avg_cost=0
total_batch=int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
c,_=m1.train(batch_xs,batch_ys)
avg_cost += c / total_batch
print('Epoch:','%04d' % (epoch + 1),'cost_function_in_layer5_node =','{:.9f}'.format(avg_cost))
print('Learning Finished!')
# Test model and check accuracy
print('Accuracy:',m1.get_accuracy(mnist.test.images,mnist.test.labels))
# @
# lab-11-5-mnist_cnn_ensemble_layers.py
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(777)
mnist=input_data.read_data_sets("MNIST_data/",one_hot=True)
learning_rate=0.001
training_epochs=20
batch_size=100
class Model:
def __init__(self,sess,name):
self.sess=sess
self.name=name
self._build_net()
def _build_net(self):
with tf.variable_scope(self.name):
# dropout (keep_prob) rate 0.7~0.5 on training,but should be 1
# for testing
self.training=tf.placeholder(tf.bool)
# input place holders
self.X_placeholder_node=tf.placeholder(tf.float32,[None,784])
# img 28x28x1 (black/white),Input Layer
reshaped_X_img_node=tf.reshape(self.X_placeholder_node,[-1,28,28,1])
self.Y_placeholder_node=tf.placeholder(tf.float32,[None,10])
# Convolutional Layer #1
conv1=tf.layers.conv2d(inputs=reshaped_X_img_node,filters=32,kernel_size=[3,3],
padding="SAME",activation=tf.nn.relu)
# Pooling Layer #1
pool1=tf.layers.max_pooling2d(inputs=conv1,pool_size=[2,2],
padding="SAME",strides=2)
dropout1=tf.layers.dropout(inputs=pool1,
rate=0.7,training=self.training)
# Convolutional Layer #2 and Pooling Layer #2
conv2=tf.layers.conv2d(inputs=dropout1,filters=64,kernel_size=[3,3],
padding="SAME",activation=tf.nn.relu)
pool2=tf.layers.max_pooling2d(inputs=conv2,pool_size=[2,2],
padding="SAME",strides=2)
dropout2=tf.layers.dropout(inputs=pool2,
rate=0.7,training=self.training)
# Convolutional Layer #3 and Pooling Layer #3
conv3=tf.layers.conv2d(inputs=dropout2,filters=128,kernel_size=[3,3],
padding="SAME",activation=tf.nn.relu)
pool3=tf.layers.max_pooling2d(inputs=conv3,pool_size=[2,2],
padding="SAME",strides=2)
dropout3=tf.layers.dropout(inputs=pool3,
rate=0.7,training=self.training)
# Dense Layer with Relu
flat=tf.reshape(dropout3,[-1,128 * 4 * 4])
dense4=tf.layers.dense(inputs=flat,
units=625,activation=tf.nn.relu)
dropout4=tf.layers.dropout(inputs=dense4,
rate=0.5,training=self.training)
# Logits (no activation) Layer: L5 Final FC 625 inputs -> 10 outputs
self.logits=tf.layers.dense(inputs=dropout4,units=10)
# define cost_function_in_layer5_node/loss & adap_optimizer_node
self.cost_function_in_layer5_node=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=self.logits,labels=self.Y_placeholder_node))
self.adap_optimizer_node=tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(self.cost_function_in_layer5_node)
correct_prediction=tf.equal(
tf.argmax(self.logits,1),tf.argmax(self.Y_placeholder_node,1))
self.accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
def predict(self,x_test,training=False):
return self.sess.run(self.logits,
feed_dict={self.X_placeholder_node: x_test,self.training: training})
def get_accuracy(self,x_test,y_test,training=False):
return self.sess.run(self.accuracy,
feed_dict={self.X_placeholder_node: x_test,
self.Y_placeholder_node: y_test,self.training: training})
def train(self,x_data,y_data,training=True):
return self.sess.run([self.cost_function_in_layer5_node,self.adap_optimizer_node],feed_dict={
self.X_placeholder_node: x_data,self.Y_placeholder_node: y_data,self.training: training})
sess=tf.Session()
# To use ensemble,first,I should create models
# I use Model class
# I create list which will contain models
models=[]
# I will 2 models
num_models=2
# I create 2 model by using for loop
for m in range(num_models):
models.append(Model(sess,"model" + str(m)))
sess.run(tf.global_variables_initializer())
print('Learning Started!')
# I train my model
for epoch in range(training_epochs):
avg_cost_list=np.zeros(len(models))
total_batch=int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
# I train each model because we use 2 models
for m_idx,m in enumerate(models):
c,_=m.train(batch_xs,batch_ys)
avg_cost_list[m_idx] += c / total_batch
print('Epoch:','%04d' % (epoch + 1),'cost_function_in_layer5_node =',avg_cost_list)
print('Learning Finished!')
# Test model and check accuracy
test_size=len(mnist.test.labels)
# Create spaces which will be contained by probabilities
predictions=np.zeros([test_size,10])
for m_idx,m in enumerate(models):
print(m_idx,'Accuracy:',m.get_accuracy(
mnist.test.images,mnist.test.labels))
p=m.predict(mnist.test.images)
predictions += p
ensemble_correct_prediction=tf.equal(
tf.argmax(predictions,1),tf.argmax(mnist.test.labels,1))
ensemble_accuracy=tf.reduce_mean(
tf.cast(ensemble_correct_prediction,tf.float32))
print('Ensemble accuracy:',sess.run(ensemble_accuracy))
# 0 Accuracy: 0.9933
# 1 Accuracy: 0.9946
# 2 Accuracy: 0.9934
# 3 Accuracy: 0.9935
# 4 Accuracy: 0.9935
# 5 Accuracy: 0.9949
# 6 Accuracy: 0.9941
# Ensemble accuracy: 0.9952
# @
# Ensemble prediction
# I let model to predict with "3"
# $$$C_{1},C_{2},...,C_{m}$$$ will output prediction values
# 0 1 2 3 4 5 6 7 8 9
# $$$C_{1}$$$ 0.1 0.01 0.02 0.8 ...
# $$$C_{2}$$$ 0.01 0.5 0.02 0.4 ...
# ...
# $$$C_{m}$$$ 0.01 0.01 0.1 0.7
# sum 0.12 0.52 0.14 1.9
# argmax(sum)=3(1.9)
# @
# lab-11-X_placeholder_node-mnist_cnn_low_memory.py
import tensorflow as tf
import random
# import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
tf.set_random_seed(777) # reproducibility
mnist=input_data.read_data_sets("MNIST_data/",one_hot=True)
learning_rate=0.001
training_epochs=15
batch_size=100
# dropout (keep_prob) rate 0.7~0.5 on training,but should be 1 for testing
keep_prob=tf.placeholder(tf.float32)
X_placeholder_node=tf.placeholder(tf.float32,[None,784])
reshaped_X_img_node=tf.reshape(X_placeholder_node,[-1,28,28,1]) # img 28x28x1 (black/white)
Y_placeholder_node=tf.placeholder(tf.float32,[None,10])
# convolved_image_in_layer1_node ImgIn shape=(?,28,28,1)
filter_W_variable_in_layer1_node=tf.Variable(tf.random_normal([3,3,1,32],stddev=0.01))
# Conv -> (?,28,28,32)
# Pool -> (?,14,14,32)
convolved_image_in_layer1_node=tf.nn.conv2d(reshaped_X_img_node,filter_W_variable_in_layer1_node,strides=[1,1,1,1],padding='SAME')
convolved_image_in_layer1_node=tf.nn.relu(convolved_image_in_layer1_node)
convolved_image_in_layer1_node=tf.nn.max_pool(convolved_image_in_layer1_node,ksize=[1,2,2,1],
strides=[1,2,2,1],padding='SAME')
convolved_image_in_layer1_node=tf.nn.dropout(convolved_image_in_layer1_node,keep_prob=keep_prob)
# Tensor("Conv2D:0",shape=(?,28,28,32),dtype=float32)
# Tensor("Relu:0",shape=(?,28,28,32),dtype=float32)
# Tensor("MaxPool:0",shape=(?,14,14,32),dtype=float32)
# Tensor("dropout/mul:0",shape=(?,14,14,32),dtype=float32)
# convolved_image_in_layer2_node ImgIn shape=(?,14,14,32)
filter_W_variable_in_layer2_node=tf.Variable(tf.random_normal([3,3,32,64],stddev=0.01))
# Conv ->(?,14,14,64)
# Pool ->(?,7,7,64)
convolved_image_in_layer2_node=tf.nn.conv2d(convolved_image_in_layer1_node,filter_W_variable_in_layer2_node,strides=[1,1,1,1],padding='SAME')
convolved_image_in_layer2_node=tf.nn.relu(convolved_image_in_layer2_node)
convolved_image_in_layer2_node=tf.nn.max_pool(convolved_image_in_layer2_node,ksize=[1,2,2,1],
strides=[1,2,2,1],padding='SAME')
convolved_image_in_layer2_node=tf.nn.dropout(convolved_image_in_layer2_node,keep_prob=keep_prob)
# Tensor("Conv2D_1:0",shape=(?,14,14,64),dtype=float32)
# Tensor("Relu_1:0",shape=(?,14,14,64),dtype=float32)
# Tensor("MaxPool_1:0",shape=(?,7,7,64),dtype=float32)
# Tensor("dropout_1/mul:0",shape=(?,7,7,64),dtype=float32)
# convolved_image_in_layer3_node ImgIn shape=(?,7,7,64)
filter_W_variable_in_layer3_node=tf.Variable(tf.random_normal([3,3,64,128],stddev=0.01))
# Conv ->(?,7,7,128)
# Pool ->(?,4,4,128)
# Reshape ->(?,4 * 4 * 128) # Flatten them for FC
convolved_image_in_layer3_node=tf.nn.conv2d(convolved_image_in_layer2_node,filter_W_variable_in_layer3_node,strides=[1,1,1,1],padding='SAME')
convolved_image_in_layer3_node=tf.nn.relu(convolved_image_in_layer3_node)
convolved_image_in_layer3_node=tf.nn.max_pool(convolved_image_in_layer3_node,ksize=[1,2,2,1],strides=[
1,2,2,1],padding='SAME')
convolved_image_in_layer3_node=tf.nn.dropout(convolved_image_in_layer3_node,keep_prob=keep_prob)
convolved_image_in_layer3_node=tf.reshape(convolved_image_in_layer3_node,[-1,128 * 4 * 4])
# Tensor("Conv2D_2:0",shape=(?,7,7,128),dtype=float32)
# Tensor("Relu_2:0",shape=(?,7,7,128),dtype=float32)
# Tensor("MaxPool_2:0",shape=(?,4,4,128),dtype=float32)
# Tensor("dropout_2/mul:0",shape=(?,4,4,128),dtype=float32)
# Tensor("Reshape_1:0",shape=(?,2048),dtype=float32)
# hypothesis_f_after_relu_in_layer4_node FC 4x4x128 inputs -> 625 outputs
W_variable_in_layer4_node=tf.get_variable("W_variable_in_layer4_node",shape=[128 * 4 * 4,625],
initializer=tf.contrib.layers.xavier_initializer())
b_variable_in_layer4_node=tf.Variable(tf.random_normal([625]))
hypothesis_f_after_relu_in_layer4_node=tf.nn.relu(tf.matmul(convolved_image_in_layer3_node,W_variable_in_layer4_node) + b_variable_in_layer4_node)
hypothesis_f_after_relu_in_layer4_node=tf.nn.dropout(hypothesis_f_after_relu_in_layer4_node,keep_prob=keep_prob)
'''
Tensor("Relu_3:0",shape=(?,625),dtype=float32)
Tensor("dropout_3/mul:0",shape=(?,625),dtype=float32)
'''
# L5 Final FC 625 inputs -> 10 outputs
W_variable_in_layer5_node=tf.get_variable("W_variable_in_layer5_node",shape=[625,10],
initializer=tf.contrib.layers.xavier_initializer())
b_variable_in_layer5_node=tf.Variable(tf.random_normal([10]))
hypothesis=tf.matmul(hypothesis_f_after_relu_in_layer4_node,W_variable_in_layer5_node) + b_variable_in_layer5_node
'''
Tensor("add_1:0",shape=(?,10),dtype=float32)
'''
# define loss function and adap_optimizer_node
cost_function_in_layer5_node=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=hypothesis,labels=Y_placeholder_node))
adap_optimizer_node=tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost_function_in_layer5_node)
# I initialize variables
sess=tf.Session()
sess.run(tf.global_variables_initializer())
# I train my model
print('Learning stared. It takes sometime.')
for epoch in range(training_epochs):
avg_cost=0
total_batch=int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
feed_dict={X_placeholder_node: batch_xs,Y_placeholder_node: batch_ys,keep_prob: 0.7}
c,_,=sess.run([cost_function_in_layer5_node,adap_optimizer_node],feed_dict=feed_dict)
avg_cost += c / total_batch
print('Epoch:','%04d' % (epoch + 1),'cost_function_in_layer5_node =','{:.9f}'.format(avg_cost))
print('Learning Finished!')
# Test model and check accuracy
correct_prediction=tf.equal(tf.argmax(hypothesis,1),tf.argmax(Y_placeholder_node,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
def evaluate(X_sample,y_sample,batch_size=512):
"""Run a minibatch accuracy op"""
N=X_sample.shape[0]
correct_sample=0
for i in range(0,N,batch_size):
X_batch=X_sample[i: i + batch_size]
y_batch=y_sample[i: i + batch_size]
N_batch=X_batch.shape[0]
feed={
X_placeholder_node: X_batch,
Y_placeholder_node: y_batch,
keep_prob: 1
}
correct_sample += sess.run(accuracy,feed_dict=feed) * N_batch
return correct_sample / N
print("\nAccuracy Evaluates")
print("-------------------------------")
print('Train Accuracy:',evaluate(mnist.train.images,mnist.train.labels))
print('Test Accuracy:',evaluate(mnist.test.images,mnist.test.labels))
# Get one and predict
print("\nGet one and predict")
print("-------------------------------")
r=random.randint(0,mnist.test.num_examples - 1)
print("Label: ",sess.run(tf.argmax(mnist.test.labels[r:r + 1],1)))
print("Prediction: ",sess.run(
tf.argmax(hypothesis,1),{X_placeholder_node: mnist.test.images[r:r + 1],keep_prob: 1}))
# plt.imshow(mnist.test.images[r:r + 1].
# reshape(28,28),cmap='Greys',interpolation='nearest')
# plt.show()
# Learning stared. It takes sometime.
# Epoch: 0001 cost_function_in_layer5_node=0.385748474
# Epoch: 0002 cost_function_in_layer5_node=0.092017397
# Epoch: 0003 cost_function_in_layer5_node=0.065854684
# Epoch: 0004 cost_function_in_layer5_node=0.055604566
# Epoch: 0005 cost_function_in_layer5_node=0.045996377
# Epoch: 0006 cost_function_in_layer5_node=0.040913645
# Epoch: 0007 cost_function_in_layer5_node=0.036924479
# Epoch: 0008 cost_function_in_layer5_node=0.032808939
# Epoch: 0009 cost_function_in_layer5_node=0.031791007
# Epoch: 0010 cost_function_in_layer5_node=0.030224456
# Epoch: 0011 cost_function_in_layer5_node=0.026849916
# Epoch: 0012 cost_function_in_layer5_node=0.026826763
# Epoch: 0013 cost_function_in_layer5_node=0.027188021
# Epoch: 0014 cost_function_in_layer5_node=0.023604777
# Epoch: 0015 cost_function_in_layer5_node=0.024607201
# Learning Finished!
# Accuracy: 0.9938
]]>