简略神经网络结构
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
tf.set_random_seed(1)
np.random.seed(1)

fake data

x = np.linspace(-1, 1, 100)[:, np.newaxis] # shape (100, 1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2) + noise # shape (100, 1) + some noise

plot data

plt.scatter(x, y)
plt.show()
tf_x = tf.placeholder(tf.float32, x.shape) # input x
tf_y = tf.placeholder(tf.float32, y.shape) # input y

neural network layers

l1 = tf.layers.dense(tf_x, 10, tf.nn.relu) # hidden layer
output = tf.layers.dense(l1, 1) # output layer
loss = tf.losses.mean_squared_error(tf_y, output) # compute cost
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5)
train_op = optimizer.minimize(loss)
sess = tf.Session() # control training and others
sess.run(tf.global_variables_initializer()) # initialize var in graph
plt.ion() # something about plotting 关上交互模式
for step in range(100):

# train and net output
_, l, pred = sess.run([train_op, loss, output], {tf_x: x, tf_y: y})
if step % 5 == 0:
    # plot and show learning process
    plt.cla()# 革除流动轴
    plt.scatter(x, y)
    plt.plot(x, pred, 'r-', lw=5)
    plt.text(0.5, 0, 'Loss=%.4f' % l, fontdict={'size': 20, 'color': 'red'})
    plt.pause(0.1) 

plt.ioff()# 敞开交互模式用于阻塞程序，不让图片敞开
plt.show()
优化器
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
tf.set_random_seed(1)
np.random.seed(1)
LR = 0.01
BATCH_SIZE = 32

fake data

x = np.linspace(-1, 1, 100)[:, np.newaxis] # shape (100, 1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2) + noise # shape (100, 1) + some noise

plot dataset

plt.scatter(x, y)
plt.show()

default network

class Net:

def __init__(self, opt, **kwargs):
    self.x = tf.placeholder(tf.float32, [None, 1])
    self.y = [PM](https://www.gendan5.com/wallet/PerfectMoney.html)tf.placeholder(tf.float32, [None, 1])
    l = tf.layers.dense(self.x, 20, tf.nn.relu)
    out = tf.layers.dense(l, 1)
    self.loss = tf.losses.mean_squared_error(self.y, out)
    self.train = opt(LR, **kwargs).minimize(self.loss)

different nets

net_SGD = Net(tf.train.GradientDescentOptimizer)
net_Momentum = Net(tf.train.MomentumOptimizer, momentum=0.9)
net_RMSprop = Net(tf.train.RMSPropOptimizer)
net_Adam = Net(tf.train.AdamOptimizer)
nets = [net_SGD, net_Momentum, net_RMSprop, net_Adam]
sess = tf.Session()
sess.run(tf.global_variables_initializer())
losses_his = [[], [], [], []] # record loss

training

for step in range(300): # for each training step

index = np.random.randint(0, x.shape[0], BATCH_SIZE)
b_x = x[index]
b_y = y[index]
for net, l_his in zip(nets, losses_his):
    _, l = sess.run([net.train, net.loss], {net.x: b_x, net.y: b_y})
    l_his.append(l)     # loss recoder

plot loss history

labels = [‘SGD’, ‘Momentum’, ‘RMSprop’, ‘Adam’]
for i, l_his in enumerate(losses_his):

plt.plot(l_his, label=labels[i])

plt.legend(loc=’best’)
plt.xlabel(‘Steps’)
plt.ylabel(‘Loss’)
plt.ylim((0, 0.2))
plt.show()