机器学习及数据清洗.zip

#训练一个全连接神经网络 import tensorflow as tf import math import numpy as np import types import matplotlib.pyplot as plt import pylab #定义一个y=sin(x)的曲线 def draw_correct_line(): x=np.arange(0,2*np.pi,0.01) x=x.reshape(len(x),1) y=np.sin(x) pylab.plot(x,y,label='标准sin曲线') plt.axhline(linewidth=1,color='r') #返回一个训练样本 def get_train_data(): train_x=np.random.uniform(0.0,2*np.pi,(1)) train_y=np.sin(train_x) return train_x,train_y #定义网络结构 def inference(input_data): with tf.variable_scope('hidden1'): #第一个隐藏层，采用16个隐藏点 weights=tf.get_variable('weight',[1,16],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) biases=tf.get_variable('bias',[1,16],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) hidden1=tf.sigmoid(tf.multiply(input_data,weights)+biases) with tf.variable_scope('hidden2'): #第二个隐藏层，采用16个隐藏点 weights=tf.get_variable('weight',[16,16],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) biases=tf.get_variable('bias',[16],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) hidden2=tf.sigmoid(tf.matmul(hidden1,weights)+biases) with tf.variable_scope('hidden3'): #第三个隐藏层，采用16个隐藏点 weights=tf.get_variable('weight',[16,16],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) biases=tf.get_variable('bias',[16],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) hidden3=tf.sigmoid(tf.matmul(hidden2,weights)+biases) with tf.variable_scope('output_layer'): #第二个隐藏层，采用16个隐藏点 weights=tf.get_variable('weight',[16,1],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) biases=tf.get_variable('bias',[1],tf.float32,initializer=tf.random_normal_initializer(0.0,1)) output=tf.matmul(hidden3,weights)+biases return output def train(): learning_rate=0.01 x = tf.placeholder(tf.float32) y = tf.placeholder(tf.float32) net_out=inference(x) #定义损失函数 loss_op=tf.square(net_out-y) #采用随机梯度下降的优化函数 opt=tf.train.GradientDescentOptimizer(learning_rate) train_op=opt.minimize(loss_op) init=tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) print('start training') for i in range(1000000): train_x,train_y=get_train_data() sess.run(train_op, feed_dict={x:train_x,y:train_y}) if i%10000==0: times=int(i/10000) test_x_ndarray=np.arange(0,2*np.pi,0.01) test_y_ndarray=np.zeros([len(test_x_ndarray)]) ind=0 for test_x in test_x_ndarray: test_y=sess.run(net_out,feed_dict={x:test_x,y:1}) np.put(test_y_ndarray,ind,test_y) ind+=1 draw_correct_line() pylab.plot(test_x_ndarray,test_y_ndarray,'--',label=str(times)+'times') if __name__=="__main__": train() #实现CNN卷积神经网络 import tensorflow as tf import os import cv2 tf_writer=tf.python_io.TFRecordWriter(path='train.tfrecords') for file in os.listdir(path='data_dir'): file_path=os.path.join('data_dir',file) image_data=cv2.imread(file_path) image_bytes=image_data.tostring() #图片的高 rows=image_data.shape[0] #第一维度的长度 这些可以通过image_data.shape来查看 #图片的宽 cols=image_data.shape[1] #图片通道数 channels=image_data.shape[2] label_data=0 example=tf.train.Example() feature=example.features.feature #相当于数据放入feature中，一个映射 feature['height'].int64_list.value.append(rows) feature['width'].int64_list.value.append(cols) feature['channels'].int64_list.value.append(channels) feature['image_data'].bytes_list.value.append(image_bytes) feature['label'].int64_list.value.append(label_data) tf_writer.write(example.SerializeToString()) tf_writer.close() #构造卷积神经网络结构 def weight_init(shape,name): #shape参数的形状 return tf.get_variable(name,shape,initializer=tf.random_normal_initializer(mean=0.0,stddev=0.1)) def bias_init(shape,name): return tf.get_variable(name,shape,initializer=tf.constant_initializer(0.0)) def conv2d(x,conv_w): return tf.nn.conv2d(x,conv_w,strides=[1,1,1,1],padding='VALID') def max_pool(x,size): return tf.nn.max_pool(x,ksize=[1,size,size,1],strides=[1,size,size,1],padding='VALID') def inference(input_data): with tf.name_scope('conv1'): #指定的区域中定义的所有对象及各种操作，他们的“name”属性上会增加该命名区的区域名，用以区别对象属于哪个区域； w_conv1=weight_init([10,10,3,16],'conv1_w') #卷积核大小是10X10，输入是3通道，输出是16通道 b_conv1=bias_init([16],'conv1_b') h_conv1=tf.nn.relu(conv2d(input_data,w_conv1)+b_conv1) #卷积过程 #池化 h_pool1=max_pool(h_conv1,2) with tf.name_scope('conv2'): #卷积核大小是5X5，输入是16通道，输出是16通道 w_conv2=weight_init([5,5,16,16],'conv2_w') b_conv2=bias_init([16],'conv2_b') #卷积之后，图片大小变成41X41（45-5+1=41） h_conv2=tf.nn.relu(conv2d(h_pool1,w_conv2)+b_conv2) h_pool2=max_pool(h_conv2,2) with tf.name_scope('conv3'): #卷积核大小是5X5,输入是16通道，输出16通道 w_conv3=weight_init([5,5,16,16],'conv3_w') b_conv3=bias_init([16],'conv3_b') h_conv3=tf.nn.relu(conv2d(h_pool2,w_conv3)+b_conv3) h_pool3=max_pool(h_conv3,2) with tf.name_scope('fc1'): w_fc1=weight_init([8*8*16,128],'fc1_w') b_fc1=bias_init([128],'fc1_b') h_fc1=tf.nn.relu(tf.matmul(tf.reshape(h_pool3,[-1,8*8*16]),w_fc1)+b_fc1) with tf.name_scope('fc2'): w_fc2=weight_init([128,2],'fc2_w') b_fc2=bias_init([2],'fc2_b') h_fc2=tf.matmul(h_fc1,w_fc2)+b_fc2 return h_fc2 def read_and_decode(filename_queue): #读取TFRecord文件队列数据，解码成张量 reader=tf.TFRecordReader() #从文件队列中读取数据 _,serialized_example=reader.read(filename_queue) #数据反序列化为结构化的数据 features=tf.parse_single_example( serialized_example, features={ 'height':tf.FixedLenFeature([],tf.int64), 'width': tf.FixedLenFeature([], tf.int64), 'channels': tf.FixedLenFeature([], tf.int64), 'image_data': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), }) #将image_data部分的数据解码成张量 image=tf.decode_raw(features['image_data'],tf.unit8) #将image_data的tensor变成100X100的大小，3通道 image=tf.cast(image,tf.float32) label=tf.cast(features['label'],tf.int32) return image,label def inputs(filename,batch_size): #读取TFRecord文件数据tensorflow中可以计算的张量数据 with tf.name_scope('input'): #生成文件队列，最多迭代2000次 filename_queue=tf.train.string_input_producer([filename],num_epochs=2000) #从文件中读入数据，并且变成张量格式 image,label=read_and_decode(filename_queue) #将数据按组大小返回，并且随机打乱 def train(): #定义一个global_step的张量，在训练过程中记录训练的步数 global_step=tf.get_variable('global_step',[],initializer=tf.constant_initializer(0),trainable=False,dtype=tf.int32) from absl import flags from absl.flags import FLAGS batch_size=flags.FLAGS.batch_size #读取之前生成的TFRecord文件，得到训练和验证的数据 train_images,train_labels=input("./tfrecord_da