deno

admin — Mon, 01 Feb 2021 13:28:59 +0000

#!/usr/bin/env python # coding: utf-8 # # Library # In[1]: import numpy as np import pandas as pd import matplotlib.pyplot as plt import os import pywt import math import statistics import csv import pickle # In[2]: # load with open('data.pickle', 'rb') as f: data = pickle.load(f) # # Analysis X Data # In[3]: data['X'].shape, data['Y'].shape # In[4]: data['X'][100:,:,0] # In[5]: plt.figure(figsize=(20,10)) for i in range(len(data['X'])): plt.plot(data['X'][i,:200,0]) # In[6]: range(0, len(data['X'][:,0,0])) # In[7]: oes_data = [] for i in range(0, len(data['X'][:,0,0])): temp = data['X'][i,:,0] oes_data.append(temp) oes_data = np.transpose(oes_data) data_df = pd.DataFrame(data = oes_data) # In[8]: data_df[0] # In[9]: plt.figure(figsize=(20,10)) # plt.xlim(1000,2000) plt.plot(data_df[0]) # ## Analysis Y Data # In[10]: data['Y'].shape # In[11]: len(data['Y']) # In[12]: plt.figure(figsize=(20,10)) plt.plot(data['Y'][:]) # In[13]: label_array = [] for i in range(0, len(data['Y'])): temp = data['Y'][i] label_array.append(temp) label_array # # Noise Data # ## Data_X save # In[14]: #Data 불러와서 노이즈 추가하여 합치기 노이즈 증폭 data_array_noise = [] x = data_df[0] noise = 3*np.random.normal(0, 10, len(x)) for i in range(0, len(data['X'][:,0,0])): temp = data['X'][i,:,0] temp = temp + noise data_array_noise.append(temp) data_noise = np.array(data_array_noise) data_noise = np.transpose(data_noise) data_df_noise = pd.DataFrame(data_noise) # In[15]: plt.figure(figsize=(20,10)) # plt.xlim(1000,2000) plt.plot(data_df_noise[0]) # # Denoised Data # ## Data_X save # In[16]: data_df.head(6) # In[17]: def Removing_detailed_(level, coeffs, wavelet): # 각각의 coefficient 분리 cA = coeffs[0] cD1 = coeffs[1]['d'] cD2 = coeffs[2]['d'] # Detailed coefficient 지우기 cD1 = np.zeros(len(cD1)) cD2 = np.zeros(len(cD2)) coeffs[0] = cA coeffs[1]['d'] = cD1 coeffs[2]['d'] = cD2 x_recon = pywt.waverecn(coeffs, wavelet) return x_recon # In[18]: # (3) Noise reduction in Lidar signal (2004) _ soft thresholding def Lidar_signal(wavelet, level, coeffs, data_df, freq): x_1 = data_df[freq] coeffs_D1 = pywt.wavedecn(x_1, wavelet, level = 1) for i in range(2): if i == 0: cA_1 = coeffs_D1[i] if i != 0: cD_list = coeffs_D1[i] cD_1 = cD_list['d'] coeffs_array = np.hstack([cA_1, cD_1]) median = statistics.median(coeffs_array) median = median/0.674 #standard deviation을 모르므로 median으로 추정. N = len(data_df[freq]) std = median thr = std*(math.sqrt(2*math.log2(N)))# for i in range(level+1): if i == 0: cA = coeffs[i] if i != 0:#Use soft threshold cD = coeffs[i] for j in range(len(cD['d'])): if cD['d'][j] >= thr: cD['d'][j] = np.sign(cD['d'][j])*(cD['d'][j]-thr) if (cD['d'][j] >= -thr and cD['d'][j] < thr): cD['d'][j] = 0 if cD['d'][j] < -thr: cD['d'][j] = np.sign(cD['d'][j])*(cD['d'][j]+thr) coeffs[i] = cD x_recon = pywt.waverecn(coeffs,wavelet) return x_recon # # Reconstruction without noise # In[19]: level = 3 # 통합 코드 생각 optimal_wavelet = [] cA_wavelet_mean = np.zeros(len(pywt.wavelist(kind='discrete'))) for j in range(0, len(data_df.T[0])): x = data_df[j] x = x + noise for i in range(0,len(pywt.wavelist(kind='discrete'))): wavelet = pywt.wavelist(kind='discrete')[i] coeffs = pywt.wavedecn(x, wavelet, level = 2) cA = coeffs[0] cD1 = coeffs[1]['d'] cD2 = coeffs[2]['d'] cA_wavelet_mean[i] = cA.mean() cA_argmin = pywt.wavelist(kind='discrete')[cA_wavelet_mean.argmin()] optimal_wavelet.append(cA_argmin) # In[20]: level = 3 x_recon1 = [] x_recon2 = [] for j in range(0, len(data_df.T[0])): x = data_df[j] wt = optimal_wavelet[j] coeffs = pywt.wavedecn(x, wt, level = 3) #temp1 = Removing_detailed_(level, coeffs, wt) temp2 = Lidar_signal(wt, level, coeffs, data_df, j) # np.transpose(temp1) np.transpose(temp2) # x_recon1.append(temp1) x_recon2.append(temp2) # In[21]: #x_recon1 = np.transpose(x_recon1) x_recon2 = np.transpose(x_recon2) # In[22]: # denoised_data_df1 = pd.DataFrame(x_recon1) # denoised_data_df1 # In[23]: denoised_data_df2 = pd.DataFrame(x_recon2) denoised_data_df2 # # Data save # In[24]: #Original Data 저장하기 #import pickle #a = {'X': data_df.values, 'Y' : label_array} #with open('data_.pickle', 'wb') as handle: # pickle.dump(a, handle, protocol=pickle.HIGHEST_PROTOCOL) # In[25]: #Noise Data 저장하기 import pickle a = {'X': data_df_noise.values, 'Y' : label_array} with open('oes_data_noise.pickle', 'wb') as handle: pickle.dump(a, handle, protocol=pickle.HIGHEST_PROTOCOL) # In[26]: #Denoised-Data 저장하기 import pickle a = {'X': denoised_data_df2.values, 'Y' : label_array} with open('oes_denoised_data.pickle', 'wb') as handle: pickle.dump(a, handle, protocol=pickle.HIGHEST_PROTOCOL) # In[ ]:

MAE

admin — Mon, 01 Feb 2021 13:28:15 +0000

#!/usr/bin/env python # coding: utf-8 # In[3]: import os, inspect, warnings, argparse os.environ['TF_CPP_MIN_LOG_LEVEL']='2' os.environ["CUDA_VISIBLE_DEVICES"]='0' warnings.filterwarnings('ignore') import tensorflow as tf import source.datamanager as dman import source.neuralnet as nn import source.tf_process as tfp # In[15]: import numpy as np import tensorflow as tf from sklearn.utils import shuffle class Dataset(object): def __init__(self, x_tr, y_tr, x_te, y_te, normalize=True): print("nInitializing Dataset...") self.normalize = normalize #(x_tr, y_tr), (x_te, y_te) = tf.keras.datasets.mnist.load_data() self.x_tr, self.y_tr = x_tr, y_tr self.x_te, self.y_te = x_te, y_te self.x_tr = np.ndarray.astype(self.x_tr, np.float32) self.x_te = np.ndarray.astype(self.x_te, np.float32) self.split_dataset() self.num_tr, self.num_te = self.x_tr.shape[0], self.x_te.shape[0] self.idx_tr, self.idx_te = 0, 0 print("Number of datanTraining: %d, Test: %dn" %(self.num_tr, self.num_te)) x_sample, y_sample = self.x_te[0], self.y_te[0] self.height = x_sample.shape[0] self.width = x_sample.shape[1] try: self.channel = x_sample.shape[2] except: self.channel = 1 self.min_val, self.max_val = x_sample.min(), x_sample.max() self.num_class = (y_te.max()+1) print("Information of data") print("Shape Height: %d, Width: %d, Channel: %d" %(self.height, self.width, self.channel)) print("Value Min: %.3f, Max: %.3f" %(self.min_val, self.max_val)) print("Class %d" %(self.num_class)) print("Normalization: %r" %(self.normalize)) if(self.normalize): print("(from %.3f-%.3f to %.3f-%.3f)" %(self.min_val, self.max_val, 0, 1)) def split_dataset(self): x_tot = np.append(self.x_tr, self.x_te, axis=0) y_tot = np.append(self.y_tr, self.y_te, axis=0) x_normal, y_normal = None, None x_abnormal, y_abnormal = None, None for yidx, y in enumerate(y_tot): x_tmp = np.expand_dims(x_tot[yidx], axis=0) y_tmp = np.expand_dims(y_tot[yidx], axis=0) if(y == 1): # as normal if(x_normal is None): x_normal = x_tmp y_normal = y_tmp else: x_normal = np.append(x_normal, x_tmp, axis=0) y_normal = np.append(y_normal, y_tmp, axis=0) else: # as abnormal if(x_abnormal is None): x_abnormal = x_tmp y_abnormal = y_tmp else: if(x_abnormal.shape[0] < 1000): x_abnormal = np.append(x_abnormal, x_tmp, axis=0) y_abnormal = np.append(y_abnormal, y_tmp, axis=0) if(not(x_normal is None) and not(x_abnormal is None)): if((x_normal.shape[0] >= 2000) and x_abnormal.shape[0] >= 1000): break self.x_tr, self.y_tr = x_normal[:1000], y_normal[:1000] self.x_te, self.y_te = x_normal[1000:], y_normal[1000:] self.x_te = np.append(self.x_te, x_abnormal, axis=0) self.y_te = np.append(self.y_te, y_abnormal, axis=0) def reset_idx(self): self.idx_tr, self.idx_te = 0, 0 def next_train(self, batch_size=1, fix=False): start, end = self.idx_tr, self.idx_tr+batch_size x_tr, y_tr = self.x_tr[start:end], self.y_tr[start:end] x_tr = np.expand_dims(x_tr, axis=3) terminator = False if(end >= self.num_tr): terminator = True self.idx_tr = 0 self.x_tr, self.y_tr = shuffle(self.x_tr, self.y_tr) else: self.idx_tr = end if(fix): self.idx_tr = start if(x_tr.shape[0] != batch_size): x_tr, y_tr = self.x_tr[-1-batch_size:-1], self.y_tr[-1-batch_size:-1] x_tr = np.expand_dims(x_tr, axis=3) if(self.normalize): min_x, max_x = x_tr.min(), x_tr.max() x_tr = (x_tr - min_x) / (max_x - min_x) return x_tr, y_tr, terminator def next_test(self, batch_size=1): start, end = self.idx_te, self.idx_te+batch_size x_te, y_te = self.x_te[start:end], self.y_te[start:end] x_te = np.expand_dims(x_te, axis=3) terminator = False if(end >= self.num_te): terminator = True self.idx_te = 0 else: self.idx_te = end if(self.normalize): min_x, max_x = x_te.min(), x_te.max() x_te = (x_te - min_x) / (max_x - min_x) return x_te, y_te, terminator # In[ ]: import tensorflow as tf class MemAE(object): def __init__(self, height, width, channel, leaning_rate=1e-3, ckpt_dir='./Checkpoint'): print("nInitializing Neural Network...") self.height, self.width, self.channel = height, width, channel self.leaning_rate = leaning_rate self.ckpt_dir = ckpt_dir self.name_bank, self.params_trainable, self.conv_shapes = [], [], [] self.initializer = tf.initializers.glorot_normal() z = self.encoder(x=tf.zeros([1, self.height, self.width, self.channel]), training=False, verbose=True) z_hat, _ = self.memory(z, verbose=True) x_hat = self.decoder(z_hat, training=False, verbose=True) self.optimizer = tf.optimizers.Adam(self.leaning_rate) self.summary_writer = tf.summary.create_file_writer(self.ckpt_dir) def step(self, x, iteration=0, train=False): with tf.GradientTape() as tape: z = self.encoder(x=x, training=train, verbose=False) z_hat, w_hat = self.memory(z, verbose=False) x_hat = self.decoder(z_hat, training=train, verbose=False) mse = tf.reduce_sum(tf.square(x - x_hat), axis=(1, 2, 3)) mem_etrp = tf.reduce_sum((-w_hat) * tf.math.log(w_hat + 1e-12), axis=(1, 2, 3)) loss = tf.reduce_mean(mse + (0.0002 * mem_etrp)) if(train): gradients = tape.gradient(loss, self.params_trainable) self.optimizer.apply_gradients(zip(gradients, self.params_trainable)) with self.summary_writer.as_default(): tf.summary.scalar('MemAE/MSE', tf.reduce_mean(mse), step=iteration) tf.summary.scalar('MemAE/Cross_Entropy', tf.reduce_mean(mem_etrp), step=iteration) tf.summary.scalar('MemAE/Total_Loss', loss, step=iteration) return x_hat, mse, mem_etrp, loss def save_params(self): vars_to_save = {} for idx, name in enumerate(self.name_bank): vars_to_save[self.name_bank[idx]] = self.params_trainable[idx] vars_to_save["optimizer"] = self.optimizer ckpt = tf.train.Checkpoint(**vars_to_save) ckptman = tf.train.CheckpointManager(ckpt, directory=self.ckpt_dir, max_to_keep=3) ckptman.save() def load_params(self): vars_to_load = {} for idx, name in enumerate(self.name_bank): vars_to_load[self.name_bank[idx]] = self.params_trainable[idx] vars_to_load["optimizer"] = self.optimizer ckpt = tf.train.Checkpoint(**vars_to_load) latest_ckpt = tf.train.latest_checkpoint(self.ckpt_dir) status = ckpt.restore(latest_ckpt) status.expect_partial() def get_weight(self, vshape, transpose=False, bias=True, name=""): try: idx_w = self.name_bank.index("%s_w" %(name)) if(bias): idx_b = self.name_bank.index("%s_b" %(name)) except: w = tf.Variable(self.initializer(vshape), name="%s_w" %(name), trainable=True, dtype=tf.float32) self.name_bank.append("%s_w" %(name)) self.params_trainable.append(w) if(bias): if(transpose): b = tf.Variable(self.initializer([vshape[-2]]), name="%s_b" %(name), trainable=True, dtype=tf.float32) else: b = tf.Variable(self.initializer([vshape[-1]]), name="%s_b" %(name), trainable=True, dtype=tf.float32) self.name_bank.append("%s_b" %(name)) self.params_trainable.append(b) else: w = self.params_trainable[idx_w] if(bias): b = self.params_trainable[idx_b] if(bias): return w, b else: return w def conv2d(self, inputs, variables, stride_size, padding): [weights, biasis] = variables out = tf.nn.conv2d(inputs, weights, strides=[1, stride_size, stride_size, 1], padding=padding) + biasis return out def conv2d_tr(self, inputs, variables, output_shape, stride_size, padding): [weights, biasis] = variables out = tf.nn.conv2d_transpose(inputs, weights, output_shape, strides=[1, stride_size, stride_size, 1], padding=padding) + biasis return out def batch_norm(self, inputs, name=""): # https://arxiv.org/pdf/1502.03167.pdf mean = tf.reduce_mean(inputs) std = tf.math.reduce_std(inputs) var = std**2 try: idx_offset = self.name_bank.index("%s_offset" %(name)) idx_scale = self.name_bank.index("%s_scale" %(name)) except: offset = tf.Variable(0, name="%s_offset" %(name), trainable=True, dtype=tf.float32) self.name_bank.append("%s_offset" %(name)) self.params_trainable.append(offset) scale = tf.Variable(1, name="%s_scale" %(name), trainable=True, dtype=tf.float32) self.name_bank.append("%s_scale" %(name)) self.params_trainable.append(scale) else: offset = self.params_trainable[idx_offset] scale = self.params_trainable[idx_scale] offset # zero scale # one out = tf.nn.batch_normalization( x = inputs, mean=mean, variance=var, offset=offset, scale=scale, variance_epsilon=1e-12, name=name ) return out def cosine_sim(self, x1, x2): num = tf.linalg.matmul(x1, tf.transpose(x2, perm=[0, 1, 3, 2]), name='attention_num') denom = tf.linalg.matmul(x1**2, tf.transpose(x2, perm=[0, 1, 3, 2])**2, name='attention_denum') w = (num + 1e-12) / (denom + 1e-12) return w def encoder(self, x, training=False, verbose=False): if(verbose): print("Encoder:", x.shape) self.conv_shapes.append(x.shape) encode1 = self.conv2d(inputs=x, variables=self.get_weight(vshape=[1, 1, self.channel, 16], name="encode1"), stride_size=2, padding='SAME') encode1_bn = self.batch_norm(inputs=encode1, name="encode1_bn") encode1_act = tf.keras.activations.relu(encode1_bn) if(verbose): print(encode1_act.shape) self.conv_shapes.append(encode1_act.shape) encode2 = self.conv2d(inputs=encode1_act, variables=self.get_weight(vshape=[3, 3, 16, 32], name="encode2"), stride_size=2, padding='SAME') encode2_bn = self.batch_norm(inputs=encode2, name="encode2_bn") encode2_act = tf.keras.activations.relu(encode2_bn) if(verbose): print(encode2_act.shape) self.conv_shapes.append(encode2_act.shape) encode3 = self.conv2d(inputs=encode2_act, variables=self.get_weight(vshape=[3, 3, 32, 64], name="encode3"), stride_size=2, padding='SAME') encode3_bn = self.batch_norm(inputs=encode3, name="encode3_bn") encode3_act = tf.keras.activations.relu(encode3_bn) if(verbose): print(encode3_act.shape) return encode3_act def memory(self, z, verbose=False): if(verbose): print("Memory:", z.shape) N = 2000 w_memory = self.get_weight(vshape=[1, 1, N, 64], bias=False, name="memory") cosim = self.cosine_sim(x1=z, x2=w_memory) # Eq.5 atteniton = tf.nn.softmax(cosim) # Eq.4 if(verbose): print(atteniton.shape) lam = 1 / N # deactivate the 1/N of N memories. addr_num = tf.keras.activations.relu(atteniton - lam) * atteniton addr_denum = tf.abs(atteniton - lam) + 1e-12 memory_addr = addr_num / addr_denum if(verbose): print(memory_addr.shape) renorm = tf.clip_by_value(memory_addr, 1e-12, 1-(1e-12)) z_hat = tf.linalg.matmul(renorm, w_memory) if(verbose): print(z_hat.shape) return z_hat, renorm def decoder(self, z_hat, training=False, verbose=False): if(verbose): print("Decoder:", z_hat.shape) [n, h, w, c] = self.conv_shapes[-1] decode1 = self.conv2d_tr(inputs=z_hat, variables=self.get_weight(vshape=[3, 3, c, 64], transpose=True, name="decode1"), output_shape=[tf.shape(z_hat)[0], 7, 7, 32], stride_size=2, padding='SAME') decode1_bn = self.batch_norm(inputs=decode1, name="decode1_bn") decode1_act = tf.keras.activations.relu(decode1_bn) if(verbose): print(decode1_act.shape) [n, h, w, c] = self.conv_shapes[-2] decode2 = self.conv2d_tr(inputs=decode1_act, variables=self.get_weight(vshape=[3, 3, c, 32], transpose=True, name="decode2"), output_shape=[tf.shape(decode1_act)[0], h, w, c], stride_size=2, padding='SAME') decode2_bn = self.batch_norm(inputs=decode2, name="decode2_bn") decode2_act = tf.keras.activations.relu(decode2_bn) if(verbose): print(decode2_act.shape) [n, h, w, c] = self.conv_shapes[-3] decode3 = self.conv2d_tr(inputs=decode2_act, variables=self.get_weight(vshape=[3, 3, c, 16], transpose=True, name="decode3"), output_shape=[tf.shape(decode2_act)[0], h, w, c], stride_size=2, padding='SAME') if(verbose): print(decode3.shape) x_hat = tf.clip_by_value(decode3, 1e-12, 1-(1e-12)) return x_hat # In[ ]: # In[24]: import os, inspect, time, math import tensorflow as tf import numpy as np import matplotlib.pyplot as plt PACK_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))+"/.." def make_dir(path): try: os.mkdir(path) except: pass def gray2rgb(gray): rgb = np.ones((gray.shape[0], gray.shape[1], 3)).astype(np.float32) rgb[:, :, 0] = gray[:, :, 0] rgb[:, :, 1] = gray[:, :, 0] rgb[:, :, 2] = gray[:, :, 0] return rgb def dat2canvas(data): numd = math.ceil(np.sqrt(data.shape[0])) [dn, dh, dw, dc] = data.shape canvas = np.ones((dh*numd, dw*numd, dc)).astype(np.float32) for y in range(numd): for x in range(numd): try: tmp = data[x+(y*numd)] except: pass else: canvas[(y*dh):(y*dh)+28, (x*dw):(x*dw)+28, :] = tmp if(dc == 1): canvas = gray2rgb(gray=canvas) return canvas def save_img(contents, names=["", "", ""], savename=""): num_cont = len(contents) plt.figure(figsize=(5*num_cont+2, 5)) for i in range(num_cont): plt.subplot(1,num_cont,i+1) plt.title(names[i]) plt.imshow(dat2canvas(data=contents[i])) plt.tight_layout() plt.savefig(savename) plt.close() def discrete_cmap(N, base_cmap=None): base = plt.cm.get_cmap(base_cmap) color_list = base(np.linspace(0, 1, N)) cmap_name = base.name + str(N) return base.from_list(cmap_name, color_list, N) def latent_plot(latent, y, n, savename=""): plt.figure(figsize=(6, 5)) plt.scatter(latent[:, 0], latent[:, 1], c=y, marker='o', edgecolor='none', cmap=discrete_cmap(n, 'jet')) plt.colorbar(ticks=range(n)) plt.grid() plt.show() # plt.tight_layout() # plt.savefig(savename) # plt.close() def boxplot(contents, savename=""): data, label = [], [] for cidx, content in enumerate(contents): data.append(content) label.append("class-%d" %(cidx)) plt.clf() fig, ax1 = plt.subplots() bp = ax1.boxplot(data, showfliers=True, whis=3) ax1.set_xticklabels(label, rotation=45) plt.show() # plt.tight_layout() # plt.savefig(savename) # plt.close() def histogram(contents, savename=""): n1, _, _ = plt.hist(contents[0], bins=100, alpha=0.5, label='Normal') n2, _, _ = plt.hist(contents[1], bins=100, alpha=0.5, label='Abnormal') h_inter = np.sum(np.minimum(n1, n2)) / np.sum(n1) plt.xlabel("MSE") plt.ylabel("Number of Data") xmax = max(contents[0].max(), contents[1].max()) plt.xlim(0, xmax) plt.text(x=xmax*0.01, y=max(n1.max(), n2.max()), s="Histogram Intersection: %.3f" %(h_inter)) plt.legend(loc='upper right') plt.show() # plt.savefig(savename) # plt.close() def training(neuralnet, dataset, epochs, batch_size, normalize=True): print("nTraining to %d epochs (%d of minibatch size)" %(epochs, batch_size)) make_dir(path="results") result_list = ["tr_resotring"] for result_name in result_list: make_dir(path=os.path.join("results", result_name)) start_time = time.time() iteration = 0 test_sq = 20 test_size = test_sq**2 for epoch in range(epochs): x_tr, y_tr, _ = dataset.next_train(batch_size=test_size, fix=True) # Initial batch x_restore, _, _, _ = neuralnet.step(x=x_tr, iteration=iteration, train=False) save_img(contents=[x_tr, x_restore, (x_tr-x_restore)**2], names=["Inputn(x)", "Restorationn(x to x-hat)", "Difference"], savename=os.path.join("results", "tr_resotring", "%08d.png" %(epoch))) while(True): x_tr, y_tr, terminator = dataset.next_train(batch_size) # y_tr does not used in this prj. _, mse, w_etrp, loss = neuralnet.step(x=x_tr, iteration=iteration, train=True) neuralnet.save_params() iteration += 1 if(terminator): break print("Epoch [%d / %d] (%d iteration) MSE:%.3f, W-ETRP:%.3f, Total:%.3f" %(epoch, epochs, iteration, np.sum(mse), np.sum(w_etrp), loss)) def test(neuralnet, dataset, batch_size): neuralnet.load_params() print("nTest...") make_dir(path="test") result_list = ["inbound", "outbound"] for result_name in result_list: make_dir(path=os.path.join("test", result_name)) scores_normal, scores_abnormal = [], [] while(True): x_te, y_te, terminator = dataset.next_test(1) # y_te does not used in this prj. _, score_anomaly, _, _ = neuralnet.step(x=x_te, iteration=0, train=False) if(y_te == 1): scores_normal.append(score_anomaly) else: scores_abnormal.append(score_anomaly) if(terminator): break scores_normal = np.asarray(scores_normal) scores_abnormal = np.asarray(scores_abnormal) normal_avg, normal_std = np.average(scores_normal), np.std(scores_normal) abnormal_avg, abnormal_std = np.average(scores_abnormal), np.std(scores_abnormal) print("Noraml avg: %.5f, std: %.5f" %(normal_avg, normal_std)) print("Abnoraml avg: %.5f, std: %.5f" %(abnormal_avg, abnormal_std)) outbound = normal_avg + (normal_std * 3) print("Outlier boundary of normal data: %.5f" %(outbound)) histogram(contents=[scores_normal, scores_abnormal], savename="histogram-test.png") fcsv = open("test-summary.csv", "w") fcsv.write("class, loss, outliern") testnum = 0 loss4box = [[], [], [], [], [], [], [], [], [], []] while(True): x_te, y_te, terminator = dataset.next_test(1) # y_te does not used in this prj. x_restore, restore_loss, _, _ = neuralnet.step(x=x_te, iteration=0, train=False) loss4box[y_te[0]].append(restore_loss) outcheck = restore_loss > outbound fcsv.write("%d, %.3f, %rn" %(y_te, restore_loss, outcheck)) [h, w, c] = x_te[0].shape canvas = np.ones((h, w*3, c), np.float32) canvas[:, :w, :] = x_te[0] canvas[:, w:w*2, :] = x_restore[0] canvas[:, w*2:, :] = (x_te[0]-x_restore[0])**2 if(outcheck): print("Out bound ", int(restore_loss)) plt.imshow(gray2rgb(gray=canvas)) plt.show() else: print("In bound ", int(restore_loss)) plt.imshow(gray2rgb(gray=canvas)) plt.show() plt.imsave(os.path.join("test", "inbound", "%08d-%08d.png" %(testnum, int(restore_loss))), gray2rgb(gray=canvas)) testnum += 1 if(terminator): break return loss4box #boxplot(contents=loss4box, savename="test-box.png") # In[ ]: # In[12]: PACK_PATH = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) CKPT_DIR = PACK_PATH+'/Checkpoint' class my_args(): #@staticmethod def __init__(self): self.batch_size = 32 self.lr = 1e-5 self.epochs = 200 ##### dataset args self.datanorm = True self.height = 28 self.width = 28 self.channel = 1 self.CKPT_DIR = CKPT_DIR args = my_args() # In[16]: (x_tr, y_tr), (x_te, y_te) = tf.keras.datasets.mnist.load_data() # In[18]: dataset = Dataset( x_tr, y_tr, x_te, y_te, normalize=args.datanorm) # In[7]: physical_devices = tf.config.list_physical_devices('GPU') tf.config.experimental.set_memory_growth(physical_devices[0], True) #dataset = dman.Dataset(normalize=FLAGS.datnorm) neuralnet = nn.MemAE(height=args.height, width=args.width, channel=args.channel, leaning_rate=args.lr, ckpt_dir=args.CKPT_DIR) # In[22]: tfp.training(neuralnet=neuralnet, dataset=dataset, epochs=args.epochs, batch_size=args.batch_size, normalize=True) # In[25]: aa = test(neuralnet=neuralnet, dataset=dataset, batch_size=args.batch_size) # In[ ]:

test

member SPaC — Tue, 24 Nov 2015 15:55:33 +0000

test

워드프레스 KBoard 피드

deno

MAE

test