Select Git revision
class_ISTM.py
class_ISTM.py 19.53 KiB
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 31 13:05:25 2018
@author: taopeng
"""
from _4rd_RegressionIncremental_andMethodAncien import IRL
import numpy as np
from datetime import datetime
import copy
import sys
class Chrono:
def __init__(self):
self.start = datetime.utcnow()
def fini(self):
end = datetime.utcnow()
# c = (end - self.start).microseconds
c = (end - self.start).seconds
return c
'''
ProcessIRL 是为了一次完整的修复工作而产生一个实例，
ProcessIRL： générer une instance pour un travail de réparation complet
'''
class ProcessIRL:
def __init__(self,E_original,E_aReparer,tauxTrain,trou,backTime,listVoisinNumero ,testModel='no'):
self.E_original = copy.deepcopy(E_original)
self.E_a_reparer = copy.deepcopy(E_aReparer)
self.E_resultat = copy.deepcopy(E_aReparer)
self.trou = trou
self.backTime = backTime
self.indexFinInitiation = int(len(E_original[0])*tauxTrain)#
#为了节约测试时间，并不使用到最后一个时间点的数据，而是在分分界点前后各取10%
# Afin de gagner du temps lors des essais, "testModel" ne prends que 10 % des donnees avant et après
if testModel == 'yes':
self.idnexStarInitiation = self.backTime + int(len(E_original[0])*(tauxTrain - 0.1))
self.indexFini_MIS_a_jour = int(len(E_original[0])*(tauxTrain + 0.02))
else:
self.idnexStarInitiation = backTime
self.indexFini_MIS_a_jour = len(E_original[0])
self.listVoisinNumero = listVoisinNumero
# print(self.listVoisinNumero)
#提取传感器名单 Extraction de la liste des capteurs
self.listNumeroSensor = []
for c in self.listVoisinNumero:
self.listNumeroSensor.append(c[0])
#model 用来存储实例， numeroModel用来存储其对应的传感器编号，将两者分开，使得regression只负责运算和数据来源脱钩
# model Utilisé pour stocker les instances （procesuss de reparation）
self.model = []
#存储消耗的时间 Temps consommé
self.comsomationTime = 0
#存储SSE值 Stockage des valeurs de l'ESS
self.SSE_score = 0
self.numDonneesManquanesInTest = 0
# print (id(self.E_a_reparer),id(self.E_resultat))
def getComsomationTime(self):
return self.comsomationTime
def calcule_and_getSSE(self):
a = []
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
for indexSensor in range(0,len(self.E_original)):
if self.E_a_reparer[indexSensor][timePoint] == -2:
self.SSE_score = self.SSE_score + np.square(self.E_original[indexSensor][timePoint]-self.E_resultat[indexSensor][timePoint])
self.numDonneesManquanesInTest +=1
a.append(self.SSE_score)
# print ("总分数 %d, 修复的个数 %d" %(self.SSE_score,self.numDonneesManquanesInTest ))
# print (self.SSE_score, self.numDonneesManquanesInTest)
# np.savetxt('SSE_score_test.csv',a)
return self.SSE_score/self.numDonneesManquanesInTest
def get_E_resultat_E_original_E_a_reparer(self):
E_resultat = np.array(self.E_resultat)[:,self.indexFinInitiation : self.indexFini_MIS_a_jour]
E_original = np.array(self.E_original)[:,self.indexFinInitiation : self.indexFini_MIS_a_jour]
E_a_reparer = np.array(self.E_a_reparer)[:,self.indexFinInitiation : self.indexFini_MIS_a_jour]
# print('np.array(self.E_resultat).shape ',np.array(self.E_resultat).shape)
# print('E_resultat.shape ', E_resultat.shape)
return E_resultat,E_original,E_a_reparer
def get_E_resultat_E_original_E_a_reparer_all(self):
E_resultat = np.array(self.E_resultat)
E_original = np.array(self.E_original)
E_a_reparer = np.array(self.E_a_reparer)
# print('np.array(self.E_resultat).shape ',np.array(self.E_resultat).shape)
# print('E_resultat.shape ', E_resultat.shape)
return E_resultat,E_original,E_a_reparer
#########################about ISTM############################################
#获取一个T形窗户里所有的信息 Obtenez toutes les informations dans une fenêtre en forme de T
def getFeatureOfISTM(self,indexSensor,timePoint):
#获取目标节点的邻居节点编号 Obtenir le numéro du nœud voisin du nœud cible
myVoisins = self.listVoisinNumero[indexSensor][1:]#
# Obtenir le numéro du nœud voisin
IndexMyVoisins = []
for c in range(0,len(myVoisins)):
# print (c)
# print (myVoisins[c])
IndexMyVoisins.append(self.listNumeroSensor.index(myVoisins[c]))
# 获取上一时间节点，周围邻居节点的观测值 Obtenir l'observation du nœud voisin au point temporel précédent
ValueVoisinPreviousTimeNode= []
for c in range(0,len(IndexMyVoisins)):
ValueVoisinPreviousTimeNode.append(self.E_resultat[IndexMyVoisins[c]][timePoint-1])
# 获得本观测点的最近的历史数据 Obtenir les données historiques les plus récentes pour ce point d'observation
ValueMyHistoricalData= self.E_resultat[indexSensor][(timePoint-self.backTime):timePoint]
return ValueMyHistoricalData + ValueVoisinPreviousTimeNode, self.E_resultat[indexSensor][timePoint]
# 在getFeatureOfISTM基础上，准备训练（初始化）数据集
#Avec getFeatureOfISTM, préparez (initialisez) le jeu de données
def getEnsembleInitiation_FeatureOfISTM(self,indexSensor):
X = []
Y = []
for timePiont in range(self.idnexStarInitiation,self.indexFinInitiation):
tempX, tempY = self.getFeatureOfISTM(indexSensor,timePiont)
X.append(tempX)
Y.append(tempY)
return X,np.transpose(Y).tolist()
# 初始化
# initialiser
def trainFirst_ISTM(self):
for indexSensor in range(0,len(self.E_original)):
temp = IRL()
X,Y = self.getEnsembleInitiation_FeatureOfISTM(indexSensor)
temp.fistTraining(X,Y)
self.model.append(temp)
print ("trainFirst_ISTM initialtion fini")
# 利用 Incremental RL 更新模型，一个更新一个实例。
# Mettre à jour le modèle en utilisant le RL incrémental, une mise à jour par instance.
def prosser_unParUn_jusquaFini_ISTM(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
if Y == -1 or Y == -2:
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
else:
self.model[indexSensor].misAjour_INCRE_unParUn(X,Y) # MIS A Jour
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
# 利用 Incremental RL 更新模型
#，Ne mettre à jour le modèle que lorsque des valeurs manquantes sont rencontrées
def prosser_unParUn_jusquaFini_ISTM_OPM(self): #
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
#sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
if Y == -1 or Y == -2:#遇到缺失值了 J'ai rencontré une valeur manquante.
bufferX, bufferY = self.model[indexSensor].buffer_Get()# Obtenir de nouvelles données dans le buffer
if len(bufferY) != 0:# ne mettre à jour le modèle que lorsqu'il y a des données dans le cache
self.model[indexSensor].buffuer_Clean()# Vider le cbuffer
self.model[indexSensor].misAJour_INCRE_lot(bufferX,bufferY)# Mise à jour avec de nouvelles données avec " misAJour_INCRE_lot "
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
# print (self.E_resultat[indexSensor][timePoint],self.E_original[indexSensor][timePoint] )
else:
self.model[indexSensor].buffuer_Add(X,Y)#Lorsque vous rencontrez des données normales, mettez-les d'abord dans le cache,
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
def prosser_unParUn_jusquaFini_ISTM_OPM_fraiche(self,fraiche):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
#sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
if Y == -1 or Y == -2:#遇到缺失值了
bufferX, bufferY = self.model[indexSensor].buffer_Get()#得到新数据
if len(bufferY) != 0:#保险，当缓存里有数据时才更新模型
self.model[indexSensor].buffuer_Clean()#清空缓存
# self.model[indexSensor].lossFunction_misAjour(fraiche,len(bufferY))# 使用损失函数对过去的中间矩阵进行"降温"
self.model[indexSensor].lossFunction_misAjour(fraiche,len(bufferY))# 使用损失函数对过去的中间矩阵进行"降温"
self.model[indexSensor].misAJour_INCRE_lot(bufferX,bufferY)#使用新数据更新
try:#防止遇到奇异矩阵
self.E_resultat[indexSensor][timePoint] = np.absolute(self.model[indexSensor].predictionUn(X)) # 使用绝对值,提升准确率
except:#解决方案是用前值 / 50
self.E_resultat[indexSensor][timePoint] = self.E_resultat[indexSensor][timePoint-1]
# self.E_resultat[indexSensor][timePoint] = 50
# print (self.E_resultat[indexSensor][timePoint],self.E_original[indexSensor][timePoint] )
else:
self.model[indexSensor].buffuer_Add(X,Y)#遇到正常数据了先放进缓存，不着急更新
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
########################time-based#############
# Les données de référence pour les valeurs manquantes proviennent uniquement de données historiques pour cette point d‘observation
# 输入传感器位置(哪个传感器)，哪个时间点
#返回某一个传感器过去小段历史值
def getRectenlyHistoricData(self,indexSensor,timePoint):
ValueMyHistoricalData = self.E_resultat[indexSensor][timePoint-self.backTime:timePoint]
return ValueMyHistoricalData,self.E_resultat[indexSensor][timePoint]#返回某一个传感器过去小段历史值(前)，当前值(后)
# 输入：传感器编号
#获得某个时间点前所有数据历史数据(可以用在(单纯)时间模型的初始化上)
def getEnsemble_RectenlyHistoricData(self,indexSensor):#返回某一个传感器过去(所有组合)小段历史值(前)，当前值(后)
X = []
Y = []
for timePiont in range(self.idnexStarInitiation,self.indexFinInitiation):
tempX, tempY = self.getRectenlyHistoricData(indexSensor,timePiont)
X.append(tempX)
Y.append(tempY)
return X,np.transpose(Y).tolist()
def trianFirst_time_based_model(self):
for indexSensor in range(0,len(self.E_original)):
temp = IRL()
X,Y = self.getEnsemble_RectenlyHistoricData(indexSensor)
temp.fistTraining(X,Y)
self.model.append(temp)
print ("trianFirst_time_based_model initialtion fini")
def prosser_unParUn_jusquaFini_time_based_model(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getRectenlyHistoricData(indexSensor,timePoint)
if Y == -1 or Y == -2:
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
########################往下space-based#############
# Les données de référence pour les valeurs manquantes proviennent uniquement de données des ses voision au point du temps precedents
def getPrivousDataVoisin(self,indexSensor,timePoint):
# 获取目标节点的邻居节点编号
myVoisins = self.listVoisinNumero[indexSensor][1:]#
# 获取周围邻居结点在E中位置，其实是利用indexInE来获得
IndexMyVoisins = []
for c in range(0,len(myVoisins)):
IndexMyVoisins.append(self.listNumeroSensor.index(myVoisins[c]))
# 获取上一时间节点，周围邻居节点的观测值
ValueVoisinPreviousTimeNode= []
for c in range(0,len(IndexMyVoisins)):
ValueVoisinPreviousTimeNode.append(self.E_resultat[IndexMyVoisins[c]][timePoint-1])
return ValueVoisinPreviousTimeNode, self.E_resultat[indexSensor][timePoint]
def getEnsemble_PrivousDataVoisin(self,indexSensor):
X = []
Y = []
for timePiont in range(self.idnexStarInitiation,self.indexFinInitiation):
tempX, tempY = self.getPrivousDataVoisin(indexSensor,timePiont)
X.append(tempX)
Y.append(tempY)
return X,np.transpose(Y).tolist()
def trianFirst_space_based_model(self):
for indexSensor in range(0,len(self.E_original)):
temp = IRL()
X,Y = self.getEnsemble_PrivousDataVoisin(indexSensor)
temp.fistTraining(X,Y)
self.model.append(temp)
print ("trianFirst_space_based_model initialtion fini")
def prosser_unParUn_jusquaFini_space_based_model(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getPrivousDataVoisin(indexSensor,timePoint)
if Y == -1 or Y == -2:
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
########################PrivousValue#############
def prosser_PrivousValue_model(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
Y = self.E_resultat[indexSensor][timePoint]
if Y == -1 or Y == -2:
self.E_resultat[indexSensor][timePoint] = self.E_resultat[indexSensor][timePoint-1]
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
########################Moyenne#############
def trainFirst_moyenne_model(self):
for indexSensor in range(0,len(self.E_original)):
temp = IRL()
temp.meanOfY(self.E_resultat[indexSensor][self.idnexStarInitiation:self.indexFinInitiation])
self.model.append(temp)
print ("trainFirst_moyenne_model initialtion fini")
def prosser_moyenne_model(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
Y = self.E_resultat[indexSensor][timePoint]
if Y == -1 or Y == -2:
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].getMeanY()
else:
self.model[indexSensor].misAjour_meanOfY(Y)
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
########################Static_time_space_model#############
# like ISTM, sans mis a jour
def trainFirst_Static_time_space_model(self):
self.trainFirst_ISTM()
print ("trainFirst_Static_time_space_model initialtion fini")
def prosser_unParUn_jusquaFini_STATIC_Time_space_based_model(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
if Y == -1 or Y == -2:
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
########################_Dynamique_but_stupide_time_space_model#############
# like ISTM, mais la mis à jour est " retraing with all old data "
def trainFirst_Dymanic_TRA_opm_time_space_model(self):
self.trainFirst_ISTM()
print ("trainFirst_Dymanic_TRA_opm_time_space_model initialtion fini")
def prosser_unParUn_jusquaFini_Dymanic_TRA_opm_Time_space_based_model(self):
tempTRL = Chrono()
for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
sys.stdout.flush()
for indexSensor in range(0,len(self.E_original)):
X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
if Y == -1 or Y == -2:
self.self.model[indexSensor].misAjour_TRA_recalcule_matric_intermediaire()
self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
else:
self.model[indexSensor].misAJour_TRA_Just_add_XY(X,Y) #仅添加新数据
self.comsomationTime = tempTRL.fini()
return self.comsomationTime,self.calcule_and_getSSE()
#
#E_original =np.load("E_origninal.npy").tolist()
#E_aReparer =np.load("simulationDonnees/E5_tauxTrain90.npy").tolist()
#tauxTrain = 0.9
#trou = 5
#backTime = 6
#
#
#oneProcessTest = ProcessIRL(E_original,E_aReparer,tauxTrain,trou,backTime,testModel='yes')
#oneProcessTest.trainFirst_ISTM()
#print oneProcessTest.prosser_unParUn_jusquaFini_ISTM()