class_ISTM.py

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""  
Created on Wed Oct 31 13:05:25 2018
新的变化
@author: taopeng
"""


from _4rd_RegressionIncremental_andMethodAncien import IRL
import numpy as np

from datetime import datetime  
import copy
import sys
   
class Chrono: 
    def __init__(self):
        self.start = datetime.utcnow()
    def fini(self):
        end = datetime.utcnow()  
#        c = (end - self.start).microseconds
        c = (end - self.start).seconds
        return c

'''
ProcessIRL 是为了一次完整的修复工作而产生一个实例，
ProcessIRL： générer une instance pour un travail de réparation complet
'''
class ProcessIRL:
    def __init__(self,E_original,E_aReparer,tauxTrain,trou,backTime,listVoisinNumero ,testModel='no'):
        self.E_original = copy.deepcopy(E_original) 
        self.E_a_reparer = copy.deepcopy(E_aReparer) 
        self.E_resultat = copy.deepcopy(E_aReparer)
        
        self.trou = trou
        self.backTime = backTime
        
        self.indexFinInitiation = int(len(E_original[0])*tauxTrain)#
        #为了节约测试时间，并不使用到最后一个时间点的数据，而是在分分界点前后各取10%
        #  Afin de gagner du temps lors des essais, "testModel" ne prends que 10 % des donnees avant et après 
        if testModel == 'yes':
            self.idnexStarInitiation = self.backTime + int(len(E_original[0])*(tauxTrain - 0.1))
            self.indexFini_MIS_a_jour = int(len(E_original[0])*(tauxTrain + 0.02))
        else:
            self.idnexStarInitiation = backTime
            self.indexFini_MIS_a_jour = len(E_original[0]) 
        
        self.listVoisinNumero = listVoisinNumero
        

#        print(self.listVoisinNumero)
        #提取传感器名单 Extraction de la liste des capteurs
        self.listNumeroSensor = []
        for c in self.listVoisinNumero:
            self.listNumeroSensor.append(c[0])
        
        #model 用来存储实例， numeroModel用来存储其对应的传感器编号，将两者分开，使得regression只负责运算和数据来源脱钩
        # model Utilisé pour stocker les instances （procesuss de reparation）
        self.model = []

        #存储消耗的时间 Temps consommé
        self.comsomationTime = 0
        
        #存储SSE值 Stockage des valeurs de l'ESS
        self.SSE_score = 0
        self.numDonneesManquanesInTest = 0
        
        
#        print (id(self.E_a_reparer),id(self.E_resultat))

    def getComsomationTime(self):
        return self.comsomationTime
        
    def calcule_and_getSSE(self):
        a = []
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            for indexSensor in range(0,len(self.E_original)):
                if self.E_a_reparer[indexSensor][timePoint] == -2:
                    self.SSE_score = self.SSE_score + np.square(self.E_original[indexSensor][timePoint]-self.E_resultat[indexSensor][timePoint])
                    self.numDonneesManquanesInTest +=1
                    a.append(self.SSE_score)
        # print ("总分数 %d, 修复的个数 %d" %(self.SSE_score,self.numDonneesManquanesInTest ))
        # print (self.SSE_score, self.numDonneesManquanesInTest)
#        np.savetxt('SSE_score_test.csv',a)
        return self.SSE_score/self.numDonneesManquanesInTest 
    

    def get_E_resultat_E_original_E_a_reparer(self):
        E_resultat = np.array(self.E_resultat)[:,self.indexFinInitiation : self.indexFini_MIS_a_jour]
        E_original = np.array(self.E_original)[:,self.indexFinInitiation : self.indexFini_MIS_a_jour]
        E_a_reparer = np.array(self.E_a_reparer)[:,self.indexFinInitiation : self.indexFini_MIS_a_jour]
#        print('np.array(self.E_resultat).shape  ',np.array(self.E_resultat).shape)
#        print('E_resultat.shape  ', E_resultat.shape)
        return E_resultat,E_original,E_a_reparer

    def get_E_resultat_E_original_E_a_reparer_all(self):
        E_resultat = np.array(self.E_resultat)
        E_original = np.array(self.E_original)
        E_a_reparer = np.array(self.E_a_reparer)
#        print('np.array(self.E_resultat).shape  ',np.array(self.E_resultat).shape)
#        print('E_resultat.shape  ', E_resultat.shape)
        return E_resultat,E_original,E_a_reparer    
        


#########################about ISTM############################################
    #获取一个T形窗户里所有的信息 Obtenez toutes les informations dans une fenêtre en forme de T
    def getFeatureOfISTM(self,indexSensor,timePoint):
        #获取目标节点的邻居节点编号 Obtenir le numéro du nœud voisin du nœud cible
        myVoisins = self.listVoisinNumero[indexSensor][1:]#

        #   Obtenir le numéro du nœud voisin
        IndexMyVoisins = []
        for c in range(0,len(myVoisins)):
#            print (c)
#            print (myVoisins[c])
            IndexMyVoisins.append(self.listNumeroSensor.index(myVoisins[c]))
        #   获取上一时间节点，周围邻居节点的观测值 Obtenir l'observation du nœud voisin au point temporel précédent
        ValueVoisinPreviousTimeNode= []
        for c in range(0,len(IndexMyVoisins)):
            ValueVoisinPreviousTimeNode.append(self.E_resultat[IndexMyVoisins[c]][timePoint-1])
        # 获得本观测点的最近的历史数据 Obtenir les données historiques les plus récentes pour ce point d'observation
        ValueMyHistoricalData= self.E_resultat[indexSensor][(timePoint-self.backTime):timePoint]

        return ValueMyHistoricalData + ValueVoisinPreviousTimeNode, self.E_resultat[indexSensor][timePoint]
    
    # 在getFeatureOfISTM基础上，准备训练（初始化）数据集
    #Avec getFeatureOfISTM, préparez (initialisez) le jeu de données
    def getEnsembleInitiation_FeatureOfISTM(self,indexSensor):
        X = []
        Y = []
        for timePiont in range(self.idnexStarInitiation,self.indexFinInitiation):
            tempX, tempY = self.getFeatureOfISTM(indexSensor,timePiont)
            X.append(tempX)
            Y.append(tempY)
        return X,np.transpose(Y).tolist()  

    # 初始化
    # initialiser
    def trainFirst_ISTM(self):
        for indexSensor in range(0,len(self.E_original)):
            temp = IRL()
            X,Y = self.getEnsembleInitiation_FeatureOfISTM(indexSensor)
            temp.fistTraining(X,Y)
            self.model.append(temp) 
        print ("trainFirst_ISTM initialtion fini")

    # 利用 Incremental RL 更新模型，一个更新一个实例。
    # Mettre à jour le modèle en utilisant le RL incrémental, une mise à jour par instance.
    def prosser_unParUn_jusquaFini_ISTM(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
                if Y == -1 or Y == -2:
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
                else:
                    self.model[indexSensor].misAjour_INCRE_unParUn(X,Y)  # MIS A Jour
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE()
    
    # 利用 Incremental RL 更新模型
    #，Ne mettre à jour le modèle que lorsque des valeurs manquantes sont rencontrées
    def prosser_unParUn_jusquaFini_ISTM_OPM(self): # 
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            #sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
                if Y == -1 or Y == -2:#遇到缺失值了 J'ai rencontré une valeur manquante.
                    bufferX, bufferY = self.model[indexSensor].buffer_Get()# Obtenir de nouvelles données dans le buffer
                    if len(bufferY) != 0:# ne mettre à jour le modèle que lorsqu'il y a des données dans le cache
                        self.model[indexSensor].buffuer_Clean()# Vider le cbuffer
                        self.model[indexSensor].misAJour_INCRE_lot(bufferX,bufferY)# Mise à jour avec de nouvelles données avec " misAJour_INCRE_lot "
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
#                    print (self.E_resultat[indexSensor][timePoint],self.E_original[indexSensor][timePoint] )
                else:
                    self.model[indexSensor].buffuer_Add(X,Y)#Lorsque vous rencontrez des données normales, mettez-les d'abord dans le cache,
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE() 
    

    def prosser_unParUn_jusquaFini_ISTM_OPM_fraiche(self,fraiche):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            #sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
                if Y == -1 or Y == -2:#遇到缺失值了
                    bufferX, bufferY = self.model[indexSensor].buffer_Get()#得到新数据
                    if len(bufferY) != 0:#保险，当缓存里有数据时才更新模型
                        self.model[indexSensor].buffuer_Clean()#清空缓存
#                        self.model[indexSensor].lossFunction_misAjour(fraiche,len(bufferY))# 使用损失函数对过去的中间矩阵进行"降温"
                        self.model[indexSensor].lossFunction_misAjour(fraiche,len(bufferY))# 使用损失函数对过去的中间矩阵进行"降温"                       
                        self.model[indexSensor].misAJour_INCRE_lot(bufferX,bufferY)#使用新数据更新
                    try:#防止遇到奇异矩阵
                        self.E_resultat[indexSensor][timePoint] = np.absolute(self.model[indexSensor].predictionUn(X)) # 使用绝对值,提升准确率
                    except:#解决方案是用前值 / 50
                        self.E_resultat[indexSensor][timePoint] = self.E_resultat[indexSensor][timePoint-1]
#                        self.E_resultat[indexSensor][timePoint] = 50
#                    print (self.E_resultat[indexSensor][timePoint],self.E_original[indexSensor][timePoint] )
                else:                    
                    self.model[indexSensor].buffuer_Add(X,Y)#遇到正常数据了先放进缓存，不着急更新
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE()     
    
    
########################time-based#############
    # Les données de référence pour les valeurs manquantes proviennent uniquement de données historiques pour cette point d‘observation
    # 输入传感器位置(哪个传感器)，哪个时间点
    #返回某一个传感器过去小段历史值
    def getRectenlyHistoricData(self,indexSensor,timePoint):
        ValueMyHistoricalData = self.E_resultat[indexSensor][timePoint-self.backTime:timePoint]
        return ValueMyHistoricalData,self.E_resultat[indexSensor][timePoint]#返回某一个传感器过去小段历史值(前)，当前值(后)
        
    # 输入：传感器编号
    #获得某个时间点前所有数据历史数据(可以用在(单纯)时间模型的初始化上)
    def getEnsemble_RectenlyHistoricData(self,indexSensor):#返回某一个传感器过去(所有组合)小段历史值(前)，当前值(后)
        X = []
        Y = []
        for timePiont in range(self.idnexStarInitiation,self.indexFinInitiation):
            tempX, tempY = self.getRectenlyHistoricData(indexSensor,timePiont)
            X.append(tempX)
            Y.append(tempY)
        return X,np.transpose(Y).tolist()
    
    def trianFirst_time_based_model(self):
        for indexSensor in range(0,len(self.E_original)):
            temp = IRL()
            X,Y = self.getEnsemble_RectenlyHistoricData(indexSensor)
            temp.fistTraining(X,Y)
            self.model.append(temp)
        print ("trianFirst_time_based_model initialtion fini")
  
    def prosser_unParUn_jusquaFini_time_based_model(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getRectenlyHistoricData(indexSensor,timePoint)
                if Y == -1 or Y == -2:
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE()

########################往下space-based#############
# Les données de référence pour les valeurs manquantes proviennent uniquement de données des ses voision au point du temps precedents 

    def getPrivousDataVoisin(self,indexSensor,timePoint):
#    获取目标节点的邻居节点编号
        myVoisins = self.listVoisinNumero[indexSensor][1:]#
#   获取周围邻居结点在E中位置，其实是利用indexInE来获得
        IndexMyVoisins = []
        for c in range(0,len(myVoisins)):
            IndexMyVoisins.append(self.listNumeroSensor.index(myVoisins[c]))
#   获取上一时间节点，周围邻居节点的观测值
        ValueVoisinPreviousTimeNode= []
        for c in range(0,len(IndexMyVoisins)):
            ValueVoisinPreviousTimeNode.append(self.E_resultat[IndexMyVoisins[c]][timePoint-1])
        return ValueVoisinPreviousTimeNode, self.E_resultat[indexSensor][timePoint]
    
    def getEnsemble_PrivousDataVoisin(self,indexSensor):
        X = []
        Y = []
        for timePiont in range(self.idnexStarInitiation,self.indexFinInitiation):
            tempX, tempY = self.getPrivousDataVoisin(indexSensor,timePiont)
            X.append(tempX)
            Y.append(tempY)
        return X,np.transpose(Y).tolist()
        
    def trianFirst_space_based_model(self):
        for indexSensor in range(0,len(self.E_original)):
            temp = IRL()
            X,Y = self.getEnsemble_PrivousDataVoisin(indexSensor)
            temp.fistTraining(X,Y)
            self.model.append(temp)
        print ("trianFirst_space_based_model initialtion fini")

    def prosser_unParUn_jusquaFini_space_based_model(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getPrivousDataVoisin(indexSensor,timePoint)
                if Y == -1 or Y == -2:
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE() 
    
########################PrivousValue#############     
    def prosser_PrivousValue_model(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                Y = self.E_resultat[indexSensor][timePoint]
                if Y == -1 or Y == -2:
                    self.E_resultat[indexSensor][timePoint] = self.E_resultat[indexSensor][timePoint-1]
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE() 
        
########################Moyenne#############
    def trainFirst_moyenne_model(self):
        for indexSensor in range(0,len(self.E_original)):
            temp = IRL()
            temp.meanOfY(self.E_resultat[indexSensor][self.idnexStarInitiation:self.indexFinInitiation])
            self.model.append(temp)
        print ("trainFirst_moyenne_model initialtion fini")

    
    def prosser_moyenne_model(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                Y = self.E_resultat[indexSensor][timePoint]
                if Y == -1 or Y == -2:
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].getMeanY()
                else:
                    self.model[indexSensor].misAjour_meanOfY(Y)
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE() 
    
########################Static_time_space_model############# 
# like ISTM, sans mis a jour
    def trainFirst_Static_time_space_model(self):
        self.trainFirst_ISTM()
        print ("trainFirst_Static_time_space_model initialtion fini")
    
    def prosser_unParUn_jusquaFini_STATIC_Time_space_based_model(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
                if Y == -1 or Y == -2:
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE()

########################_Dynamique_but_stupide_time_space_model#############
# like ISTM, mais la mis à jour est " retraing with all old data "
    def trainFirst_Dymanic_TRA_opm_time_space_model(self):
        self.trainFirst_ISTM()
        print ("trainFirst_Dymanic_TRA_opm_time_space_model initialtion fini")
    
    def prosser_unParUn_jusquaFini_Dymanic_TRA_opm_Time_space_based_model(self):
        tempTRL = Chrono()   
        for timePoint in range(self.indexFinInitiation,self.indexFini_MIS_a_jour):
            sys.stdout.write("\r %s of %s " % (timePoint,self.indexFini_MIS_a_jour))
            sys.stdout.flush()
            for indexSensor in range(0,len(self.E_original)):
                X,Y = self.getFeatureOfISTM(indexSensor,timePoint)
                if Y == -1 or Y == -2:
                    self.self.model[indexSensor].misAjour_TRA_recalcule_matric_intermediaire()
                    self.E_resultat[indexSensor][timePoint] = self.model[indexSensor].predictionUn(X)
                else:
                    self.model[indexSensor].misAJour_TRA_Just_add_XY(X,Y) #仅添加新数据
                    
        self.comsomationTime = tempTRL.fini()
        return self.comsomationTime,self.calcule_and_getSSE()



#
#E_original =np.load("E_origninal.npy").tolist()
#E_aReparer =np.load("simulationDonnees/E5_tauxTrain90.npy").tolist()
#tauxTrain = 0.9
#trou = 5
#backTime = 6
#
#
#oneProcessTest = ProcessIRL(E_original,E_aReparer,tauxTrain,trou,backTime,testModel='yes')
#oneProcessTest.trainFirst_ISTM()
#print oneProcessTest.prosser_unParUn_jusquaFini_ISTM()