Commit a0bbc28d authored by 蒙律师's avatar 蒙律师

1st version code

parent b0f23f3c
from Model import CustomizedLSTM
import pandas as pd
import numpy as np
import torch
# from import_files import CustomDataset
# from RNN import NaiveCustomRNN
from tqdm import tqdm
from argparse import ArgumentParser
import torch.nn.utils.rnn as rnn_utils
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
if __name__ == "__main__":
coef = 10
Data = pd.read_csv('result_B=100.csv', header=None).to_numpy()
X = (Data[:, :-2].T)[np.newaxis, :, :]
x_size = X.shape[1]
event = Data[:, -1][np.newaxis, :]
time = Data[:, -2][np.newaxis, :]
hidden_size = 5
event_class = len(set(event.flatten()))
emb_dim = 5
model = CustomizedLSTM(x_size, hidden_size, event_class, emb_dim)
model.set_optimizer(lr=1e-3, weight_decay=1e-3)
train_X = X[:, :, :-1]
train_event = event[:, :-1]
train_time = time[:, :-1]
epoch = 0
train_loss_list = []
test_loss_list = []
while epoch < 30:
train_loss = model.train_batch_all([train_X, train_event, train_time])
# train_loss = model.train_batch([train_X, train_event, train_time])
test_loss, log_f_t, predicted_time_list, event_acc = model.predict_batch([X, event, time])
epoch += 1
train_loss_c, test_loss_c = train_loss.clone().detach().numpy(), test_loss.clone().detach().numpy()
# print(train_loss_c, test_loss_c)
train_loss_list.append(train_loss_c)
test_loss_list.append(test_loss_c)
import matplotlib.pyplot as plt
plt.figure()
plt.plot(np.arange(len(train_loss_list)), np.log(train_loss_list), np.log(test_loss_list))
plt.show()
print(predicted_time_list)
print(event_acc)
# predicted_time_list = np.array(predicted_time_list)
# np.savetxt('noting really matters.txt',predicted_time_list)
import torch
import torchvision
from torch import optim, nn
import math
import numpy as np
import pandas as pd
class CustomizedLSTM(nn.Module):
def __init__(self, x_size, hidden_size, event_class, emb_dim):
'''
:param hidden_size: the dimension of h
:param event_class: the number of event classes
:param emb_dim: the dimension of embedding event
'''
super(CustomizedLSTM, self).__init__()
self.hidden_size = hidden_size
self.embedding = nn.Embedding(num_embeddings=event_class, embedding_dim=emb_dim)
# Parameter to define forget gate
self.Wf = nn.Parameter(torch.Tensor(x_size + hidden_size + 1, hidden_size))
# self.bf = nn.Parameter(torch.Tensor(hidden_size))
# Parameter to define input gate
self.Wi = nn.Parameter(torch.Tensor(x_size + hidden_size + 1, hidden_size))
# Parameter to define C helper \tilde{c}:
self.Wc = nn.Parameter(torch.Tensor(x_size + hidden_size + 1, hidden_size))
# Parameter to define Time Gate
self.Wtx = nn.Parameter(torch.Tensor(x_size + 1, hidden_size))
self.Wt = nn.Parameter(torch.Tensor(1 + 1, hidden_size))
# self.bt = nn.Parameter(torch.Tensor(hidden_size))
# Parameter to define Event Gate
self.Wex = nn.Parameter(torch.Tensor(x_size + 1, hidden_size))
self.We = nn.Parameter(torch.Tensor(emb_dim + 1, hidden_size))
# self.bt = nn.Parameter(torch.Tensor(hidden_size))
# Parameter to define Delta_Feature Gate
self.Wdx = nn.Parameter(torch.Tensor(x_size + 1, hidden_size))
self.Wd = nn.Parameter(torch.Tensor(x_size + 1, hidden_size))
# Parameter to define output Gate
self.Wo = nn.Parameter(torch.Tensor(2 * x_size + hidden_size + 1 + emb_dim + 1, hidden_size))
# Parameter to define \lambda
self.v_t = nn.Parameter(torch.Tensor(2 * hidden_size, 1))
self.w_t = nn.Parameter(torch.tensor(0.1))
# network architecture to derive event
self.event_linear = torch.nn.Linear(in_features=hidden_size * 2, out_features=event_class, bias=True)
#
self.softplus = torch.nn.Softplus()
self.reset_Parameter()
def forward(self, X, Event, D_time, D_x, init_h=None, init_c=None):
'''
:param X: [batch_size, feature_size, seq_len]
:param Event: [batch_size, seq_len]
:param D_time: [batch_size, seq_len]
:param D_x: [batch_size, feature_size, seq_len]
:param init_h: [batch_size, hidden_size]
:param init_c: [batch_size, hidden_size]
:return:
'''
batch_size, sequence_len = Event.shape
h_seq = torch.Tensor([])
c_seq = torch.Tensor([])
if init_h is None and init_c is None:
h_last = torch.zeros(batch_size, self.hidden_size) # .to(x.device),
c_last = torch.zeros(batch_size, self.hidden_size) # .to(x.device),
else:
h_last = init_h
c_last = init_c
AllOnesTensor = torch.ones(batch_size, 1)
Embed_E = torch.Tensor([])
for event in Event:
# event is the event list for one entity.
e_lst = torch.Tensor([])
for single_event in event:
if single_event != 0:
embedding_event = self.embedding(single_event.to(torch.int64))
else:
embedding_event = torch.zeros(self.embedding.embedding_dim)
e_lst = torch.cat((e_lst, embedding_event.unsqueeze(1)), dim=1)
Embed_E = torch.cat((Embed_E, e_lst.unsqueeze(0)), dim=0)
for t in range(sequence_len):
e = Embed_E[:, :, t]
x_t = X[:, :, t]
d_time = D_time[:, t].reshape(-1, 1)
d_x = D_x[:, :, t]
combine = torch.cat((x_t, h_last, AllOnesTensor), dim=1).float()
x_t_bias = torch.cat((x_t, AllOnesTensor), dim=1).float()
d_time_bias = torch.cat((d_time, AllOnesTensor), dim=1).float()
d_x_bias = torch.cat((d_x, AllOnesTensor), dim=1).float()
Embed_E_bias = torch.cat((e, AllOnesTensor), dim=1).float()
# Forget gate:
f = torch.sigmoid(torch.mm(combine, self.Wf))
# Input gate:
i = torch.sigmoid(torch.mm(combine, self.Wi))
# Time gate:
T = torch.sigmoid(torch.mm(x_t_bias, self.Wtx) + torch.sigmoid(torch.mm(d_time_bias, self.Wt)))
# Event Gate:
E = torch.sigmoid(torch.mm(x_t_bias, self.Wex) + torch.sigmoid(torch.mm(Embed_E_bias, self.We)))
# Delta Feature Gate:
D = torch.sigmoid(torch.mm(x_t_bias, self.Wdx) + torch.sigmoid(torch.mm(d_x_bias, self.Wd)))
# C helper:
c_helper = torch.tanh(torch.mm(combine, self.Wc))
# Output Gate:
allvariable = torch.cat((x_t, h_last, d_time, e, d_x, AllOnesTensor), dim=1).float()
o = torch.sigmoid(torch.mm(allvariable, self.Wo))
# C short term memory:
C_s = f * D * c_last + i * c_helper
# C long term meomry:
C = f * c_last + i * T * E * c_helper
# Output h:
h = o * torch.tanh(C_s)
h_seq = torch.cat((h_seq, h.unsqueeze(0)), 1)
c_seq = torch.cat((c_seq, C.unsqueeze(0)), 1)
return h_seq, c_seq
def set_optimizer(self, lr=1e-3, weight_decay=1e-5):
# from torch.optim import Adam
# self.optimizer = Adam(self.parameters(), lr=lr, weight_decay=weight_decay)
from torch.optim import SGD
self.optimizer = SGD(self.parameters(), lr=lr, weight_decay=weight_decay) # , momentum=0.9)
'''
if use_bert:
self.optimizer = BertAdam(params=self.parameters(),
lr=self.config.lr,
warmup=0.1,
t_total=total_step)
else:
'''
def reset_Parameter(self):
stdv = 1.0 / math.sqrt(self.hidden_size)
for weight in self.parameters():
weight.data.uniform_(-stdv, stdv)
def init_Parameter(self):
for weight in self.parameters():
torch.nn.init.normal_(weight, mean=0, std=1)
def softplus(self, x):
return torch.log(1 + torch.exp(x))
def my_loss(self, target_event, t, t_j, h_j, c_j):
'''
The loss is log(P(y_t|h_{tj})) + log(f(t|h_{tj})), where:
P(y_t|h_{tj}): the predicted probability that entity will experience ground truth event y_t at t, given history till t_last
f(t|h_{t_last}): the predicted condition intensity function that entity will experience an unspecific event at t, given hisotry till t_last
:param t: ground truth time when next event occur
:param y_t: ground truth event type for next event
:param t_j: the last time entity experience an event
:param h_j: the history till t_j
:return: Loss
'''
combine = torch.cat((h_j, c_j), dim=1)
event_pred = torch.softmax(self.event_linear(combine), dim=1)
event_loss = torch.zeros_like(target_event).float()
for idx, event_id in enumerate(target_event):
event_loss[idx] = event_pred[idx, int(event_id.item())]
event_loss = (-1) * torch.log(event_loss)
self.w_t = nn.Parameter(self.softplus(self.w_t))
S = torch.mm(combine, self.v_t)
log_f_t = S + self.w_t * (t - t_j) + (1 / self.w_t) * (torch.exp(S) - torch.exp(S + self.w_t * (t - t_j)))
time_loss = (-1) * log_f_t
loss = torch.mean(time_loss) + torch.mean(event_loss)
return loss, log_f_t
def train_batch(self, batch):
'''
:param batch: includes X, event and time.
X: [batch_size, feature_size, seq_len]
event: [batch_size, seq_len]
time: [batch_size, seq_len]
:return: train loss
'''
X, event, time = batch
X, event, time = torch.tensor(X), torch.tensor(event), torch.tensor(time)
batch_size, feature_size, seq_len = X.shape
D_time = time - torch.cat((torch.zeros(batch_size, 1), time[:, 1:]), dim=1)
D_time[:, 0] = 0 # Time interval is 0 at first event.
D_X = X - torch.cat((torch.zeros(batch_size, feature_size, 1), X[:, :, 1:]), dim=2)
D_X[:, :, 0] = 0 # Feature difference is 0 at first event
target_event, target_t, t_j = event[:, -1], time[:, -1], time[:, -2]
h_seq, c_seq = self.forward(X, event, D_time, D_X)
h_j, c_j = h_seq[:, -2, :], c_seq[:, -2, :]
loss, log_f_t = self.my_loss(target_event, target_t, t_j, h_j, c_j)
loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
return loss
def train_batch_all(self, batch):
'''
:param batch: includes X, event and time.
X: [batch_size, feature_size, seq_len]
event: [batch_size, seq_len]
time: [batch_size, seq_len]
:return: train loss
'''
X, event, time = batch
X, event, time = torch.tensor(X), torch.tensor(event), torch.tensor(time)
batch_size, feature_size, seq_len = X.shape
D_time = time - torch.cat((torch.zeros(batch_size, 1), time[:, 1:]), dim=1)
D_time[:, 0] = 0 # Time interval is 0 at first event.
D_X = X - torch.cat((torch.zeros(batch_size, feature_size, 1), X[:, :, 1:]), dim=2)
D_X[:, :, 0] = 0 # Feature difference is 0 at first event
h_seq, c_seq = self.forward(X, event, D_time, D_X)
LOSS = None
for t_idx in range(seq_len - 1):
current_t, current_h, current_c = time[:, t_idx], h_seq[:, t_idx, :], c_seq[:, t_idx, :]
target_event, target_t = event[:, t_idx + 1], time[:, t_idx + 1]
loss, log_f_t = self.my_loss(target_event, target_t, current_t, current_h, current_c)
# print('what is the log_f_t looks like?', log_f_t)
# print('what is the loss looks like?', loss)
if LOSS is None:
LOSS = loss
else:
LOSS += loss
LOSS /= seq_len
LOSS.backward()
self.optimizer.step()
self.optimizer.zero_grad()
return LOSS
def calculate_ft(self, t, t_j, h_j, c_j):
v_t = self.v_t.detach().numpy()
w_t = self.softplus(self.w_t).detach().item()
combine = np.concatenate((h_j, c_j), axis=0)[np.newaxis, :]
S = np.dot(combine, v_t)
log_f_t = S + w_t * (t - t_j) + 1 / w_t * (np.exp(S) - np.exp(S + w_t * (t - t_j)))
f_t = np.exp(log_f_t)
return f_t
def f_t(self, t, t_j, h_j, c_j):
'''
This function is used to derive the definite integral: equation (13).
Firstly we should generated f^{*}(t) in equation (12),
Then return t*f^{*}(t) to scipy.integrate.quad to get the result.
:param t: t in equation (12)
:param t_j: t_j in equation (12)
:param h_j: h_j in equation (12)
:return: t * f^{*}(t) , simplictly, t * f_t
'''
v_t = self.v_t.clone().detach().numpy()
w_t = self.softplus(self.w_t.clone()).detach().item()
combine = np.concatenate((h_j, c_j), axis=0)[np.newaxis, :]
S = np.dot(combine, v_t)
log_f_t = S + w_t * (t - t_j) + 1 / w_t * (np.exp(S) - np.exp(S + w_t * (t - t_j)))
f_t = np.exp(log_f_t)
return (t * f_t)
def predict_batch(self, batch):
'''
:param batch: includes X, event and time.
X: [batch_size, feature_size, seq_len]
event: [batch_size, seq_len]
time: [batch_size, seq_len]
:return: train loss
'''
X, event, time = batch
X = torch.tensor(X)
event = torch.tensor(event)
time = torch.tensor(time)
batch_size, feature_size, seq_len = X.shape
D_time = time - torch.cat((torch.zeros(batch_size, 1), time[:, 1:]), dim=1)
D_time[:, 0] = 0 # Time interval is 0 at first event.
D_X = X - torch.cat((torch.zeros(batch_size, feature_size, 1), X[:, :, 1:]), dim=2)
D_X[:, :, 0] = 0 # Feature difference is 0 at first event
h_seq, c_seq = self.forward(X, event, D_time, D_X)
LOSS = 0
predicted_time_list = []
for t_idx in range(seq_len - 1):
current_t, current_h, current_c = time[:, t_idx], h_seq[:, t_idx, :], c_seq[:, t_idx, :]
target_event, target_t = event[:, t_idx + 1], time[:, t_idx + 1]
event_pred = torch.argmax(torch.softmax(self.event_linear(torch.cat((current_h, current_c), dim=1)), dim=1),
dim=1)
event_acc = torch.sum((event_pred - target_event) == 0).float() / batch_size
pred_time = []
from scipy import integrate
for idx, t_j in enumerate(current_t):
t_j = t_j.clone().detach().numpy()
h_j = current_h[idx].clone().detach().numpy()
c_j = current_c[idx].clone().detach().numpy()
predicted_time, err = integrate.quad(self.f_t, t_j, np.inf,
args=(t_j, h_j, c_j)) # equation (13), do the integration
print('the current time is, {}, the target time is, {}, the predicted time is {}'
.format(t_j, target_t[idx], predicted_time))
pred_time.append(predicted_time)
loss, log_f_t = self.my_loss(target_event, target_t, current_t, current_h, current_c)
LOSS += loss
predicted_time_list.append(pred_time)
LOSS /= seq_len
return LOSS, log_f_t, predicted_time_list, event_acc
import numpy as np
import pandas as pd
def Simulating_X(Dim, max_T, scale):
# initialization
t = 0
X = [] # Feature List
X_time = [] # Feature Time List
x_0 = np.random.multivariate_normal(np.zeros(Dim), np.ones((Dim, Dim)))
X.append(x_0)
X_time.append(t)
while True:
w = np.random.exponential(scale=scale)
t = t + w
if t <= max_T:
x = np.random.multivariate_normal(np.zeros(Dim), np.ones((Dim, Dim)))
X.append(x)
X_time.append(t)
else:
return np.array(X), np.array(X_time)
def Generate_lambda(base, s, X_history, Event_history, Event_time, y, alpha_x, A, B):
result_lambda = base
temp_Xhistory = X_history.copy()
temp_Xhistory[:, 1:] = temp_Xhistory[:, :-1]
result_lambda += alpha_x * np.sqrt(
np.sum((X_history - temp_Xhistory) ** 2)) # Historical Feature Changing's impact on current \lambda
event_caused_lambda = 0
for event_No, e in enumerate(Event_history): # Historical Event's impact on current \lambda
t_j = Event_time[event_No]
event_caused_lambda += A[e - 1, y - 1] * np.exp(B[e - 1, y - 1] * (-1) * (s - t_j))
result_lambda += event_caused_lambda
return result_lambda
def Thining(base, max_T, Event_class, X_history, X_time, alpha_x, A, B):
s = 0 # Current Time
Event_list = [] # Event list
T_e = [] # Event time list
while s < max_T:
T_candidate = []
[latest_x_t_id] = np.argwhere(X_time <= s)[-1]
sub_Xhistory = X_history[:latest_x_t_id + 1]
for y in np.arange(Event_class) + 1:
lambda_y_s = Generate_lambda(base, s, sub_Xhistory, Event_list, T_e, y, alpha_x, A, B)
u = np.random.uniform(low=0, high=1)
w = - np.log(u) / lambda_y_s
s_temp = s + w
D = np.random.uniform(low=0, high=1)
if D * lambda_y_s <= Generate_lambda(base, s_temp, sub_Xhistory, Event_list, T_e, y, alpha_x, A, B):
T_candidate.append(s_temp)
if len(T_candidate) == 0:
s = s_temp
continue
t_n = np.min(T_candidate)
[[y_star]] = np.argwhere(T_candidate == t_n) + 1
# print('hey this y_star: {}'.format(y_star))
flag = 1
for t_i in X_time:
if t_i > s and t_i < t_n:
s = t_i
flag = 0
break
if flag:
Event_list.append(y_star)
T_e.append(t_n)
s = t_n
if T_e[-1] > max_T:
return Event_list[:-1], T_e[:-1]
else:
return Event_list, T_e
def Merge(X, X_time, Event_list, T_e):
'''
This function is applied to form a table, every row is like: [X Time Event_type].
:param X:
:param X_time:
:param Event_list:
:param T_e:
:return:
'''
result = np.concatenate((X, X_time[:, np.newaxis], np.zeros((X.shape[0], 1))), axis=1)
for t_id, t in enumerate(T_e):
mask = X_time <= t
index = int(len(X_time[mask]))
X_candidate = X[index - 1]
print()
X_candidate = np.append(X_candidate, [t, Event_list[t_id]])
result = np.insert(arr=result, obj=index + t_id, values=X_candidate, axis=0)
return result
if __name__ == '__main__':
X_dim = 5 # The x dimension
max_T = 50 # maximum observation time
inverse_lambda = 5 # How long will a new x be generated?
Event_class = 3 # how man types of Event in our scenario?
coefficient = 10 # the coefficient to define random seed and matrix B
alpha_x = 0.1 # alpha_x in zhidong's lambda definition
A = np.eye(Event_class) # A matrix in zhidong's lambda definition
B = np.eye(Event_class) * coefficient # B matrix in zhidong's lambda definition,
np.random.seed(coefficient)
X, X_time = Simulating_X(Dim=X_dim, max_T=max_T,
scale=inverse_lambda) # Generate feature, the time to record feature
Event_list, T_e = Thining(base=0.1, max_T=max_T, Event_class=Event_class, X_history=X, X_time=X_time,
alpha_x=alpha_x, A=A,
B=B) # Generate Event, the time that event occurs
# This step can be confused: to put it simpltly, we have generate 2 seq: X, and Event_list with their time respectivelhy, Now we want to merge them both
X, X_time, Event_list, T_e = np.array(X), np.array(X_time), np.array(Event_list), np.array(T_e)
result = Merge(X, X_time, Event_list, T_e)
result = pd.DataFrame(result)
fn = 'result' + '_B=' + str(coefficient) + '.csv' # we define our file name with the random seed! Very old school.
result.to_csv(fn, index=False, header=False)
'''
X = pd.DataFrame(X)
X_time = pd.DataFrame(X_time)
Event_list = pd.DataFrame(Event_list)
T_e = pd.DataFrame(T_e)
X.to_csv('X.csv', index=False, header=False)
X_time.to_csv('X_time.csv', index=False, header=False)
Event_list.to_csv('Event_list.csv', index=False, header=False)
T_e.to_csv('Event_Time.csv', index=False, header=False)
'''
'''
TAU, LAMBDA = Thining(miu, alpha, beta, T)
TAU = np.array(TAU)
print(TAU)
len = TAU.__len__()
lambda_list = []
for tau_id in np.arange(len - 1):