Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

T decao2.0 #4

Open
wants to merge 30 commits into
base: master
Choose a base branch
from
Open

T decao2.0 #4

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
ab9c94d
'fix_math_utils'
t-decao Jun 19, 2020
9fd2a3c
'fix_z_score'
t-decao Jun 19, 2020
7858efc
fix z_score
t-decao Jun 22, 2020
1389b64
add trainer
t-decao Jun 22, 2020
01c316f
add tester
t-decao Jun 22, 2020
e4737c2
fix commit
t-decao Jun 22, 2020
b0b3f05
fix commit_1 on math utils
t-decao Jun 22, 2020
554f584
normalize train data
t-decao Jun 22, 2020
84ac591
only normalize train seq
t-decao Jun 22, 2020
2c894a7
fix math utils
t-decao Jun 22, 2020
e663678
Update math_utils.py
t-decao Jun 22, 2020
13f7a3a
Update data_utils.py
t-decao Jun 22, 2020
7b09bd6
Update math_utils.py
t-decao Jun 22, 2020
906857f
Update tester.py
t-decao Jun 22, 2020
13e5a91
Update data_utils.py
t-decao Jun 22, 2020
6bbf840
Update tester.py
t-decao Jun 22, 2020
e5ca2a9
Update tester.py
t-decao Jun 22, 2020
027bb83
Update data_utils.py
t-decao Jun 22, 2020
4ed0ea2
Update tester.py
t-decao Jun 23, 2020
d615b65
Update math_utils.py
t-decao Jun 23, 2020
9d1b60c
fix layer
t-decao Jul 1, 2020
cb9ec08
Update data_utils.py
t-decao Jul 11, 2020
89eeef1
Update trainer.py
t-decao Jul 11, 2020
00bf3b2
Update tester.py
t-decao Jul 11, 2020
e72b285
Update layers.py
t-decao Jul 11, 2020
8e8eef1
Update data_utils.py
t-decao Jul 11, 2020
cd32127
Update main.py
t-decao Jul 13, 2020
4be74a2
Update trainer.py
t-decao Jul 13, 2020
8893591
Update trainer.py
t-decao Jul 13, 2020
41ae829
Update math_utils.py
t-decao Jul 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 47 additions & 18 deletions data_loader/data_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,21 @@
class Dataset(object):
def __init__(self, data, stats):
self.__data = data
self.mean = stats['mean']
self.std = stats['std']
self.stats = stats
# self.mean = stats['mean']
# self.std = stats['std']

def get_data(self, type):
return self.__data[type]

def get_stats(self):
return {'mean': self.mean, 'std': self.std}
return self.stats #{'mean': self.mean, 'std': self.std}

def get_len(self, type):
return len(self.__data[type])

def z_inverse(self, type):
return self.__data[type] * self.std + self.mean
# def z_inverse(self, type):
# return self.__data[type] * self.std + self.mean


def seq_gen(len_seq, data_seq, offset, n_frame, n_route, day_slot, C_0=1):
Expand Down Expand Up @@ -111,16 +112,16 @@ def data_gen(file_path, n_route, train_val_test_ratio, scalar, n_frame, day_slot

# data_seq = pd.read_csv(file_path, header=None) # .values
# for column in list(data_seq.columns):
# #print(column)
# mean_val = data_seq[column].mean()
# data_seq[column].replace(0, mean_val, inplace=True)
data_seq = data_seq.values

if scalar == 'min_max': # TODO: unify the covering range with zscore
my_matrix = np.array(data_seq)
scaler = MinMaxScaler()
scaler.fit(my_matrix)
data_seq = scaler.transform(my_matrix)
# # #print(column)
# mean_val = data_seq[column].mean()
# data_seq[column].replace(0, mean_val, inplace=True)
#data_seq = data_seq#.values

# if scalar == 'min_max': # TODO: unify the covering range with zscore
# my_matrix = np.array(data_seq)
# scaler = MinMaxScaler()
# scaler.fit(my_matrix)
# data_seq = scaler.transform(my_matrix)
# data_seq=data_seq[~(data_seq==0).all(axis=1), :]
print(data_seq.shape)
except FileNotFoundError:
Expand All @@ -133,12 +134,40 @@ def data_gen(file_path, n_route, train_val_test_ratio, scalar, n_frame, day_slot

seq_train = data_seq[:train_len]

if scalar == 'min_max': # TODO: unify the covering range with zscore
my_matrix = np.array(seq_train)
all_matrix = np.array(data_seq)
scaler = MinMaxScaler()
scaler.fit(my_matrix)
data_seq = scaler.transform(all_matrix)

x_stats = []
data_seq2 = pd.DataFrame(seq_train)

if scalar == 'z_score':
x_stats = {'mean': np.mean(seq_train), 'std': np.std(seq_train)} # TODO: fix the zscore

#x_stats = {'mean': np.mean(seq_train), 'std': np.std(seq_train)}

for column in list(data_seq2.columns):
#print(column)
stats = {}
data = np.array(data_seq2[column])
stats = {'mean': np.mean(data,axis=1), 'std': np.std(data,axis=1)}
x_stats.append(stats)
#x_stats = {'mean': np.mean(seq_train), 'std': np.std(seq_train)} # TODO: fix the zscore
else:
x_stats = {'mean': 0, 'std': 1}

data_seq = z_score(data_seq, x_stats['mean'], x_stats['std'])
for column in list(data_seq2.columns):
t-decao marked this conversation as resolved.
Show resolved Hide resolved
#print(column)
stats = {}
data = np.array(data_seq2[column])
stats = {'mean': 0, 'std': 1}
x_stats.append(stats)

#data_seq = data_seq.values


data_seq = z_score(data_seq.values, x_stats)

seq_train = seq_gen(train_len, data_seq, 0, n_frame, n_route, day_slot)
seq_val = seq_gen(val_len, data_seq, train_len, n_frame, n_route, day_slot)
Expand Down
Loading