Trained three different models nonstop

graphs.py

+import matplotlib.pyplot as plt
+import numpy as np
+from util import read_list
+
+def plot(loss, label=None):
+	x = np.arange(0, len(loss))
+	plt.plot(loss, label=label)
+	plt.xlabel('Epochs')
+	plt.ylabel('Loss')
+
+def plot_best_model():
+	loss = [2.6129, 2.4543, 2.3803, 2.3188, 2.2835, 2.2500, 2.2371, 2.2178, 2.2071, 2.1918, 2.1795, 2.1839, 2.1855, 2.1756, 2.1869, 2.0661, 2.08390, 2.0756, 2.0439, 2.0376, 2.0501, 2.0166, 1.9766, 1.9441, 1.9409, 1.9263 , 1.9020, 1.8844, 1.8782, 1.8675, 1.8649, 1.8323, 1.8085, 1.7909, 1.7810, 1.7713, 1.7688, 1.7230, 1.7030, 1.6947, 1.6947, 1.6292, 1.5968, 1.5830]
+	plot(loss, 'Best model')
+
+def plot_lstm_50_model():
+	loss = [2.6357, 2.5246, 2.4718, 2.4106, 2.3716, 2.3487, 2.3232, 2.3089, 2.2949, 2.2901, 2.2137, 2.2068, 2.2021, 2.1978, 2.1933, 2.2126, 2.2105, 2.2200]
+	plot(loss)
+
+def plot_rnn_val_loss():
+	loss = [2.5573, 2.4642, 2.4089, 2.3709, 2.3442, 2.3265, 2.3136, 2.3000, 2.2965, 2.2883, 2.2848, 2.2210, 2.1682, 2.1604, 2.1587, 2.1231, 2.0898, 2.0942, 2.1014, 2.1023, 2.1073, 2.1093, 2.1107, 2.1167, 2.1199, 2.1216, 2.1240]
+	plot(loss, 'Validation')
+
+def plot_rnn_train_loss():
+	loss = [2.7632, 2.5124, 2.4267, 2.3725, 2.3330, 2.3032, 2.2820, 2.2667, 2.2471, 2.2353, 2.2209, 2.1950, 2.1639, 2.1431, 2.1302, 2.1260, 2.1147, 2.1086, 2.1036, 2.0993, 2.0946, 2.0899, 2.0858, 2.0806, 2.0766, 2.0717, 2.0675]
+	plot(loss, 'Training')
+
+
+def plot_lstm_nonstop():
+	hist100 = read_list('lstm_30_epochs_nonstop_dropout/history_69')
+	hist50 = read_list('lstm_70_epochs_50_units_nonstop_dropout/history_69')
+	hist_RNN = read_list('rnn_50_epochs_nonstop_dropout/history_69')
+
+	plot(hist100['val_loss'], 'Validation LSTM 100')
+	plot(hist100['loss'], 'Training LSTM 100')
+	plot(hist50['val_loss'], 'Validation LSTM 50')
+	plot(hist50['loss'], 'Training LSTM 50')
+	plot(hist_RNN['val_loss'], 'Validation RNN 100')
+	plot(hist_RNN['loss'], 'Training RNN 100')
+
+# plot_best_model()
+# # plot_lstm_50_model()
+# plot_rnn_val_loss()
+# plot_rnn_train_loss()
+
+plot_lstm_nonstop()
+plt.legend()
+plt.show()
\ No newline at end of file

main.py

 from sklearn.model_selection import train_test_split
-from keras.models import Sequential, load_model
-from keras.layers import Dense, LSTM, Embedding
-from util import read_list, read_array
+from keras.models import Sequential, load_model, clone_model
+from keras.layers import Dense, LSTM, Embedding, SimpleRNN
+from util import read_list, read_array, save_list
 from keras.utils import to_categorical
 import numpy as np
 
 VOCAB_SIZE = 85
 
-def get_train_test_val():
-	sequences = read_array('sequences_splitted.csv')
+def get_train_test_val(sequences):
 	X, y = sequences[:,:-1], sequences[:,-1]
 	y = to_categorical(y, num_classes=VOCAB_SIZE)
 	x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
 	x_test, x_validate, y_test, y_validate = train_test_split(x_test, y_test, test_size=0.2)
 	return x_train, y_train, x_test, y_test, x_validate, y_validate
 
-
-def run_n_epochs(n, model, x_train, y_train, x_validate, y_validate, x_test, y_test):
-	for i in range(n):
-		print('%i iteration out of %i' % (i, n))
-		model.fit(x_train, y_train, batch_size=128, epochs=1, validation_data=(x_validate, y_validate))
-		print('Saving the model...')
-		name = 'lstm_' + str(i) + '.h5'
-		model.save(name)
-
-def train():
-	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val()
-	seq_length = x_train.shape[1]
-
+def create_RNN_model(seq_length, hidden_units=100):
 	model = Sequential()
 	model.add(Embedding(VOCAB_SIZE, 50, input_length=seq_length))
-	model.add(LSTM(50))
-	model.add(Dense(VOCAB_SIZE, activation='sigmoid'))
+	model.add(SimpleRNN(hidden_units, return_sequences=True))
+	model.add(SimpleRNN(hidden_units))
+	model.add(Dense(hidden_units, activation='relu'))
+	model.add(Dense(VOCAB_SIZE, activation='softmax'))
 	print(model.summary())
 
 	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
-	run_n_epochs(100, model, x_train, y_train, x_validate, y_validate, x_test, y_test)
-
+	return model
 
-def train_saved_model():
-	model = load_model('lstm_test_validation_40_epochs.h5')
-	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val()
-	model.fit(x_train, y_train, epochs=1, batch_size=128, validation_data=(x_validate, y_validate))
-	print('Saving the model...')
-	model.save('lstm_test_validation_41_epochs.h5')
-	print(model.evaluate(x_test, y_test))
+def create_LSTM_2_layers(seq_length, hidden_units=100):
+	model = Sequential()
+	model.add(Embedding(VOCAB_SIZE, 50, input_length=seq_length))
+	model.add(LSTM(hidden_units, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))
+	model.add(LSTM(hidden_units, dropout=0.3, recurrent_dropout=0.3))
+	model.add(Dense(hidden_units, activation='relu'))
+	model.add(Dense(VOCAB_SIZE, activation='softmax'))
+	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
+	return model
 
-def model_name(epoch):
+def last_word_lstm_model_name(epoch, hidden_units=None):
 	return 'lstm_test_validation_' + str(epoch) + '_epochs.h5'
 
-def train_the_last_word_model():
-	previous_loss = 1.5555
-	epoch = 43
-	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val()
-	while(True):
-		model = load_model(model_name(epoch))
-		print(model.summary())
-		train_history = model.fit(x_train, y_train, epochs=1, batch_size=2048, validation_data=(x_validate, y_validate))
+def last_word_rnn_model_name(epoch, units):
+	return 'rnn_' + str(units) + '_units_' + str(epoch) + '.h5'
+
+def last_word_lstm_2_layers_model_name(epoch):
+	return 'lstm_2_layers_' + str(epoch) + '.h5'
+
+def last_word_lstm150_2_layers_model_name(epoch):
+	return 'lstm150_2_layers_' + str(epoch) + '.h5'
+
+def last_word_lstm50_2_layers_model_name(epoch):
+	return 'lstm50_2_layers_' + str(epoch) + '.h5'
+
+def last_word_lstm_n_2_layers_model_name(epoch, units):
+	return 'lstm_' + str(units) + '_2_layers_' + str(epoch) + '.h5'
+
+def train_model_with_decreasing_val_loss(model, x_train, y_train, x_test, y_test, x_validate, y_validate, model_name, epoch, best_loss, hidden_units):
+	print('Epoch %i' % epoch)
+	train_history = model.fit(x_train, y_train, epochs=1, batch_size=800, validation_data=(x_validate, y_validate))
 	val_loss = train_history.history['val_loss'][-1]
-		if val_loss < previous_loss:
+	print('____Improved the loss_____',)
 	print(model.evaluate(x_test, y_test))
-			print('Improved the loss')
 	epoch += 1
-			model.save(model_name(epoch))
-			previous_loss = val_loss
+	model.save(model_name(epoch, hidden_units))
+	best_loss = val_loss
+	return epoch, best_loss
+
+def train_last_word_lstm_model():
+	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val(read_array('sequences.csv'))
+	epoch = 49
+	val_loss = 1.5351
+	model = load_model(last_word_lstm_model_name(epoch))
+	while(True):
+		epoch, val_loss = train_model_with_decreasing_val_loss(model, x_train, y_train, x_test, y_test, x_validate, y_validate, last_word_lstm_model_name, epoch, val_loss, None)
+		model = load_model(last_word_lstm_model_name(epoch))
+
+def train_last_word_rnn_model(epoch=None, hidden_units=100):
+	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val(read_array('sequences.csv'))
+	val_loss = 2.1231
+	losses = []
+	if epoch == None:
+		model = create_RNN_model(x_train.shape[1], hidden_units)
+		epoch = 0
 	else:
-			print('No loss improvements')
+		model = load_model(last_word_rnn_model_name(epoch, hidden_units))
+	while(True):
+		epoch, val_loss = train_model_with_decreasing_val_loss(model, x_train, y_train, x_test, y_test, x_validate, y_validate, last_word_rnn_model_name, epoch, val_loss, hidden_units)
+		losses.append(val_loss)
+		print(losses)
+		model = load_model(last_word_rnn_model_name(epoch, hidden_units))
+
+def print_rnn_model_val_loss():
+	model = load_model(last_word_lstm_n_2_layers_model_name(11, 100))
+
+
+def train_lstm_nonstop():
+	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val(read_array('sequences.csv'))
+	# model = create_LSTM_2_layers(x_train.shape[1], 100)
+	model = load_model('lstm_30_epochs_nonstop_dropout/model_22.h5')
+	epochs = 70
+	history = read_list('lstm_30_epochs_nonstop_dropout/history_22')
+	for epoch in range(23, epochs):
+		print('Epoch', epoch)
+		train_history = model.fit(x_train, y_train, epochs=1, batch_size=32, validation_data=(x_validate, y_validate))
+		model.save('lstm_30_epochs_nonstop_dropout/model_' + str(epoch) + '.h5')
+
+		for key in train_history.history.keys():
+			history[key].append(train_history.history[key][0])
+			save_list(history, 'lstm_30_epochs_nonstop_dropout/history_' + str(epoch))
+
+def train_lstm_50_units_nonstop():
+	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val(read_array('sequences.csv'))
+	model = create_LSTM_2_layers(x_train.shape[1], 50)
+	epochs = 70
+	directory = 'lstm_70_epochs_50_units_nonstop_dropout/'
+	history = {'loss': [], 'acc': [], 'val_loss': [], 'val_acc': []}
+	for epoch in range(epochs):
+		print('Epoch', epoch)
+		train_history = model.fit(x_train, y_train, epochs=1, batch_size=32, validation_data=(x_validate, y_validate))
+		model.save(directory + 'model_' + str(epoch) + '.h5')
+
+		for key in train_history.history.keys():
+			history[key].append(train_history.history[key][0])
+			save_list(history, directory + 'history_' + str(epoch))
+
+def train_rnn_nonstop():
+	x_train, y_train, x_test, y_test, x_validate, y_validate = get_train_test_val(read_array('sequences.csv'))
+	model = create_RNN_model(x_train.shape[1], 100)
+	directory = 'rnn_50_epochs_nonstop_dropout/'
+	epochs = 70
+	history = {'loss': [], 'acc': [], 'val_loss': [], 'val_acc': []}
+	for epoch in range(epochs):
+		print('Epoch', epoch)
+		train_history = model.fit(x_train, y_train, epochs=1, batch_size=32, validation_data=(x_validate, y_validate))
+		model.save(directory + 'model_' + str(epoch) + '.h5')
+
+		for key in train_history.history.keys():
+			history[key].append(train_history.history[key][0])
+			save_list(history, directory + 'history_' + str(epoch))
+
+def print_lstm_30_history():
+	hist = read_list('lstm_30_epochs_nonstop_dropout/history_22')
+	loss_missed = [2.1326, 2.1257, 2.1175, 2.1137, 2.1104, 2.1042]
+	acc_missed = [0.4142, 0.4156, 0.4170, 0.4169, 0.4180, 0.4197]
+	val_loss_missed = [2.0993, 2.1017, 2.1083, 2.1216, 2.1224, 2.1213]
+	val_acc_missed = [0.4226, 0.4172, 0.4221, 0.4234, 0.4230, 0.4258]
+
+	hist['loss'] = hist['loss'][:17] + loss_missed + hist['loss'][17:]
+	hist['val_loss'] = hist['val_loss'][:17] + val_loss_missed + hist['val_loss'][17:]
+	hist['acc'] = hist['acc'][:17] + acc_missed + hist['acc'][17:]
+	hist['val_acc'] = hist['val_acc'][:17] + val_acc_missed + hist['val_acc'][17:]
+	save_list(hist, 'lstm_30_epochs_nonstop_dropout/history_22')
+	print(read_list('lstm_30_epochs_nonstop_dropout/history_22'))
 
-train()
\ No newline at end of file
+train_lstm_50_units_nonstop()

preprocessing.py

@@ -134,6 +134,6 @@ plt.hist(lengths, bins=100)
 plt.show()
 
 
-sequences = pad_sequences(sequences, 40)
+sequences = pad_sequences(sequences, 30)
 sequences = np.array(sequences)
 save_array(sequences, 'sequences_splitted.csv')
\ No newline at end of file