Coverage for biobb_ml/neural_networks/autoencoder_neural_network.py: 89%

1#!/usr/bin/env python3 

2 

3"""Module containing the AutoencoderNeuralNetwork class and the command line interface.""" 

4import argparse 

5import h5py 

6import json 

7import numpy as np 

8import pandas as pd 

9from biobb_common.generic.biobb_object import BiobbObject 

10from tensorflow.python.keras.saving import hdf5_format 

11from tensorflow.keras.models import Model 

12from tensorflow.keras.layers import Input, LSTM, Dense, RepeatVector, TimeDistributed 

13from biobb_common.configuration import settings 

14from biobb_common.tools import file_utils as fu 

15from biobb_common.tools.file_utils import launchlogger 

16from biobb_ml.neural_networks.common import check_input_path, check_output_path 

17 

18 

19class AutoencoderNeuralNetwork(BiobbObject): 

20 """ 

21 | biobb_ml AutoencoderNeuralNetwork 

22 | Wrapper of the TensorFlow Keras LSTM method for encoding. 

23 | Fits and tests a given dataset and save the compiled model for an Autoencoder Neural Network. Visit the `LSTM documentation page <https://www.tensorflow.org/api_docs/python/tf/keras/layers/LSTM>`_ in the TensorFlow Keras official website for further information. 

24 

25 Args: 

26 input_decode_path (str): Path to the input decode dataset. File type: input. `Sample file <https://github.com/bioexcel/biobb_ml/raw/master/biobb_ml/test/data/neural_networks/dataset_autoencoder_decode.csv>`_. Accepted formats: csv (edam:format_3752). 

27 input_predict_path (str) (Optional): Path to the input predict dataset. File type: input. `Sample file <https://github.com/bioexcel/biobb_ml/raw/master/biobb_ml/test/data/neural_networks/dataset_autoencoder_predict.csv>`_. Accepted formats: csv (edam:format_3752). 

28 output_model_path (str): Path to the output model file. File type: output. `Sample file <https://github.com/bioexcel/biobb_ml/raw/master/biobb_ml/test/reference/neural_networks/ref_output_model_autoencoder.h5>`_. Accepted formats: h5 (edam:format_3590). 

29 output_test_decode_path (str) (Optional): Path to the test decode table file. File type: output. `Sample file <https://github.com/bioexcel/biobb_ml/raw/master/biobb_ml/test/reference/neural_networks/ref_output_test_decode_autoencoder.csv>`_. Accepted formats: csv (edam:format_3752). 

30 output_test_predict_path (str) (Optional): Path to the test predict table file. File type: output. `Sample file <https://github.com/bioexcel/biobb_ml/raw/master/biobb_ml/test/reference/neural_networks/ref_output_test_predict_autoencoder.csv>`_. Accepted formats: csv (edam:format_3752). 

31 properties (dic - Python dictionary object containing the tool parameters, not input/output files): 

32 * **optimizer** (*string*) - ("Adam") Name of optimizer instance. Values: Adadelta (Adadelta optimization is a stochastic gradient descent method that is based on adaptive learning rate per dimension to address two drawbacks: the continual decay of learning rates throughout training and the need for a manually selected global learning rate), Adagrad (Adagrad is an optimizer with parameter-specific learning rates; which are adapted relative to how frequently a parameter gets updated during training. The more updates a parameter receives; the smaller the updates), Adam (Adam optimization is a stochastic gradient descent method that is based on adaptive estimation of first-order and second-order moments), Adamax (It is a variant of Adam based on the infinity norm. Default parameters follow those provided in the paper. Adamax is sometimes superior to adam; specially in models with embeddings), Ftrl (Optimizer that implements the FTRL algorithm), Nadam (Much like Adam is essentially RMSprop with momentum; Nadam is Adam with Nesterov momentum), RMSprop (Optimizer that implements the RMSprop algorithm), SGD (Gradient descent -with momentum- optimizer). 

33 * **learning_rate** (*float*) - (0.02) [0~100|0.01] Determines the step size at each iteration while moving toward a minimum of a loss function 

34 * **batch_size** (*int*) - (100) [0~1000|1] Number of samples per gradient update. 

35 * **max_epochs** (*int*) - (100) [0~1000|1] Number of epochs to train the model. As the early stopping is enabled, this is a maximum. 

36 * **remove_tmp** (*bool*) - (True) [WF property] Remove temporal files. 

37 * **restart** (*bool*) - (False) [WF property] Do not execute if output files exist. 

38 

39 Examples: 

40 This is a use example of how to use the building block from Python:: 

41 

42 from biobb_ml.neural_networks.autoencoder_neural_network import autoencoder_neural_network 

43 prop = { 

44 'optimizer': 'Adam', 

45 'learning_rate': 0.01, 

46 'batch_size': 32, 

47 'max_epochs': 300 

48 } 

49 autoencoder_neural_network(input_decode_path='/path/to/myDecodeDataset.csv', 

50 output_model_path='/path/to/newModel.h5', 

51 input_predict_path='/path/to/myPredictDataset.csv', 

52 output_test_decode_path='/path/to/newDecodeDataset.csv', 

53 output_test_predict_path='/path/to/newPredictDataset.csv', 

54 properties=prop) 

55 

56 Info: 

57 * wrapped_software: 

58 * name: TensorFlow Keras LSTM 

59 * version: >2.1.0 

60 * license: MIT 

61 * ontology: 

62 * name: EDAM 

63 * schema: http://edamontology.org/EDAM.owl 

64 

65 """ 

66 

67 def __init__(self, input_decode_path, output_model_path, 

68 input_predict_path=None, output_test_decode_path=None, 

69 output_test_predict_path=None, properties=None, **kwargs) -> None: 

70 properties = properties or {} 

71 

72 # Call parent class constructor 

73 super().__init__(properties) 

74 self.locals_var_dict = locals().copy() 

75 

76 # Input/Output files 

77 self.io_dict = { 

78 "in": {"input_decode_path": input_decode_path, "input_predict_path": input_predict_path}, 

79 "out": {"output_model_path": output_model_path, "output_test_decode_path": output_test_decode_path, "output_test_predict_path": output_test_predict_path} 

80 } 

81 

82 # Properties specific for BB 

83 self.optimizer = properties.get('optimizer', 'Adam') 

84 self.learning_rate = properties.get('learning_rate', 0.02) 

85 self.batch_size = properties.get('batch_size', 100) 

86 self.max_epochs = properties.get('max_epochs', 100) 

87 self.properties = properties 

88 

89 # Check the properties 

90 self.check_properties(properties) 

91 self.check_arguments() 

92 

93 def check_data_params(self, out_log, err_log): 

94 """ Checks all the input/output paths and parameters """ 

95 self.io_dict["in"]["input_decode_path"] = check_input_path(self.io_dict["in"]["input_decode_path"], "input_decode_path", False, out_log, self.__class__.__name__) 

96 if self.io_dict["in"]["input_predict_path"]: 

97 self.io_dict["in"]["input_predict_path"] = check_input_path(self.io_dict["in"]["input_predict_path"], "input_predict_path", True, out_log, self.__class__.__name__) 

98 self.io_dict["out"]["output_model_path"] = check_output_path(self.io_dict["out"]["output_model_path"], "output_model_path", False, out_log, self.__class__.__name__) 

99 if self.io_dict["out"]["output_test_decode_path"]: 

100 self.io_dict["out"]["output_test_decode_path"] = check_output_path(self.io_dict["out"]["output_test_decode_path"], "output_test_decode_path", True, out_log, self.__class__.__name__) 

101 if self.io_dict["out"]["output_test_predict_path"]: 

102 self.io_dict["out"]["output_test_predict_path"] = check_output_path(self.io_dict["out"]["output_test_predict_path"], "output_test_predict_path", True, out_log, self.__class__.__name__) 

103 

104 def build_model(self, n_in, n_out=None): 

105 """ Builds Neural network """ 

106 

107 # outputs list 

108 outputs = [] 

109 

110 # define encoder 

111 visible = Input(shape=(n_in, 1)) 

112 encoder = LSTM(100, activation='relu')(visible) 

113 

114 # define reconstruct decoder 

115 decoder1 = RepeatVector(n_in)(encoder) 

116 decoder1 = LSTM(100, activation='relu', return_sequences=True)(decoder1) 

117 decoder1 = TimeDistributed(Dense(1))(decoder1) 

118 

119 outputs.append(decoder1) 

120 

121 # define predict decoder 

122 if n_out: 

123 decoder2 = RepeatVector(n_out)(encoder) 

124 decoder2 = LSTM(100, activation='relu', return_sequences=True)(decoder2) 

125 decoder2 = TimeDistributed(Dense(1))(decoder2) 

126 outputs.append(decoder2) 

127 

128 # tie it together 

129 model = Model(inputs=visible, outputs=outputs) 

130 

131 return model 

132 

133 @launchlogger 

134 def launch(self) -> int: 

135 """Execute the :class:`AutoencoderNeuralNetwork <neural_networks.autoencoder_neural_network.AutoencoderNeuralNetwork>` neural_networks.autoencoder_neural_network.AutoencoderNeuralNetwork object.""" 

136 

137 # check input/output paths and parameters 

138 self.check_data_params(self.out_log, self.err_log) 

139 

140 # Setup Biobb 

141 if self.check_restart(): 

142 return 0 

143 self.stage_files() 

144 

145 # load decode dataset 

146 fu.log('Getting decode dataset from %s' % self.io_dict["in"]["input_decode_path"], self.out_log, self.global_log) 

147 data_dec = pd.read_csv(self.io_dict["in"]["input_decode_path"]) 

148 seq_in = np.array(data_dec) 

149 

150 # reshape input into [samples, timesteps, features] 

151 n_in = len(seq_in) 

152 seq_in = seq_in.reshape((1, n_in, 1)) 

153 

154 # load predict dataset 

155 n_out = None 

156 if (self.io_dict["in"]["input_predict_path"]): 

157 fu.log('Getting predict dataset from %s' % self.io_dict["in"]["input_predict_path"], self.out_log, self.global_log) 

158 data_pred = pd.read_csv(self.io_dict["in"]["input_predict_path"]) 

159 seq_out = np.array(data_pred) 

160 

161 # reshape output into [samples, timesteps, features] 

162 n_out = len(seq_out) 

163 seq_out = seq_out.reshape((1, n_out, 1)) 

164 

165 # build model 

166 fu.log('Building model', self.out_log, self.global_log) 

167 model = self.build_model(n_in, n_out) 

168 

169 # model summary 

170 stringlist = [] 

171 model.summary(print_fn=lambda x: stringlist.append(x)) 

172 model_summary = "\n".join(stringlist) 

173 fu.log('Model summary:\n\n%s\n' % model_summary, self.out_log, self.global_log) 

174 

175 # get optimizer 

176 mod = __import__('tensorflow.keras.optimizers', fromlist=[self.optimizer]) 

177 opt_class = getattr(mod, self.optimizer) 

178 opt = opt_class(lr=self.learning_rate) 

179 # compile model 

180 model.compile(optimizer=opt, loss='mse', metrics=['mse', 'mae']) 

181 

182 # fitting 

183 fu.log('Training model', self.out_log, self.global_log) 

184 y_list = [seq_in] 

185 if n_out: 

186 y_list.append(seq_out) 

187 # fit the model 

188 mf = model.fit(seq_in, 

189 y_list, 

190 batch_size=self.batch_size, 

191 epochs=self.max_epochs, 

192 verbose=1) 

193 

194 train_metrics = pd.DataFrame() 

195 metric = [] 

196 coefficient = [] 

197 for key, lst in mf.history.items(): 

198 metric.append(' '.join(x.capitalize() or '_' for x in key.split('_'))) 

199 coefficient.append(lst[-1]) 

200 

201 train_metrics['metric'] = metric 

202 train_metrics['coefficient'] = coefficient 

203 

204 fu.log('Calculating metrics\n\nMETRICS TABLE\n\n%s\n' % train_metrics, self.out_log, self.global_log) 

205 

206 # predicting 

207 fu.log('Predicting model', self.out_log, self.global_log) 

208 yhat = model.predict(seq_in, verbose=1) 

209 

210 decoding_table = pd.DataFrame() 

211 if (self.io_dict["in"]["input_predict_path"]): 

212 decoding_table['reconstructed'] = np.squeeze(np.asarray(yhat[0][0])) 

213 decoding_table['original'] = data_dec 

214 else: 

215 decoding_table['reconstructed'] = np.squeeze(np.asarray(yhat[0])) 

216 decoding_table['original'] = np.squeeze(np.asarray(data_dec)) 

217 decoding_table['residual'] = decoding_table['original'] - decoding_table['reconstructed'] 

218 decoding_table['difference %'] = np.absolute(decoding_table['residual']/decoding_table['original']*100) 

219 pd.set_option('display.float_format', lambda x: '%.5f' % x) 

220 # sort by difference in % 

221 decoding_table = decoding_table.sort_values(by=['difference %']) 

222 decoding_table = decoding_table.reset_index(drop=True) 

223 fu.log('RECONSTRUCTION TABLE\n\n%s\n' % decoding_table, self.out_log, self.global_log) 

224 

225 # save reconstruction data 

226 if (self.io_dict["out"]["output_test_decode_path"]): 

227 fu.log('Saving reconstruction data to %s' % self.io_dict["out"]["output_test_decode_path"], self.out_log, self.global_log) 

228 decoding_table.to_csv(self.io_dict["out"]["output_test_decode_path"], index=False, header=True) 

229 

230 if (self.io_dict["in"]["input_predict_path"]): 

231 prediction_table = pd.DataFrame() 

232 prediction_table['predicted'] = np.squeeze(np.asarray(yhat[1][0])) 

233 prediction_table['original'] = data_pred 

234 prediction_table['residual'] = prediction_table['original'] - prediction_table['predicted'] 

235 prediction_table['difference %'] = np.absolute(prediction_table['residual']/prediction_table['original']*100) 

236 pd.set_option('display.float_format', lambda x: '%.5f' % x) 

237 # sort by difference in % 

238 prediction_table = prediction_table.sort_values(by=['difference %']) 

239 prediction_table = prediction_table.reset_index(drop=True) 

240 fu.log('PREDICTION TABLE\n\n%s\n' % prediction_table, self.out_log, self.global_log) 

241 

242 # save decoding data 

243 if (self.io_dict["out"]["output_test_predict_path"]): 

244 fu.log('Saving prediction data to %s' % self.io_dict["out"]["output_test_predict_path"], self.out_log, self.global_log) 

245 prediction_table.to_csv(self.io_dict["out"]["output_test_predict_path"], index=False, header=True) 

246 

247 # save model and parameters 

248 vars_obj = { 

249 'type': 'autoencoder' 

250 } 

251 variables = json.dumps(vars_obj) 

252 fu.log('Saving model to %s' % self.io_dict["out"]["output_model_path"], self.out_log, self.global_log) 

253 with h5py.File(self.io_dict["out"]["output_model_path"], mode='w') as f: 

254 hdf5_format.save_model_to_hdf5(model, f) 

255 f.attrs['variables'] = variables 

256 

257 # Copy files to host 

258 self.copy_to_host() 

259 

260 self.tmp_files.extend([ 

261 self.stage_io_dict.get("unique_dir") 

262 ]) 

263 self.remove_tmp_files() 

264 

265 self.check_arguments(output_files_created=True, raise_exception=False) 

266 

267 return 0 

268 

269 

270def autoencoder_neural_network(input_decode_path: str, output_model_path: str, input_predict_path: str = None, output_test_decode_path: str = None, output_test_predict_path: str = None, properties: dict = None, **kwargs) -> int: 

271 """Execute the :class:`AutoencoderNeuralNetwork <neural_networks.autoencoder_neural_network.AutoencoderNeuralNetwork>` class and 

272 execute the :meth:`launch() <neural_networks.autoencoder_neural_network.AutoencoderNeuralNetwork.launch>` method.""" 

273 

274 return AutoencoderNeuralNetwork(input_decode_path=input_decode_path, 

275 output_model_path=output_model_path, 

276 input_predict_path=input_predict_path, 

277 output_test_decode_path=output_test_decode_path, 

278 output_test_predict_path=output_test_predict_path, 

279 properties=properties, **kwargs).launch() 

280 

281 

282def main(): 

283 """Command line execution of this building block. Please check the command line documentation.""" 

284 parser = argparse.ArgumentParser(description="Wrapper of the TensorFlow Keras LSTM method for encoding.", formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=99999)) 

285 parser.add_argument('--config', required=False, help='Configuration file') 

286 

287 # Specific args of each building block 

288 required_args = parser.add_argument_group('required arguments') 

289 required_args.add_argument('--input_decode_path', required=True, help='Path to the input decode dataset. Accepted formats: csv.') 

290 parser.add_argument('--input_predict_path', required=False, help='Path to the input predict dataset. Accepted formats: csv.') 

291 required_args.add_argument('--output_model_path', required=True, help='Path to the output model file. Accepted formats: h5.') 

292 parser.add_argument('--output_test_decode_path', required=False, help='Path to the test decode table file. Accepted formats: csv.') 

293 parser.add_argument('--output_test_predict_path', required=False, help='Path to the test predict table file. Accepted formats: csv.') 

294 

295 args = parser.parse_args() 

296 args.config = args.config or "{}" 

297 properties = settings.ConfReader(config=args.config).get_prop_dic() 

298 

299 # Specific call of each building block 

300 autoencoder_neural_network(input_decode_path=args.input_decode_path, 

301 output_model_path=args.output_model_path, 

302 input_predict_path=args.input_predict_path, 

303 output_test_decode_path=args.output_test_decode_path, 

304 output_test_predict_path=args.output_test_predict_path, 

305 properties=properties) 

306 

307 

308if __name__ == '__main__': 

309 main()