diff --git a/OAR.2071925.stderr b/OAR.2071925.stderr new file mode 100644 index 0000000000000000000000000000000000000000..a9c628a29c58ebc65df754dd5dfbbd99e2282b5f --- /dev/null +++ b/OAR.2071925.stderr @@ -0,0 +1,2 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071925.stdout b/OAR.2071925.stdout new file mode 100644 index 0000000000000000000000000000000000000000..af0b94e171570c9bc101981af319553b1f2a6858 --- /dev/null +++ b/OAR.2071925.stdout @@ -0,0 +1,614 @@ +Namespace(batch_size=64, beta=None, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='VAE_bs_64_ls_30', gpu_devices=[0, 1], is_beta_VAE=False, latent_spec_cont=30, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +Tesla K40c +Tesla K80 +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=60, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=30, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 780735 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last (iter 129)' +0/69092 Loss: 102.534 +3200/69092 Loss: 102.036 +6400/69092 Loss: 102.218 +9600/69092 Loss: 102.707 +12800/69092 Loss: 100.736 +16000/69092 Loss: 101.408 +19200/69092 Loss: 101.575 +22400/69092 Loss: 99.826 +25600/69092 Loss: 101.678 +28800/69092 Loss: 102.767 +32000/69092 Loss: 103.212 +35200/69092 Loss: 101.544 +38400/69092 Loss: 101.354 +41600/69092 Loss: 100.512 +44800/69092 Loss: 101.251 +48000/69092 Loss: 101.995 +51200/69092 Loss: 101.104 +54400/69092 Loss: 102.314 +57600/69092 Loss: 103.081 +60800/69092 Loss: 102.314 +64000/69092 Loss: 100.943 +67200/69092 Loss: 102.164 +Training time 0:13:38.847035 +Epoch: 1 Average loss: 101.76 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 130) +0/69092 Loss: 114.800 +3200/69092 Loss: 101.073 +6400/69092 Loss: 100.484 +9600/69092 Loss: 100.862 +12800/69092 Loss: 101.996 +16000/69092 Loss: 101.816 +19200/69092 Loss: 99.611 +22400/69092 Loss: 102.735 +25600/69092 Loss: 100.626 +28800/69092 Loss: 101.978 +32000/69092 Loss: 103.225 +35200/69092 Loss: 103.234 +38400/69092 Loss: 102.373 +41600/69092 Loss: 102.292 +44800/69092 Loss: 101.657 +48000/69092 Loss: 102.683 +51200/69092 Loss: 101.117 +54400/69092 Loss: 101.758 +57600/69092 Loss: 100.509 +60800/69092 Loss: 103.382 +64000/69092 Loss: 101.072 +67200/69092 Loss: 102.621 +Training time 0:13:40.612847 +Epoch: 2 Average loss: 101.79 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 131) +0/69092 Loss: 90.926 +3200/69092 Loss: 101.620 +6400/69092 Loss: 102.379 +9600/69092 Loss: 101.165 +12800/69092 Loss: 101.947 +16000/69092 Loss: 101.924 +19200/69092 Loss: 101.611 +22400/69092 Loss: 102.199 +25600/69092 Loss: 101.403 +28800/69092 Loss: 102.176 +32000/69092 Loss: 102.298 +35200/69092 Loss: 100.362 +38400/69092 Loss: 101.987 +41600/69092 Loss: 101.600 +44800/69092 Loss: 100.519 +48000/69092 Loss: 102.598 +51200/69092 Loss: 103.294 +54400/69092 Loss: 102.111 +57600/69092 Loss: 102.187 +60800/69092 Loss: 100.776 +64000/69092 Loss: 100.429 +67200/69092 Loss: 100.845 +Training time 0:13:42.175355 +Epoch: 3 Average loss: 101.66 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 132) +0/69092 Loss: 93.964 +3200/69092 Loss: 101.516 +6400/69092 Loss: 102.362 +9600/69092 Loss: 101.001 +12800/69092 Loss: 100.701 +16000/69092 Loss: 101.798 +19200/69092 Loss: 100.819 +22400/69092 Loss: 102.256 +25600/69092 Loss: 101.712 +28800/69092 Loss: 101.544 +32000/69092 Loss: 103.227 +35200/69092 Loss: 101.216 +38400/69092 Loss: 101.431 +41600/69092 Loss: 101.230 +44800/69092 Loss: 100.323 +48000/69092 Loss: 102.344 +51200/69092 Loss: 100.128 +54400/69092 Loss: 100.639 +57600/69092 Loss: 102.672 +60800/69092 Loss: 101.396 +64000/69092 Loss: 102.506 +67200/69092 Loss: 102.220 +Training time 0:13:39.965118 +Epoch: 4 Average loss: 101.61 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 133) +0/69092 Loss: 94.169 +3200/69092 Loss: 101.753 +6400/69092 Loss: 101.400 +9600/69092 Loss: 99.887 +12800/69092 Loss: 102.732 +16000/69092 Loss: 101.759 +19200/69092 Loss: 101.569 +22400/69092 Loss: 101.869 +25600/69092 Loss: 101.560 +28800/69092 Loss: 103.078 +32000/69092 Loss: 101.424 +35200/69092 Loss: 100.131 +38400/69092 Loss: 103.394 +41600/69092 Loss: 101.449 +44800/69092 Loss: 100.854 +48000/69092 Loss: 102.384 +51200/69092 Loss: 102.121 +54400/69092 Loss: 100.495 +57600/69092 Loss: 100.939 +60800/69092 Loss: 100.973 +64000/69092 Loss: 100.886 +67200/69092 Loss: 101.098 +Training time 0:13:39.968470 +Epoch: 5 Average loss: 101.54 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 134) +0/69092 Loss: 102.490 +3200/69092 Loss: 101.323 +6400/69092 Loss: 100.960 +9600/69092 Loss: 101.437 +12800/69092 Loss: 103.048 +16000/69092 Loss: 100.110 +19200/69092 Loss: 102.289 +22400/69092 Loss: 101.156 +25600/69092 Loss: 102.418 +28800/69092 Loss: 102.892 +32000/69092 Loss: 101.713 +35200/69092 Loss: 102.122 +38400/69092 Loss: 101.573 +41600/69092 Loss: 102.055 +44800/69092 Loss: 101.294 +48000/69092 Loss: 101.903 +51200/69092 Loss: 101.024 +54400/69092 Loss: 101.170 +57600/69092 Loss: 101.105 +60800/69092 Loss: 99.360 +64000/69092 Loss: 100.887 +67200/69092 Loss: 100.272 +Training time 0:13:41.708014 +Epoch: 6 Average loss: 101.43 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 135) +0/69092 Loss: 103.209 +3200/69092 Loss: 101.513 +6400/69092 Loss: 99.168 +9600/69092 Loss: 100.131 +12800/69092 Loss: 101.549 +16000/69092 Loss: 101.060 +19200/69092 Loss: 102.095 +22400/69092 Loss: 101.854 +25600/69092 Loss: 100.587 +28800/69092 Loss: 101.554 +32000/69092 Loss: 100.996 +35200/69092 Loss: 101.225 +38400/69092 Loss: 103.453 +41600/69092 Loss: 101.601 +44800/69092 Loss: 101.659 +48000/69092 Loss: 100.187 +51200/69092 Loss: 101.299 +54400/69092 Loss: 102.864 +57600/69092 Loss: 101.159 +60800/69092 Loss: 102.019 +64000/69092 Loss: 100.970 +67200/69092 Loss: 101.300 +Training time 0:13:32.946165 +Epoch: 7 Average loss: 101.37 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 136) +0/69092 Loss: 101.395 +3200/69092 Loss: 100.712 +6400/69092 Loss: 103.126 +9600/69092 Loss: 102.173 +12800/69092 Loss: 99.536 +16000/69092 Loss: 101.181 +19200/69092 Loss: 100.161 +22400/69092 Loss: 100.936 +25600/69092 Loss: 100.097 +28800/69092 Loss: 101.380 +32000/69092 Loss: 101.109 +35200/69092 Loss: 101.755 +38400/69092 Loss: 100.048 +41600/69092 Loss: 102.796 +44800/69092 Loss: 101.363 +48000/69092 Loss: 102.326 +51200/69092 Loss: 102.086 +54400/69092 Loss: 102.963 +57600/69092 Loss: 103.234 +60800/69092 Loss: 101.840 +64000/69092 Loss: 100.346 +67200/69092 Loss: 100.669 +Training time 0:13:37.533460 +Epoch: 8 Average loss: 101.40 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 137) +0/69092 Loss: 97.749 +3200/69092 Loss: 101.572 +6400/69092 Loss: 102.015 +9600/69092 Loss: 100.706 +12800/69092 Loss: 100.540 +16000/69092 Loss: 101.968 +19200/69092 Loss: 100.329 +22400/69092 Loss: 102.381 +25600/69092 Loss: 101.223 +28800/69092 Loss: 101.436 +32000/69092 Loss: 101.072 +35200/69092 Loss: 100.891 +38400/69092 Loss: 100.335 +41600/69092 Loss: 100.955 +44800/69092 Loss: 101.962 +48000/69092 Loss: 101.707 +51200/69092 Loss: 101.662 +54400/69092 Loss: 101.518 +57600/69092 Loss: 101.136 +60800/69092 Loss: 99.977 +64000/69092 Loss: 100.367 +67200/69092 Loss: 102.240 +Training time 0:13:37.159430 +Epoch: 9 Average loss: 101.27 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 138) +0/69092 Loss: 96.843 +3200/69092 Loss: 100.628 +6400/69092 Loss: 98.623 +9600/69092 Loss: 101.173 +12800/69092 Loss: 101.865 +16000/69092 Loss: 100.244 +19200/69092 Loss: 103.324 +22400/69092 Loss: 100.510 +25600/69092 Loss: 100.481 +28800/69092 Loss: 100.251 +32000/69092 Loss: 100.785 +35200/69092 Loss: 102.649 +38400/69092 Loss: 100.703 +41600/69092 Loss: 101.355 +44800/69092 Loss: 102.376 +48000/69092 Loss: 103.408 +51200/69092 Loss: 101.330 +54400/69092 Loss: 101.807 +57600/69092 Loss: 101.790 +60800/69092 Loss: 101.363 +64000/69092 Loss: 102.553 +67200/69092 Loss: 100.505 +Training time 0:13:45.293890 +Epoch: 10 Average loss: 101.31 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 139) +0/69092 Loss: 112.161 +3200/69092 Loss: 101.777 +6400/69092 Loss: 100.855 +9600/69092 Loss: 101.773 +12800/69092 Loss: 102.093 +16000/69092 Loss: 100.143 +19200/69092 Loss: 100.434 +22400/69092 Loss: 102.122 +25600/69092 Loss: 102.210 +28800/69092 Loss: 99.864 +32000/69092 Loss: 102.064 +35200/69092 Loss: 101.566 +38400/69092 Loss: 101.978 +41600/69092 Loss: 100.993 +44800/69092 Loss: 101.276 +48000/69092 Loss: 102.266 +51200/69092 Loss: 102.357 +54400/69092 Loss: 101.238 +57600/69092 Loss: 98.933 +60800/69092 Loss: 100.897 +64000/69092 Loss: 100.563 +67200/69092 Loss: 102.239 +Training time 0:13:47.168059 +Epoch: 11 Average loss: 101.35 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 140) +0/69092 Loss: 110.515 +3200/69092 Loss: 101.177 +6400/69092 Loss: 100.226 +9600/69092 Loss: 101.056 +12800/69092 Loss: 101.302 +16000/69092 Loss: 99.309 +19200/69092 Loss: 101.269 +22400/69092 Loss: 102.534 +25600/69092 Loss: 101.872 +28800/69092 Loss: 101.718 +32000/69092 Loss: 102.067 +35200/69092 Loss: 102.550 +38400/69092 Loss: 101.435 +41600/69092 Loss: 101.267 +44800/69092 Loss: 100.383 +48000/69092 Loss: 100.301 +51200/69092 Loss: 99.972 +54400/69092 Loss: 102.224 +57600/69092 Loss: 102.747 +60800/69092 Loss: 100.852 +64000/69092 Loss: 100.935 +67200/69092 Loss: 100.921 +Training time 0:13:35.979609 +Epoch: 12 Average loss: 101.24 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 141) +0/69092 Loss: 104.926 +3200/69092 Loss: 101.516 +6400/69092 Loss: 102.450 +9600/69092 Loss: 101.628 +12800/69092 Loss: 100.340 +16000/69092 Loss: 102.149 +19200/69092 Loss: 100.950 +22400/69092 Loss: 100.328 +25600/69092 Loss: 102.320 +28800/69092 Loss: 101.453 +32000/69092 Loss: 100.098 +35200/69092 Loss: 101.167 +38400/69092 Loss: 102.012 +41600/69092 Loss: 101.536 +44800/69092 Loss: 102.023 +48000/69092 Loss: 99.752 +51200/69092 Loss: 99.400 +54400/69092 Loss: 100.739 +57600/69092 Loss: 99.739 +60800/69092 Loss: 100.977 +64000/69092 Loss: 101.560 +67200/69092 Loss: 102.735 +Training time 0:13:36.534861 +Epoch: 13 Average loss: 101.17 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 142) +0/69092 Loss: 101.172 +3200/69092 Loss: 101.438 +6400/69092 Loss: 102.836 +9600/69092 Loss: 100.026 +12800/69092 Loss: 101.309 +16000/69092 Loss: 101.171 +19200/69092 Loss: 100.645 +22400/69092 Loss: 100.532 +25600/69092 Loss: 100.822 +28800/69092 Loss: 100.009 +32000/69092 Loss: 101.301 +35200/69092 Loss: 100.742 +38400/69092 Loss: 101.677 +41600/69092 Loss: 101.701 +44800/69092 Loss: 103.015 +48000/69092 Loss: 102.149 +51200/69092 Loss: 100.553 +54400/69092 Loss: 101.219 +57600/69092 Loss: 101.769 +60800/69092 Loss: 100.160 +64000/69092 Loss: 102.333 +67200/69092 Loss: 101.800 +Training time 0:13:37.293015 +Epoch: 14 Average loss: 101.31 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 143) +0/69092 Loss: 108.805 +3200/69092 Loss: 100.707 +6400/69092 Loss: 100.088 +9600/69092 Loss: 101.559 +12800/69092 Loss: 101.380 +16000/69092 Loss: 101.018 +19200/69092 Loss: 100.826 +22400/69092 Loss: 101.537 +25600/69092 Loss: 100.539 +28800/69092 Loss: 100.839 +32000/69092 Loss: 100.947 +35200/69092 Loss: 102.349 +38400/69092 Loss: 100.047 +41600/69092 Loss: 101.840 +44800/69092 Loss: 100.186 +48000/69092 Loss: 102.917 +51200/69092 Loss: 101.335 +54400/69092 Loss: 101.732 +57600/69092 Loss: 99.359 +60800/69092 Loss: 99.792 +64000/69092 Loss: 101.038 +67200/69092 Loss: 100.563 +Training time 0:13:38.617665 +Epoch: 15 Average loss: 101.00 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 144) +0/69092 Loss: 103.428 +3200/69092 Loss: 102.019 +6400/69092 Loss: 101.325 +9600/69092 Loss: 102.409 +12800/69092 Loss: 100.924 +16000/69092 Loss: 101.393 +19200/69092 Loss: 99.708 +22400/69092 Loss: 101.004 +25600/69092 Loss: 100.803 +28800/69092 Loss: 102.168 +32000/69092 Loss: 98.200 +35200/69092 Loss: 102.215 +38400/69092 Loss: 102.296 +41600/69092 Loss: 101.760 +44800/69092 Loss: 100.265 +48000/69092 Loss: 99.268 +51200/69092 Loss: 100.277 +54400/69092 Loss: 102.682 +57600/69092 Loss: 101.112 +60800/69092 Loss: 101.129 +64000/69092 Loss: 101.590 +67200/69092 Loss: 100.514 +Training time 0:13:36.266593 +Epoch: 16 Average loss: 101.10 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 145) +0/69092 Loss: 102.075 +3200/69092 Loss: 101.324 +6400/69092 Loss: 100.904 +9600/69092 Loss: 100.849 +12800/69092 Loss: 101.884 +16000/69092 Loss: 100.862 +19200/69092 Loss: 101.149 +22400/69092 Loss: 101.627 +25600/69092 Loss: 100.603 +28800/69092 Loss: 100.834 +32000/69092 Loss: 101.490 +35200/69092 Loss: 99.884 +38400/69092 Loss: 102.170 +41600/69092 Loss: 100.476 +44800/69092 Loss: 100.072 +48000/69092 Loss: 100.504 +51200/69092 Loss: 101.009 +54400/69092 Loss: 99.449 +57600/69092 Loss: 102.071 +60800/69092 Loss: 101.257 +64000/69092 Loss: 101.457 +67200/69092 Loss: 100.634 +Training time 0:13:44.392443 +Epoch: 17 Average loss: 101.02 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 146) +0/69092 Loss: 102.443 +3200/69092 Loss: 101.318 +6400/69092 Loss: 103.426 +9600/69092 Loss: 102.822 +12800/69092 Loss: 101.628 +16000/69092 Loss: 101.508 +19200/69092 Loss: 101.354 +22400/69092 Loss: 101.734 +25600/69092 Loss: 101.245 +28800/69092 Loss: 99.157 +32000/69092 Loss: 101.736 +35200/69092 Loss: 101.405 +38400/69092 Loss: 99.550 +41600/69092 Loss: 100.731 +44800/69092 Loss: 102.071 +48000/69092 Loss: 101.316 +51200/69092 Loss: 101.294 +54400/69092 Loss: 100.452 +57600/69092 Loss: 100.354 +60800/69092 Loss: 101.136 +64000/69092 Loss: 100.233 +67200/69092 Loss: 101.333 +Training time 0:13:40.917111 +Epoch: 18 Average loss: 101.20 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 147) +0/69092 Loss: 96.744 +3200/69092 Loss: 101.016 +6400/69092 Loss: 100.818 +9600/69092 Loss: 100.498 +12800/69092 Loss: 101.251 +16000/69092 Loss: 100.795 +19200/69092 Loss: 101.181 +22400/69092 Loss: 102.624 +25600/69092 Loss: 100.103 +28800/69092 Loss: 102.347 +32000/69092 Loss: 98.818 +35200/69092 Loss: 102.360 +38400/69092 Loss: 102.031 +41600/69092 Loss: 100.385 +44800/69092 Loss: 100.641 +48000/69092 Loss: 101.366 +51200/69092 Loss: 101.084 +54400/69092 Loss: 100.710 +57600/69092 Loss: 101.880 +60800/69092 Loss: 100.524 +64000/69092 Loss: 100.779 +67200/69092 Loss: 101.739 +Training time 0:13:39.137368 +Epoch: 19 Average loss: 101.09 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 148) +0/69092 Loss: 110.795 +3200/69092 Loss: 100.172 +6400/69092 Loss: 100.835 +9600/69092 Loss: 102.914 +12800/69092 Loss: 100.496 +16000/69092 Loss: 99.775 +19200/69092 Loss: 101.589 +22400/69092 Loss: 101.415 +25600/69092 Loss: 100.814 +28800/69092 Loss: 101.546 +32000/69092 Loss: 100.133 +35200/69092 Loss: 100.845 +38400/69092 Loss: 101.924 +41600/69092 Loss: 102.358 +44800/69092 Loss: 102.181 +48000/69092 Loss: 99.168 +51200/69092 Loss: 102.921 +54400/69092 Loss: 99.917 +57600/69092 Loss: 99.900 +60800/69092 Loss: 99.359 +64000/69092 Loss: 102.033 +67200/69092 Loss: 101.576 +Training time 0:13:41.223759 +Epoch: 20 Average loss: 101.08 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 149) +0/69092 Loss: 95.958 +3200/69092 Loss: 99.846 +6400/69092 Loss: 101.695 +9600/69092 Loss: 101.363 +12800/69092 Loss: 101.017 +16000/69092 Loss: 101.406 +19200/69092 Loss: 100.208 +22400/69092 Loss: 100.811 +25600/69092 Loss: 102.059 +28800/69092 Loss: 100.063 +32000/69092 Loss: 101.056 +35200/69092 Loss: 99.961 +38400/69092 Loss: 101.395 +41600/69092 Loss: 100.612 +44800/69092 Loss: 101.276 +48000/69092 Loss: 100.546 +51200/69092 Loss: 101.843 +54400/69092 Loss: 101.545 +57600/69092 Loss: 101.428 +60800/69092 Loss: 102.501 +64000/69092 Loss: 102.094 +67200/69092 Loss: 102.237 +Training time 0:13:38.211588 +Epoch: 21 Average loss: 101.20 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 150) +0/69092 Loss: 97.744 +3200/69092 Loss: 98.888 +6400/69092 Loss: 100.604 +9600/69092 Loss: 100.722 +12800/69092 Loss: 100.720 +16000/69092 Loss: 101.149 +19200/69092 Loss: 100.248 +22400/69092 Loss: 100.703 +25600/69092 Loss: 101.915 +28800/69092 Loss: 103.134 +32000/69092 Loss: 102.218 +35200/69092 Loss: 100.886 +38400/69092 Loss: 100.174 +41600/69092 Loss: 102.200 +44800/69092 Loss: 100.549 +48000/69092 Loss: 101.682 +51200/69092 Loss: 101.382 +54400/69092 Loss: 100.393 +57600/69092 Loss: 99.708 +60800/69092 Loss: 99.162 +64000/69092 Loss: 101.203 +67200/69092 Loss: 101.223 +Training time 0:13:38.859805 +Epoch: 22 Average loss: 100.89 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last' (iter 151) +0/69092 Loss: 104.809 +3200/69092 Loss: 100.932 +6400/69092 Loss: 101.622 +9600/69092 Loss: 101.292 +12800/69092 Loss: 102.105 +16000/69092 Loss: 100.229 +19200/69092 Loss: 99.238 +22400/69092 Loss: 100.010 +25600/69092 Loss: 99.492 +28800/69092 Loss: 102.281 +32000/69092 Loss: 101.187 +35200/69092 Loss: 101.881 +38400/69092 Loss: 101.507 +41600/69092 Loss: 99.676 +44800/69092 Loss: 100.538 +48000/69092 Loss: 99.951 +51200/69092 Loss: 103.184 diff --git a/OAR.2071926.stderr b/OAR.2071926.stderr new file mode 100644 index 0000000000000000000000000000000000000000..3fa38d97833abb3d5f736549b02733f2cf4af075 --- /dev/null +++ b/OAR.2071926.stderr @@ -0,0 +1,8 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py:26: UserWarning: + There is an imbalance between your GPUs. You may want to exclude GPU 1 which + has less than 75% of the memory or cores of GPU 0. You can do so by setting + the device_ids argument to DataParallel, or by setting the CUDA_VISIBLE_DEVICES + environment variable. + warnings.warn(imbalance_warn.format(device_ids[min_pos], device_ids[max_pos])) +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071926.stdout b/OAR.2071926.stdout new file mode 100644 index 0000000000000000000000000000000000000000..60f9a19ef929affaa4ce598008f60b7bfd567872 --- /dev/null +++ b/OAR.2071926.stdout @@ -0,0 +1,1050 @@ +Namespace(batch_size=64, beta=None, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='VAE_bs_64_ls_40', gpu_devices=[0, 1], is_beta_VAE=False, latent_spec_cont=40, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +Tesla K40c +Tesla K20m +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=80, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=40, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 788435 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last (iter 239)' +0/69092 Loss: 107.115 +3200/69092 Loss: 96.144 +6400/69092 Loss: 94.953 +9600/69092 Loss: 93.092 +12800/69092 Loss: 94.239 +16000/69092 Loss: 95.661 +19200/69092 Loss: 96.719 +22400/69092 Loss: 93.757 +25600/69092 Loss: 95.639 +28800/69092 Loss: 94.715 +32000/69092 Loss: 94.777 +35200/69092 Loss: 94.920 +38400/69092 Loss: 95.831 +41600/69092 Loss: 93.925 +44800/69092 Loss: 95.380 +48000/69092 Loss: 94.872 +51200/69092 Loss: 94.707 +54400/69092 Loss: 96.006 +57600/69092 Loss: 93.720 +60800/69092 Loss: 95.304 +64000/69092 Loss: 94.826 +67200/69092 Loss: 94.861 +Training time 0:07:50.791595 +Epoch: 1 Average loss: 94.92 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 240) +0/69092 Loss: 88.184 +3200/69092 Loss: 95.239 +6400/69092 Loss: 95.823 +9600/69092 Loss: 96.482 +12800/69092 Loss: 94.365 +16000/69092 Loss: 94.666 +19200/69092 Loss: 94.810 +22400/69092 Loss: 94.448 +25600/69092 Loss: 94.133 +28800/69092 Loss: 93.688 +32000/69092 Loss: 95.754 +35200/69092 Loss: 93.416 +38400/69092 Loss: 95.240 +41600/69092 Loss: 95.343 +44800/69092 Loss: 94.340 +48000/69092 Loss: 95.801 +51200/69092 Loss: 93.977 +54400/69092 Loss: 94.068 +57600/69092 Loss: 95.123 +60800/69092 Loss: 94.618 +64000/69092 Loss: 94.918 +67200/69092 Loss: 94.817 +Training time 0:07:41.487764 +Epoch: 2 Average loss: 94.86 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 241) +0/69092 Loss: 91.296 +3200/69092 Loss: 93.777 +6400/69092 Loss: 94.320 +9600/69092 Loss: 94.114 +12800/69092 Loss: 94.358 +16000/69092 Loss: 95.152 +19200/69092 Loss: 93.773 +22400/69092 Loss: 96.712 +25600/69092 Loss: 94.708 +28800/69092 Loss: 93.869 +32000/69092 Loss: 94.225 +35200/69092 Loss: 94.478 +38400/69092 Loss: 94.703 +41600/69092 Loss: 93.975 +44800/69092 Loss: 96.143 +48000/69092 Loss: 96.074 +51200/69092 Loss: 95.269 +54400/69092 Loss: 95.887 +57600/69092 Loss: 94.271 +60800/69092 Loss: 95.953 +64000/69092 Loss: 95.280 +67200/69092 Loss: 94.895 +Training time 0:07:36.187335 +Epoch: 3 Average loss: 94.88 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 242) +0/69092 Loss: 95.492 +3200/69092 Loss: 93.854 +6400/69092 Loss: 93.991 +9600/69092 Loss: 95.113 +12800/69092 Loss: 94.993 +16000/69092 Loss: 94.576 +19200/69092 Loss: 96.490 +22400/69092 Loss: 94.842 +25600/69092 Loss: 93.689 +28800/69092 Loss: 94.116 +32000/69092 Loss: 96.230 +35200/69092 Loss: 95.289 +38400/69092 Loss: 95.706 +41600/69092 Loss: 95.148 +44800/69092 Loss: 95.772 +48000/69092 Loss: 95.783 +51200/69092 Loss: 94.778 +54400/69092 Loss: 95.549 +57600/69092 Loss: 93.751 +60800/69092 Loss: 95.438 +64000/69092 Loss: 92.468 +67200/69092 Loss: 94.668 +Training time 0:07:34.470143 +Epoch: 4 Average loss: 94.88 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 243) +0/69092 Loss: 96.341 +3200/69092 Loss: 94.349 +6400/69092 Loss: 94.858 +9600/69092 Loss: 94.632 +12800/69092 Loss: 94.771 +16000/69092 Loss: 95.489 +19200/69092 Loss: 94.117 +22400/69092 Loss: 95.811 +25600/69092 Loss: 95.329 +28800/69092 Loss: 94.223 +32000/69092 Loss: 95.555 +35200/69092 Loss: 95.143 +38400/69092 Loss: 93.479 +41600/69092 Loss: 95.180 +44800/69092 Loss: 95.523 +48000/69092 Loss: 95.209 +51200/69092 Loss: 94.912 +54400/69092 Loss: 94.412 +57600/69092 Loss: 93.693 +60800/69092 Loss: 94.666 +64000/69092 Loss: 94.290 +67200/69092 Loss: 94.244 +Training time 0:07:31.288607 +Epoch: 5 Average loss: 94.77 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 244) +0/69092 Loss: 100.813 +3200/69092 Loss: 94.317 +6400/69092 Loss: 94.578 +9600/69092 Loss: 95.588 +12800/69092 Loss: 93.439 +16000/69092 Loss: 96.193 +19200/69092 Loss: 94.873 +22400/69092 Loss: 94.870 +25600/69092 Loss: 94.276 +28800/69092 Loss: 94.297 +32000/69092 Loss: 94.598 +35200/69092 Loss: 92.408 +38400/69092 Loss: 95.609 +41600/69092 Loss: 94.230 +44800/69092 Loss: 95.950 +48000/69092 Loss: 95.544 +51200/69092 Loss: 97.070 +54400/69092 Loss: 93.768 +57600/69092 Loss: 95.706 +60800/69092 Loss: 94.584 +64000/69092 Loss: 94.515 +67200/69092 Loss: 93.712 +Training time 0:07:40.407501 +Epoch: 6 Average loss: 94.80 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 245) +0/69092 Loss: 84.678 +3200/69092 Loss: 94.838 +6400/69092 Loss: 93.629 +9600/69092 Loss: 96.019 +12800/69092 Loss: 94.610 +16000/69092 Loss: 95.538 +19200/69092 Loss: 94.371 +22400/69092 Loss: 95.503 +25600/69092 Loss: 95.000 +28800/69092 Loss: 94.244 +32000/69092 Loss: 94.539 +35200/69092 Loss: 94.524 +38400/69092 Loss: 94.806 +41600/69092 Loss: 94.387 +44800/69092 Loss: 93.409 +48000/69092 Loss: 93.685 +51200/69092 Loss: 94.124 +54400/69092 Loss: 95.382 +57600/69092 Loss: 94.497 +60800/69092 Loss: 94.849 +64000/69092 Loss: 94.038 +67200/69092 Loss: 94.939 +Training time 0:07:37.721995 +Epoch: 7 Average loss: 94.63 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 246) +0/69092 Loss: 89.671 +3200/69092 Loss: 94.643 +6400/69092 Loss: 94.845 +9600/69092 Loss: 95.375 +12800/69092 Loss: 94.272 +16000/69092 Loss: 94.186 +19200/69092 Loss: 92.535 +22400/69092 Loss: 95.824 +25600/69092 Loss: 95.247 +28800/69092 Loss: 94.298 +32000/69092 Loss: 94.496 +35200/69092 Loss: 95.532 +38400/69092 Loss: 94.614 +41600/69092 Loss: 95.700 +44800/69092 Loss: 95.332 +48000/69092 Loss: 94.459 +51200/69092 Loss: 94.370 +54400/69092 Loss: 94.947 +57600/69092 Loss: 92.311 +60800/69092 Loss: 95.635 +64000/69092 Loss: 95.287 +67200/69092 Loss: 96.058 +Training time 0:07:35.909222 +Epoch: 8 Average loss: 94.74 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 247) +0/69092 Loss: 100.903 +3200/69092 Loss: 94.753 +6400/69092 Loss: 95.036 +9600/69092 Loss: 93.688 +12800/69092 Loss: 94.177 +16000/69092 Loss: 94.756 +19200/69092 Loss: 94.085 +22400/69092 Loss: 94.164 +25600/69092 Loss: 94.147 +28800/69092 Loss: 94.651 +32000/69092 Loss: 95.327 +35200/69092 Loss: 95.869 +38400/69092 Loss: 95.395 +41600/69092 Loss: 94.101 +44800/69092 Loss: 93.981 +48000/69092 Loss: 94.858 +51200/69092 Loss: 94.705 +54400/69092 Loss: 95.636 +57600/69092 Loss: 95.710 +60800/69092 Loss: 93.323 +64000/69092 Loss: 96.510 +67200/69092 Loss: 93.088 +Training time 0:07:37.958332 +Epoch: 9 Average loss: 94.67 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 248) +0/69092 Loss: 98.991 +3200/69092 Loss: 93.498 +6400/69092 Loss: 95.857 +9600/69092 Loss: 93.504 +12800/69092 Loss: 93.981 +16000/69092 Loss: 93.482 +19200/69092 Loss: 95.759 +22400/69092 Loss: 95.438 +25600/69092 Loss: 94.804 +28800/69092 Loss: 94.057 +32000/69092 Loss: 95.488 +35200/69092 Loss: 94.579 +38400/69092 Loss: 93.834 +41600/69092 Loss: 93.938 +44800/69092 Loss: 96.286 +48000/69092 Loss: 95.217 +51200/69092 Loss: 93.604 +54400/69092 Loss: 94.940 +57600/69092 Loss: 95.563 +60800/69092 Loss: 94.343 +64000/69092 Loss: 95.703 +67200/69092 Loss: 93.698 +Training time 0:07:34.264781 +Epoch: 10 Average loss: 94.64 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 249) +0/69092 Loss: 96.338 +3200/69092 Loss: 94.590 +6400/69092 Loss: 94.183 +9600/69092 Loss: 94.754 +12800/69092 Loss: 94.780 +16000/69092 Loss: 95.641 +19200/69092 Loss: 93.974 +22400/69092 Loss: 95.738 +25600/69092 Loss: 94.854 +28800/69092 Loss: 94.071 +32000/69092 Loss: 95.264 +35200/69092 Loss: 94.423 +38400/69092 Loss: 94.114 +41600/69092 Loss: 95.172 +44800/69092 Loss: 94.337 +48000/69092 Loss: 95.276 +51200/69092 Loss: 94.170 +54400/69092 Loss: 93.907 +57600/69092 Loss: 95.703 +60800/69092 Loss: 93.719 +64000/69092 Loss: 93.856 +67200/69092 Loss: 95.361 +Training time 0:07:33.702575 +Epoch: 11 Average loss: 94.66 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 250) +0/69092 Loss: 100.942 +3200/69092 Loss: 93.738 +6400/69092 Loss: 94.480 +9600/69092 Loss: 94.316 +12800/69092 Loss: 94.968 +16000/69092 Loss: 94.243 +19200/69092 Loss: 95.541 +22400/69092 Loss: 95.174 +25600/69092 Loss: 94.635 +28800/69092 Loss: 95.222 +32000/69092 Loss: 95.156 +35200/69092 Loss: 94.535 +38400/69092 Loss: 94.693 +41600/69092 Loss: 93.758 +44800/69092 Loss: 95.036 +48000/69092 Loss: 94.345 +51200/69092 Loss: 93.736 +54400/69092 Loss: 95.426 +57600/69092 Loss: 94.698 +60800/69092 Loss: 95.613 +64000/69092 Loss: 94.012 +67200/69092 Loss: 94.687 +Training time 0:07:45.223018 +Epoch: 12 Average loss: 94.72 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 251) +0/69092 Loss: 90.434 +3200/69092 Loss: 94.635 +6400/69092 Loss: 94.848 +9600/69092 Loss: 96.252 +12800/69092 Loss: 93.579 +16000/69092 Loss: 93.772 +19200/69092 Loss: 94.314 +22400/69092 Loss: 95.190 +25600/69092 Loss: 94.962 +28800/69092 Loss: 95.252 +32000/69092 Loss: 95.202 +35200/69092 Loss: 94.123 +38400/69092 Loss: 95.166 +41600/69092 Loss: 95.511 +44800/69092 Loss: 93.693 +48000/69092 Loss: 94.801 +51200/69092 Loss: 94.598 +54400/69092 Loss: 95.383 +57600/69092 Loss: 94.257 +60800/69092 Loss: 94.973 +64000/69092 Loss: 93.321 +67200/69092 Loss: 93.447 +Training time 0:07:52.187914 +Epoch: 13 Average loss: 94.66 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 252) +0/69092 Loss: 85.430 +3200/69092 Loss: 94.940 +6400/69092 Loss: 95.067 +9600/69092 Loss: 94.559 +12800/69092 Loss: 94.629 +16000/69092 Loss: 95.475 +19200/69092 Loss: 93.454 +22400/69092 Loss: 94.783 +25600/69092 Loss: 94.548 +28800/69092 Loss: 94.813 +32000/69092 Loss: 95.987 +35200/69092 Loss: 95.610 +38400/69092 Loss: 93.338 +41600/69092 Loss: 95.800 +44800/69092 Loss: 95.104 +48000/69092 Loss: 95.476 +51200/69092 Loss: 93.797 +54400/69092 Loss: 93.919 +57600/69092 Loss: 94.411 +60800/69092 Loss: 94.585 +64000/69092 Loss: 94.856 +67200/69092 Loss: 94.021 +Training time 0:07:49.779707 +Epoch: 14 Average loss: 94.72 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 253) +0/69092 Loss: 95.120 +3200/69092 Loss: 94.783 +6400/69092 Loss: 93.848 +9600/69092 Loss: 94.201 +12800/69092 Loss: 94.406 +16000/69092 Loss: 95.225 +19200/69092 Loss: 94.874 +22400/69092 Loss: 94.254 +25600/69092 Loss: 94.114 +28800/69092 Loss: 93.878 +32000/69092 Loss: 95.399 +35200/69092 Loss: 94.656 +38400/69092 Loss: 94.471 +41600/69092 Loss: 94.607 +44800/69092 Loss: 93.156 +48000/69092 Loss: 94.416 +51200/69092 Loss: 94.910 +54400/69092 Loss: 93.830 +57600/69092 Loss: 94.580 +60800/69092 Loss: 93.446 +64000/69092 Loss: 95.133 +67200/69092 Loss: 95.964 +Training time 0:07:40.646446 +Epoch: 15 Average loss: 94.48 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 254) +0/69092 Loss: 86.913 +3200/69092 Loss: 94.085 +6400/69092 Loss: 94.676 +9600/69092 Loss: 96.533 +12800/69092 Loss: 94.517 +16000/69092 Loss: 94.314 +19200/69092 Loss: 94.220 +22400/69092 Loss: 94.264 +25600/69092 Loss: 92.935 +28800/69092 Loss: 94.763 +32000/69092 Loss: 94.426 +35200/69092 Loss: 94.177 +38400/69092 Loss: 95.931 +41600/69092 Loss: 95.531 +44800/69092 Loss: 93.417 +48000/69092 Loss: 93.912 +51200/69092 Loss: 94.897 +54400/69092 Loss: 94.307 +57600/69092 Loss: 94.409 +60800/69092 Loss: 94.410 +64000/69092 Loss: 94.081 +67200/69092 Loss: 95.346 +Training time 0:07:36.018561 +Epoch: 16 Average loss: 94.57 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 255) +0/69092 Loss: 96.777 +3200/69092 Loss: 94.232 +6400/69092 Loss: 94.977 +9600/69092 Loss: 94.023 +12800/69092 Loss: 95.017 +16000/69092 Loss: 93.906 +19200/69092 Loss: 94.546 +22400/69092 Loss: 94.468 +25600/69092 Loss: 95.171 +28800/69092 Loss: 93.640 +32000/69092 Loss: 94.658 +35200/69092 Loss: 95.302 +38400/69092 Loss: 94.736 +41600/69092 Loss: 93.309 +44800/69092 Loss: 95.051 +48000/69092 Loss: 94.710 +51200/69092 Loss: 93.588 +54400/69092 Loss: 95.197 +57600/69092 Loss: 94.820 +60800/69092 Loss: 93.350 +64000/69092 Loss: 94.439 +67200/69092 Loss: 95.510 +Training time 0:07:37.855103 +Epoch: 17 Average loss: 94.54 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 256) +0/69092 Loss: 101.684 +3200/69092 Loss: 94.559 +6400/69092 Loss: 94.739 +9600/69092 Loss: 95.764 +12800/69092 Loss: 93.370 +16000/69092 Loss: 94.570 +19200/69092 Loss: 95.003 +22400/69092 Loss: 93.765 +25600/69092 Loss: 94.356 +28800/69092 Loss: 94.998 +32000/69092 Loss: 94.055 +35200/69092 Loss: 93.653 +38400/69092 Loss: 94.553 +41600/69092 Loss: 93.680 +44800/69092 Loss: 95.310 +48000/69092 Loss: 93.775 +51200/69092 Loss: 91.700 +54400/69092 Loss: 95.288 +57600/69092 Loss: 94.939 +60800/69092 Loss: 95.419 +64000/69092 Loss: 95.868 +67200/69092 Loss: 94.779 +Training time 0:07:37.858746 +Epoch: 18 Average loss: 94.49 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 257) +0/69092 Loss: 96.910 +3200/69092 Loss: 95.853 +6400/69092 Loss: 94.429 +9600/69092 Loss: 94.754 +12800/69092 Loss: 93.127 +16000/69092 Loss: 93.772 +19200/69092 Loss: 94.906 +22400/69092 Loss: 95.004 +25600/69092 Loss: 94.717 +28800/69092 Loss: 95.597 +32000/69092 Loss: 95.501 +35200/69092 Loss: 94.846 +38400/69092 Loss: 94.447 +41600/69092 Loss: 93.667 +44800/69092 Loss: 93.858 +48000/69092 Loss: 93.926 +51200/69092 Loss: 94.066 +54400/69092 Loss: 94.172 +57600/69092 Loss: 93.810 +60800/69092 Loss: 93.445 +64000/69092 Loss: 94.430 +67200/69092 Loss: 95.836 +Training time 0:07:37.058295 +Epoch: 19 Average loss: 94.54 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 258) +0/69092 Loss: 98.137 +3200/69092 Loss: 95.687 +6400/69092 Loss: 93.072 +9600/69092 Loss: 92.561 +12800/69092 Loss: 95.054 +16000/69092 Loss: 95.547 +19200/69092 Loss: 94.516 +22400/69092 Loss: 94.784 +25600/69092 Loss: 94.360 +28800/69092 Loss: 94.958 +32000/69092 Loss: 93.761 +35200/69092 Loss: 95.370 +38400/69092 Loss: 94.824 +41600/69092 Loss: 95.439 +44800/69092 Loss: 94.956 +48000/69092 Loss: 94.107 +51200/69092 Loss: 93.872 +54400/69092 Loss: 94.252 +57600/69092 Loss: 94.347 +60800/69092 Loss: 94.054 +64000/69092 Loss: 94.425 +67200/69092 Loss: 93.218 +Training time 0:07:39.688092 +Epoch: 20 Average loss: 94.49 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 259) +0/69092 Loss: 86.177 +3200/69092 Loss: 93.400 +6400/69092 Loss: 94.399 +9600/69092 Loss: 95.195 +12800/69092 Loss: 94.801 +16000/69092 Loss: 94.270 +19200/69092 Loss: 93.821 +22400/69092 Loss: 93.997 +25600/69092 Loss: 94.577 +28800/69092 Loss: 94.196 +32000/69092 Loss: 95.066 +35200/69092 Loss: 94.995 +38400/69092 Loss: 94.059 +41600/69092 Loss: 94.573 +44800/69092 Loss: 93.481 +48000/69092 Loss: 95.030 +51200/69092 Loss: 95.501 +54400/69092 Loss: 94.967 +57600/69092 Loss: 94.556 +60800/69092 Loss: 94.177 +64000/69092 Loss: 95.666 +67200/69092 Loss: 94.133 +Training time 0:07:43.573102 +Epoch: 21 Average loss: 94.53 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 260) +0/69092 Loss: 99.793 +3200/69092 Loss: 94.147 +6400/69092 Loss: 95.434 +9600/69092 Loss: 95.217 +12800/69092 Loss: 94.321 +16000/69092 Loss: 95.361 +19200/69092 Loss: 94.131 +22400/69092 Loss: 93.712 +25600/69092 Loss: 94.402 +28800/69092 Loss: 94.374 +32000/69092 Loss: 93.928 +35200/69092 Loss: 93.980 +38400/69092 Loss: 93.457 +41600/69092 Loss: 94.467 +44800/69092 Loss: 94.433 +48000/69092 Loss: 94.036 +51200/69092 Loss: 95.764 +54400/69092 Loss: 95.252 +57600/69092 Loss: 93.298 +60800/69092 Loss: 94.516 +64000/69092 Loss: 94.321 +67200/69092 Loss: 96.485 +Training time 0:07:45.702914 +Epoch: 22 Average loss: 94.51 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 261) +0/69092 Loss: 91.129 +3200/69092 Loss: 94.574 +6400/69092 Loss: 94.748 +9600/69092 Loss: 93.301 +12800/69092 Loss: 94.842 +16000/69092 Loss: 93.721 +19200/69092 Loss: 96.039 +22400/69092 Loss: 94.234 +25600/69092 Loss: 94.628 +28800/69092 Loss: 94.492 +32000/69092 Loss: 94.608 +35200/69092 Loss: 94.682 +38400/69092 Loss: 94.898 +41600/69092 Loss: 94.128 +44800/69092 Loss: 93.289 +48000/69092 Loss: 94.227 +51200/69092 Loss: 95.205 +54400/69092 Loss: 95.398 +57600/69092 Loss: 93.717 +60800/69092 Loss: 94.521 +64000/69092 Loss: 94.331 +67200/69092 Loss: 94.194 +Training time 0:07:37.814544 +Epoch: 23 Average loss: 94.48 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 262) +0/69092 Loss: 100.275 +3200/69092 Loss: 93.559 +6400/69092 Loss: 93.706 +9600/69092 Loss: 94.569 +12800/69092 Loss: 94.809 +16000/69092 Loss: 93.883 +19200/69092 Loss: 95.446 +22400/69092 Loss: 94.435 +25600/69092 Loss: 93.197 +28800/69092 Loss: 94.484 +32000/69092 Loss: 93.676 +35200/69092 Loss: 95.907 +38400/69092 Loss: 95.711 +41600/69092 Loss: 94.039 +44800/69092 Loss: 95.335 +48000/69092 Loss: 94.150 +51200/69092 Loss: 93.520 +54400/69092 Loss: 94.110 +57600/69092 Loss: 93.849 +60800/69092 Loss: 94.094 +64000/69092 Loss: 94.586 +67200/69092 Loss: 93.006 +Training time 0:07:42.206996 +Epoch: 24 Average loss: 94.30 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 263) +0/69092 Loss: 88.416 +3200/69092 Loss: 94.540 +6400/69092 Loss: 94.009 +9600/69092 Loss: 93.845 +12800/69092 Loss: 94.440 +16000/69092 Loss: 95.946 +19200/69092 Loss: 94.573 +22400/69092 Loss: 94.471 +25600/69092 Loss: 94.279 +28800/69092 Loss: 93.898 +32000/69092 Loss: 94.539 +35200/69092 Loss: 94.141 +38400/69092 Loss: 94.315 +41600/69092 Loss: 94.690 +44800/69092 Loss: 95.521 +48000/69092 Loss: 94.180 +51200/69092 Loss: 94.408 +54400/69092 Loss: 94.011 +57600/69092 Loss: 93.877 +60800/69092 Loss: 95.488 +64000/69092 Loss: 94.899 +67200/69092 Loss: 95.261 +Training time 0:07:40.103666 +Epoch: 25 Average loss: 94.52 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 264) +0/69092 Loss: 85.111 +3200/69092 Loss: 94.827 +6400/69092 Loss: 94.699 +9600/69092 Loss: 96.204 +12800/69092 Loss: 94.342 +16000/69092 Loss: 93.345 +19200/69092 Loss: 94.901 +22400/69092 Loss: 95.114 +25600/69092 Loss: 94.180 +28800/69092 Loss: 95.085 +32000/69092 Loss: 93.044 +35200/69092 Loss: 92.472 +38400/69092 Loss: 95.928 +41600/69092 Loss: 93.398 +44800/69092 Loss: 93.856 +48000/69092 Loss: 94.865 +51200/69092 Loss: 94.287 +54400/69092 Loss: 94.606 +57600/69092 Loss: 93.693 +60800/69092 Loss: 95.496 +64000/69092 Loss: 94.321 +67200/69092 Loss: 93.808 +Training time 0:07:34.503459 +Epoch: 26 Average loss: 94.38 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 265) +0/69092 Loss: 88.303 +3200/69092 Loss: 95.841 +6400/69092 Loss: 93.394 +9600/69092 Loss: 95.016 +12800/69092 Loss: 94.989 +16000/69092 Loss: 94.769 +19200/69092 Loss: 93.279 +22400/69092 Loss: 94.236 +25600/69092 Loss: 95.781 +28800/69092 Loss: 95.774 +32000/69092 Loss: 93.033 +35200/69092 Loss: 92.807 +38400/69092 Loss: 94.712 +41600/69092 Loss: 94.715 +44800/69092 Loss: 95.253 +48000/69092 Loss: 94.960 +51200/69092 Loss: 94.723 +54400/69092 Loss: 93.044 +57600/69092 Loss: 93.961 +60800/69092 Loss: 94.069 +64000/69092 Loss: 94.524 +67200/69092 Loss: 93.462 +Training time 0:07:39.386642 +Epoch: 27 Average loss: 94.38 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 266) +0/69092 Loss: 94.946 +3200/69092 Loss: 95.531 +6400/69092 Loss: 93.108 +9600/69092 Loss: 93.064 +12800/69092 Loss: 93.868 +16000/69092 Loss: 94.985 +19200/69092 Loss: 93.235 +22400/69092 Loss: 96.176 +25600/69092 Loss: 93.569 +28800/69092 Loss: 93.586 +32000/69092 Loss: 93.847 +35200/69092 Loss: 95.598 +38400/69092 Loss: 94.281 +41600/69092 Loss: 93.923 +44800/69092 Loss: 92.984 +48000/69092 Loss: 94.620 +51200/69092 Loss: 93.596 +54400/69092 Loss: 94.434 +57600/69092 Loss: 93.549 +60800/69092 Loss: 95.144 +64000/69092 Loss: 94.602 +67200/69092 Loss: 93.778 +Training time 0:07:38.999495 +Epoch: 28 Average loss: 94.20 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 267) +0/69092 Loss: 101.820 +3200/69092 Loss: 95.849 +6400/69092 Loss: 94.409 +9600/69092 Loss: 94.295 +12800/69092 Loss: 94.057 +16000/69092 Loss: 93.504 +19200/69092 Loss: 94.535 +22400/69092 Loss: 94.866 +25600/69092 Loss: 94.070 +28800/69092 Loss: 94.800 +32000/69092 Loss: 95.176 +35200/69092 Loss: 93.409 +38400/69092 Loss: 94.730 +41600/69092 Loss: 95.053 +44800/69092 Loss: 94.510 +48000/69092 Loss: 93.804 +51200/69092 Loss: 93.469 +54400/69092 Loss: 94.109 +57600/69092 Loss: 93.638 +60800/69092 Loss: 93.365 +64000/69092 Loss: 93.684 +67200/69092 Loss: 93.409 +Training time 0:07:45.171285 +Epoch: 29 Average loss: 94.28 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 268) +0/69092 Loss: 96.377 +3200/69092 Loss: 94.557 +6400/69092 Loss: 94.312 +9600/69092 Loss: 92.308 +12800/69092 Loss: 94.452 +16000/69092 Loss: 93.923 +19200/69092 Loss: 94.038 +22400/69092 Loss: 94.325 +25600/69092 Loss: 94.091 +28800/69092 Loss: 94.862 +32000/69092 Loss: 95.689 +35200/69092 Loss: 95.604 +38400/69092 Loss: 93.382 +41600/69092 Loss: 94.479 +44800/69092 Loss: 93.209 +48000/69092 Loss: 94.666 +51200/69092 Loss: 92.685 +54400/69092 Loss: 95.245 +57600/69092 Loss: 94.772 +60800/69092 Loss: 94.947 +64000/69092 Loss: 93.205 +67200/69092 Loss: 94.040 +Training time 0:07:42.609167 +Epoch: 30 Average loss: 94.33 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 269) +0/69092 Loss: 103.367 +3200/69092 Loss: 94.699 +6400/69092 Loss: 95.023 +9600/69092 Loss: 95.379 +12800/69092 Loss: 94.293 +16000/69092 Loss: 94.712 +19200/69092 Loss: 93.848 +22400/69092 Loss: 93.260 +25600/69092 Loss: 93.806 +28800/69092 Loss: 95.096 +32000/69092 Loss: 95.030 +35200/69092 Loss: 93.984 +38400/69092 Loss: 94.321 +41600/69092 Loss: 94.702 +44800/69092 Loss: 95.071 +48000/69092 Loss: 93.694 +51200/69092 Loss: 94.426 +54400/69092 Loss: 92.545 +57600/69092 Loss: 94.269 +60800/69092 Loss: 94.967 +64000/69092 Loss: 93.812 +67200/69092 Loss: 93.935 +Training time 0:07:43.513473 +Epoch: 31 Average loss: 94.36 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 270) +0/69092 Loss: 86.774 +3200/69092 Loss: 92.884 +6400/69092 Loss: 94.309 +9600/69092 Loss: 94.701 +12800/69092 Loss: 93.789 +16000/69092 Loss: 94.992 +19200/69092 Loss: 94.160 +22400/69092 Loss: 94.172 +25600/69092 Loss: 94.512 +28800/69092 Loss: 94.316 +32000/69092 Loss: 93.107 +35200/69092 Loss: 93.197 +38400/69092 Loss: 94.714 +41600/69092 Loss: 94.126 +44800/69092 Loss: 93.114 +48000/69092 Loss: 93.985 +51200/69092 Loss: 94.220 +54400/69092 Loss: 95.437 +57600/69092 Loss: 94.879 +60800/69092 Loss: 95.131 +64000/69092 Loss: 93.899 +67200/69092 Loss: 93.811 +Training time 0:07:34.525610 +Epoch: 32 Average loss: 94.18 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 271) +0/69092 Loss: 79.082 +3200/69092 Loss: 93.919 +6400/69092 Loss: 92.965 +9600/69092 Loss: 93.827 +12800/69092 Loss: 94.259 +16000/69092 Loss: 93.290 +19200/69092 Loss: 93.047 +22400/69092 Loss: 93.228 +25600/69092 Loss: 94.482 +28800/69092 Loss: 94.434 +32000/69092 Loss: 95.008 +35200/69092 Loss: 94.336 +38400/69092 Loss: 94.221 +41600/69092 Loss: 93.217 +44800/69092 Loss: 94.862 +48000/69092 Loss: 96.200 +51200/69092 Loss: 94.070 +54400/69092 Loss: 95.314 +57600/69092 Loss: 95.655 +60800/69092 Loss: 93.137 +64000/69092 Loss: 94.254 +67200/69092 Loss: 94.553 +Training time 0:07:47.004933 +Epoch: 33 Average loss: 94.22 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 272) +0/69092 Loss: 92.115 +3200/69092 Loss: 94.322 +6400/69092 Loss: 94.053 +9600/69092 Loss: 94.973 +12800/69092 Loss: 94.937 +16000/69092 Loss: 93.713 +19200/69092 Loss: 93.324 +22400/69092 Loss: 94.396 +25600/69092 Loss: 93.361 +28800/69092 Loss: 94.258 +32000/69092 Loss: 94.962 +35200/69092 Loss: 93.783 +38400/69092 Loss: 93.671 +41600/69092 Loss: 95.809 +44800/69092 Loss: 94.409 +48000/69092 Loss: 95.979 +51200/69092 Loss: 93.135 +54400/69092 Loss: 94.423 +57600/69092 Loss: 95.068 +60800/69092 Loss: 94.002 +64000/69092 Loss: 93.866 +67200/69092 Loss: 94.337 +Training time 0:07:54.671753 +Epoch: 34 Average loss: 94.34 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 273) +0/69092 Loss: 88.563 +3200/69092 Loss: 95.024 +6400/69092 Loss: 93.990 +9600/69092 Loss: 94.811 +12800/69092 Loss: 94.545 +16000/69092 Loss: 94.481 +19200/69092 Loss: 94.026 +22400/69092 Loss: 94.225 +25600/69092 Loss: 93.618 +28800/69092 Loss: 94.643 +32000/69092 Loss: 93.735 +35200/69092 Loss: 94.191 +38400/69092 Loss: 93.790 +41600/69092 Loss: 94.506 +44800/69092 Loss: 94.931 +48000/69092 Loss: 94.278 +51200/69092 Loss: 92.736 +54400/69092 Loss: 92.957 +57600/69092 Loss: 95.095 +60800/69092 Loss: 96.386 +64000/69092 Loss: 93.339 +67200/69092 Loss: 93.537 +Training time 0:07:40.178223 +Epoch: 35 Average loss: 94.24 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 274) +0/69092 Loss: 87.948 +3200/69092 Loss: 95.042 +6400/69092 Loss: 94.420 +9600/69092 Loss: 94.592 +12800/69092 Loss: 93.621 +16000/69092 Loss: 94.335 +19200/69092 Loss: 92.897 +22400/69092 Loss: 95.980 +25600/69092 Loss: 94.108 +28800/69092 Loss: 95.702 +32000/69092 Loss: 94.126 +35200/69092 Loss: 94.676 +38400/69092 Loss: 93.227 +41600/69092 Loss: 93.622 +44800/69092 Loss: 93.259 +48000/69092 Loss: 93.458 +51200/69092 Loss: 93.247 +54400/69092 Loss: 94.638 +57600/69092 Loss: 94.964 +60800/69092 Loss: 94.570 +64000/69092 Loss: 93.521 +67200/69092 Loss: 92.679 +Training time 0:07:46.574816 +Epoch: 36 Average loss: 94.20 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 275) +0/69092 Loss: 98.575 +3200/69092 Loss: 94.629 +6400/69092 Loss: 94.557 +9600/69092 Loss: 95.206 +12800/69092 Loss: 93.700 +16000/69092 Loss: 92.573 +19200/69092 Loss: 94.727 +22400/69092 Loss: 94.275 +25600/69092 Loss: 93.691 +28800/69092 Loss: 93.505 +32000/69092 Loss: 94.334 +35200/69092 Loss: 95.038 +38400/69092 Loss: 93.536 +41600/69092 Loss: 93.998 +44800/69092 Loss: 95.422 +48000/69092 Loss: 92.996 +51200/69092 Loss: 93.841 +54400/69092 Loss: 94.419 +57600/69092 Loss: 94.256 +60800/69092 Loss: 94.062 +64000/69092 Loss: 94.341 +67200/69092 Loss: 93.731 +Training time 0:07:36.066927 +Epoch: 37 Average loss: 94.15 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 276) +0/69092 Loss: 106.238 +3200/69092 Loss: 93.937 +6400/69092 Loss: 92.894 +9600/69092 Loss: 94.920 +12800/69092 Loss: 94.130 +16000/69092 Loss: 93.202 +19200/69092 Loss: 94.408 +22400/69092 Loss: 94.472 +25600/69092 Loss: 94.444 +28800/69092 Loss: 93.888 +32000/69092 Loss: 95.488 +35200/69092 Loss: 95.220 +38400/69092 Loss: 94.451 +41600/69092 Loss: 95.058 +44800/69092 Loss: 93.607 +48000/69092 Loss: 94.011 +51200/69092 Loss: 94.617 +54400/69092 Loss: 94.563 +57600/69092 Loss: 94.011 +60800/69092 Loss: 95.383 +64000/69092 Loss: 95.589 +67200/69092 Loss: 92.824 +Training time 0:07:44.982394 +Epoch: 38 Average loss: 94.31 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 277) +0/69092 Loss: 96.650 +3200/69092 Loss: 93.978 +6400/69092 Loss: 94.503 +9600/69092 Loss: 94.344 +12800/69092 Loss: 94.054 +16000/69092 Loss: 93.252 +19200/69092 Loss: 93.878 +22400/69092 Loss: 94.215 +25600/69092 Loss: 92.709 +28800/69092 Loss: 93.490 +32000/69092 Loss: 95.771 +35200/69092 Loss: 96.025 +38400/69092 Loss: 93.638 +41600/69092 Loss: 94.336 +44800/69092 Loss: 93.556 +48000/69092 Loss: 94.772 +51200/69092 Loss: 94.093 +54400/69092 Loss: 94.135 +57600/69092 Loss: 93.590 +60800/69092 Loss: 95.128 +64000/69092 Loss: 93.634 +67200/69092 Loss: 94.064 +Training time 0:07:59.348702 +Epoch: 39 Average loss: 94.17 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 278) +0/69092 Loss: 85.455 +3200/69092 Loss: 94.549 +6400/69092 Loss: 93.275 +9600/69092 Loss: 94.619 +12800/69092 Loss: 94.962 +16000/69092 Loss: 95.033 +19200/69092 Loss: 93.486 +22400/69092 Loss: 94.033 +25600/69092 Loss: 93.805 +28800/69092 Loss: 93.607 +32000/69092 Loss: 94.808 +35200/69092 Loss: 94.589 +38400/69092 Loss: 93.609 +41600/69092 Loss: 94.311 +44800/69092 Loss: 96.463 +48000/69092 Loss: 94.146 +51200/69092 Loss: 93.707 +54400/69092 Loss: 94.317 +57600/69092 Loss: 93.891 +60800/69092 Loss: 95.680 +64000/69092 Loss: 94.708 +67200/69092 Loss: 93.152 +Training time 0:07:41.063716 +Epoch: 40 Average loss: 94.34 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last' (iter 279) +0/69092 Loss: 95.901 +3200/69092 Loss: 93.413 +6400/69092 Loss: 94.176 diff --git a/OAR.2071927.stderr b/OAR.2071927.stderr new file mode 100644 index 0000000000000000000000000000000000000000..a9c628a29c58ebc65df754dd5dfbbd99e2282b5f --- /dev/null +++ b/OAR.2071927.stderr @@ -0,0 +1,2 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071927.stdout b/OAR.2071927.stdout new file mode 100644 index 0000000000000000000000000000000000000000..e01dbf8014c1032b3fb423e87ad6e9e68e8f62e7 --- /dev/null +++ b/OAR.2071927.stdout @@ -0,0 +1,1138 @@ +Namespace(batch_size=64, beta=None, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='VAE_bs_64_ls_50', gpu_devices=[0, 1], is_beta_VAE=False, latent_spec_cont=50, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +Tesla K80 +Tesla K80 +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=100, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=50, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 796135 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last (iter 231)' +0/69092 Loss: 91.954 +3200/69092 Loss: 94.564 +6400/69092 Loss: 93.376 +9600/69092 Loss: 93.990 +12800/69092 Loss: 92.945 +16000/69092 Loss: 95.209 +19200/69092 Loss: 93.579 +22400/69092 Loss: 94.560 +25600/69092 Loss: 94.213 +28800/69092 Loss: 94.965 +32000/69092 Loss: 94.429 +35200/69092 Loss: 93.918 +38400/69092 Loss: 95.537 +41600/69092 Loss: 93.524 +44800/69092 Loss: 96.094 +48000/69092 Loss: 93.614 +51200/69092 Loss: 93.349 +54400/69092 Loss: 94.956 +57600/69092 Loss: 94.136 +60800/69092 Loss: 95.075 +64000/69092 Loss: 95.717 +67200/69092 Loss: 96.182 +Training time 0:06:56.489005 +Epoch: 1 Average loss: 94.52 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 232) +0/69092 Loss: 90.646 +3200/69092 Loss: 94.751 +6400/69092 Loss: 94.286 +9600/69092 Loss: 91.797 +12800/69092 Loss: 94.522 +16000/69092 Loss: 95.491 +19200/69092 Loss: 93.723 +22400/69092 Loss: 94.257 +25600/69092 Loss: 94.333 +28800/69092 Loss: 93.729 +32000/69092 Loss: 94.455 +35200/69092 Loss: 94.175 +38400/69092 Loss: 93.918 +41600/69092 Loss: 92.870 +44800/69092 Loss: 93.593 +48000/69092 Loss: 95.452 +51200/69092 Loss: 94.741 +54400/69092 Loss: 94.695 +57600/69092 Loss: 94.203 +60800/69092 Loss: 95.056 +64000/69092 Loss: 93.085 +67200/69092 Loss: 96.027 +Training time 0:06:57.405771 +Epoch: 2 Average loss: 94.26 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 233) +0/69092 Loss: 94.921 +3200/69092 Loss: 94.141 +6400/69092 Loss: 94.381 +9600/69092 Loss: 94.649 +12800/69092 Loss: 94.833 +16000/69092 Loss: 93.805 +19200/69092 Loss: 93.936 +22400/69092 Loss: 94.193 +25600/69092 Loss: 92.998 +28800/69092 Loss: 95.128 +32000/69092 Loss: 94.053 +35200/69092 Loss: 94.406 +38400/69092 Loss: 94.352 +41600/69092 Loss: 93.951 +44800/69092 Loss: 94.333 +48000/69092 Loss: 93.812 +51200/69092 Loss: 94.470 +54400/69092 Loss: 93.909 +57600/69092 Loss: 93.817 +60800/69092 Loss: 94.382 +64000/69092 Loss: 95.342 +67200/69092 Loss: 93.656 +Training time 0:07:06.982462 +Epoch: 3 Average loss: 94.21 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 234) +0/69092 Loss: 91.938 +3200/69092 Loss: 93.216 +6400/69092 Loss: 92.719 +9600/69092 Loss: 94.298 +12800/69092 Loss: 95.344 +16000/69092 Loss: 94.878 +19200/69092 Loss: 93.279 +22400/69092 Loss: 94.137 +25600/69092 Loss: 95.604 +28800/69092 Loss: 94.071 +32000/69092 Loss: 93.699 +35200/69092 Loss: 92.445 +38400/69092 Loss: 94.563 +41600/69092 Loss: 94.948 +44800/69092 Loss: 95.028 +48000/69092 Loss: 95.213 +51200/69092 Loss: 93.304 +54400/69092 Loss: 93.944 +57600/69092 Loss: 94.320 +60800/69092 Loss: 93.398 +64000/69092 Loss: 95.316 +67200/69092 Loss: 94.405 +Training time 0:07:02.932819 +Epoch: 4 Average loss: 94.19 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 235) +0/69092 Loss: 96.773 +3200/69092 Loss: 94.703 +6400/69092 Loss: 94.128 +9600/69092 Loss: 94.920 +12800/69092 Loss: 94.056 +16000/69092 Loss: 94.226 +19200/69092 Loss: 93.954 +22400/69092 Loss: 93.243 +25600/69092 Loss: 93.155 +28800/69092 Loss: 94.354 +32000/69092 Loss: 94.070 +35200/69092 Loss: 94.949 +38400/69092 Loss: 95.588 +41600/69092 Loss: 94.516 +44800/69092 Loss: 95.082 +48000/69092 Loss: 94.117 +51200/69092 Loss: 94.425 +54400/69092 Loss: 94.776 +57600/69092 Loss: 92.593 +60800/69092 Loss: 94.033 +64000/69092 Loss: 95.372 +67200/69092 Loss: 92.841 +Training time 0:07:00.806582 +Epoch: 5 Average loss: 94.26 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 236) +0/69092 Loss: 87.616 +3200/69092 Loss: 94.689 +6400/69092 Loss: 94.062 +9600/69092 Loss: 91.786 +12800/69092 Loss: 93.490 +16000/69092 Loss: 94.050 +19200/69092 Loss: 94.785 +22400/69092 Loss: 93.769 +25600/69092 Loss: 93.951 +28800/69092 Loss: 94.472 +32000/69092 Loss: 94.971 +35200/69092 Loss: 94.899 +38400/69092 Loss: 94.376 +41600/69092 Loss: 94.080 +44800/69092 Loss: 95.883 +48000/69092 Loss: 92.942 +51200/69092 Loss: 95.381 +54400/69092 Loss: 93.424 +57600/69092 Loss: 94.124 +60800/69092 Loss: 93.805 +64000/69092 Loss: 93.311 +67200/69092 Loss: 93.765 +Training time 0:06:54.264797 +Epoch: 6 Average loss: 94.11 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 237) +0/69092 Loss: 100.211 +3200/69092 Loss: 93.414 +6400/69092 Loss: 92.689 +9600/69092 Loss: 93.568 +12800/69092 Loss: 93.602 +16000/69092 Loss: 93.310 +19200/69092 Loss: 93.892 +22400/69092 Loss: 95.053 +25600/69092 Loss: 95.893 +28800/69092 Loss: 93.569 +32000/69092 Loss: 95.481 +35200/69092 Loss: 94.790 +38400/69092 Loss: 93.807 +41600/69092 Loss: 93.254 +44800/69092 Loss: 92.942 +48000/69092 Loss: 94.108 +51200/69092 Loss: 92.903 +54400/69092 Loss: 95.305 +57600/69092 Loss: 94.821 +60800/69092 Loss: 93.781 +64000/69092 Loss: 94.486 +67200/69092 Loss: 93.924 +Training time 0:06:57.509953 +Epoch: 7 Average loss: 94.09 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 238) +0/69092 Loss: 89.657 +3200/69092 Loss: 94.601 +6400/69092 Loss: 94.332 +9600/69092 Loss: 94.131 +12800/69092 Loss: 94.798 +16000/69092 Loss: 94.570 +19200/69092 Loss: 94.333 +22400/69092 Loss: 93.240 +25600/69092 Loss: 94.566 +28800/69092 Loss: 93.813 +32000/69092 Loss: 93.390 +35200/69092 Loss: 94.517 +38400/69092 Loss: 95.254 +41600/69092 Loss: 94.114 +44800/69092 Loss: 92.701 +48000/69092 Loss: 92.974 +51200/69092 Loss: 93.490 +54400/69092 Loss: 93.963 +57600/69092 Loss: 95.039 +60800/69092 Loss: 92.638 +64000/69092 Loss: 92.846 +67200/69092 Loss: 94.671 +Training time 0:07:03.287176 +Epoch: 8 Average loss: 94.03 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 239) +0/69092 Loss: 92.391 +3200/69092 Loss: 94.234 +6400/69092 Loss: 92.268 +9600/69092 Loss: 93.573 +12800/69092 Loss: 95.393 +16000/69092 Loss: 93.456 +19200/69092 Loss: 94.691 +22400/69092 Loss: 94.237 +25600/69092 Loss: 92.955 +28800/69092 Loss: 93.153 +32000/69092 Loss: 94.726 +35200/69092 Loss: 93.210 +38400/69092 Loss: 94.574 +41600/69092 Loss: 95.294 +44800/69092 Loss: 93.646 +48000/69092 Loss: 94.459 +51200/69092 Loss: 94.643 +54400/69092 Loss: 94.692 +57600/69092 Loss: 94.198 +60800/69092 Loss: 93.658 +64000/69092 Loss: 96.210 +67200/69092 Loss: 94.079 +Training time 0:07:04.094575 +Epoch: 9 Average loss: 94.21 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 240) +0/69092 Loss: 93.099 +3200/69092 Loss: 94.090 +6400/69092 Loss: 94.101 +9600/69092 Loss: 94.080 +12800/69092 Loss: 94.538 +16000/69092 Loss: 93.421 +19200/69092 Loss: 95.078 +22400/69092 Loss: 93.787 +25600/69092 Loss: 94.570 +28800/69092 Loss: 94.530 +32000/69092 Loss: 95.694 +35200/69092 Loss: 93.178 +38400/69092 Loss: 93.827 +41600/69092 Loss: 93.383 +44800/69092 Loss: 94.239 +48000/69092 Loss: 95.279 +51200/69092 Loss: 92.860 +54400/69092 Loss: 93.146 +57600/69092 Loss: 94.589 +60800/69092 Loss: 92.408 +64000/69092 Loss: 93.531 +67200/69092 Loss: 94.841 +Training time 0:07:08.030146 +Epoch: 10 Average loss: 94.05 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 241) +0/69092 Loss: 86.579 +3200/69092 Loss: 94.167 +6400/69092 Loss: 95.532 +9600/69092 Loss: 93.748 +12800/69092 Loss: 95.083 +16000/69092 Loss: 92.132 +19200/69092 Loss: 93.660 +22400/69092 Loss: 94.725 +25600/69092 Loss: 94.670 +28800/69092 Loss: 93.832 +32000/69092 Loss: 93.552 +35200/69092 Loss: 93.077 +38400/69092 Loss: 94.118 +41600/69092 Loss: 92.951 +44800/69092 Loss: 94.034 +48000/69092 Loss: 93.996 +51200/69092 Loss: 93.599 +54400/69092 Loss: 94.383 +57600/69092 Loss: 93.775 +60800/69092 Loss: 93.624 +64000/69092 Loss: 94.947 +67200/69092 Loss: 93.503 +Training time 0:06:58.476348 +Epoch: 11 Average loss: 93.96 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 242) +0/69092 Loss: 89.013 +3200/69092 Loss: 94.977 +6400/69092 Loss: 94.617 +9600/69092 Loss: 93.343 +12800/69092 Loss: 95.139 +16000/69092 Loss: 94.109 +19200/69092 Loss: 94.013 +22400/69092 Loss: 93.279 +25600/69092 Loss: 93.785 +28800/69092 Loss: 93.132 +32000/69092 Loss: 94.397 +35200/69092 Loss: 93.784 +38400/69092 Loss: 93.331 +41600/69092 Loss: 94.190 +44800/69092 Loss: 92.857 +48000/69092 Loss: 93.608 +51200/69092 Loss: 94.303 +54400/69092 Loss: 95.162 +57600/69092 Loss: 94.672 +60800/69092 Loss: 94.972 +64000/69092 Loss: 94.054 +67200/69092 Loss: 93.682 +Training time 0:07:00.233406 +Epoch: 12 Average loss: 94.04 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 243) +0/69092 Loss: 90.844 +3200/69092 Loss: 92.882 +6400/69092 Loss: 92.729 +9600/69092 Loss: 93.199 +12800/69092 Loss: 94.173 +16000/69092 Loss: 93.976 +19200/69092 Loss: 96.533 +22400/69092 Loss: 94.087 +25600/69092 Loss: 93.430 +28800/69092 Loss: 93.224 +32000/69092 Loss: 94.892 +35200/69092 Loss: 94.992 +38400/69092 Loss: 93.290 +41600/69092 Loss: 94.268 +44800/69092 Loss: 92.762 +48000/69092 Loss: 94.361 +51200/69092 Loss: 94.622 +54400/69092 Loss: 94.934 +57600/69092 Loss: 94.316 +60800/69092 Loss: 94.388 +64000/69092 Loss: 93.523 +67200/69092 Loss: 93.511 +Training time 0:06:52.749979 +Epoch: 13 Average loss: 94.01 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 244) +0/69092 Loss: 90.390 +3200/69092 Loss: 93.877 +6400/69092 Loss: 94.948 +9600/69092 Loss: 94.875 +12800/69092 Loss: 93.920 +16000/69092 Loss: 93.379 +19200/69092 Loss: 95.375 +22400/69092 Loss: 93.995 +25600/69092 Loss: 93.755 +28800/69092 Loss: 93.181 +32000/69092 Loss: 93.946 +35200/69092 Loss: 93.749 +38400/69092 Loss: 94.093 +41600/69092 Loss: 93.199 +44800/69092 Loss: 94.266 +48000/69092 Loss: 94.846 +51200/69092 Loss: 93.399 +54400/69092 Loss: 93.975 +57600/69092 Loss: 93.662 +60800/69092 Loss: 93.924 +64000/69092 Loss: 94.626 +67200/69092 Loss: 94.234 +Training time 0:06:59.646990 +Epoch: 14 Average loss: 94.11 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 245) +0/69092 Loss: 93.325 +3200/69092 Loss: 94.246 +6400/69092 Loss: 94.331 +9600/69092 Loss: 93.115 +12800/69092 Loss: 94.181 +16000/69092 Loss: 94.444 +19200/69092 Loss: 93.339 +22400/69092 Loss: 94.276 +25600/69092 Loss: 93.336 +28800/69092 Loss: 94.324 +32000/69092 Loss: 93.356 +35200/69092 Loss: 94.727 +38400/69092 Loss: 93.308 +41600/69092 Loss: 93.842 +44800/69092 Loss: 94.296 +48000/69092 Loss: 93.447 +51200/69092 Loss: 94.448 +54400/69092 Loss: 92.919 +57600/69092 Loss: 94.079 +60800/69092 Loss: 94.296 +64000/69092 Loss: 94.785 +67200/69092 Loss: 93.389 +Training time 0:07:05.580641 +Epoch: 15 Average loss: 93.92 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 246) +0/69092 Loss: 96.399 +3200/69092 Loss: 93.748 +6400/69092 Loss: 92.157 +9600/69092 Loss: 93.750 +12800/69092 Loss: 94.277 +16000/69092 Loss: 93.989 +19200/69092 Loss: 92.692 +22400/69092 Loss: 94.263 +25600/69092 Loss: 94.895 +28800/69092 Loss: 94.126 +32000/69092 Loss: 93.989 +35200/69092 Loss: 92.535 +38400/69092 Loss: 93.822 +41600/69092 Loss: 94.990 +44800/69092 Loss: 94.324 +48000/69092 Loss: 93.366 +51200/69092 Loss: 94.301 +54400/69092 Loss: 94.812 +57600/69092 Loss: 93.258 +60800/69092 Loss: 93.779 +64000/69092 Loss: 95.196 +67200/69092 Loss: 93.353 +Training time 0:07:05.686162 +Epoch: 16 Average loss: 93.94 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 247) +0/69092 Loss: 91.840 +3200/69092 Loss: 93.470 +6400/69092 Loss: 93.651 +9600/69092 Loss: 93.924 +12800/69092 Loss: 93.208 +16000/69092 Loss: 92.839 +19200/69092 Loss: 93.419 +22400/69092 Loss: 92.472 +25600/69092 Loss: 95.862 +28800/69092 Loss: 93.224 +32000/69092 Loss: 93.050 +35200/69092 Loss: 95.159 +38400/69092 Loss: 94.040 +41600/69092 Loss: 95.416 +44800/69092 Loss: 94.545 +48000/69092 Loss: 94.206 +51200/69092 Loss: 92.688 +54400/69092 Loss: 93.768 +57600/69092 Loss: 95.083 +60800/69092 Loss: 94.196 +64000/69092 Loss: 93.584 +67200/69092 Loss: 94.232 +Training time 0:07:04.703433 +Epoch: 17 Average loss: 93.92 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 248) +0/69092 Loss: 94.937 +3200/69092 Loss: 93.893 +6400/69092 Loss: 93.498 +9600/69092 Loss: 94.041 +12800/69092 Loss: 93.068 +16000/69092 Loss: 93.918 +19200/69092 Loss: 93.568 +22400/69092 Loss: 93.672 +25600/69092 Loss: 93.894 +28800/69092 Loss: 95.021 +32000/69092 Loss: 93.935 +35200/69092 Loss: 95.891 +38400/69092 Loss: 93.123 +41600/69092 Loss: 93.906 +44800/69092 Loss: 93.249 +48000/69092 Loss: 94.828 +51200/69092 Loss: 94.011 +54400/69092 Loss: 94.052 +57600/69092 Loss: 94.746 +60800/69092 Loss: 93.490 +64000/69092 Loss: 93.899 +67200/69092 Loss: 94.776 +Training time 0:07:03.482239 +Epoch: 18 Average loss: 94.02 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 249) +0/69092 Loss: 89.131 +3200/69092 Loss: 92.850 +6400/69092 Loss: 92.899 +9600/69092 Loss: 95.201 +12800/69092 Loss: 93.583 +16000/69092 Loss: 95.122 +19200/69092 Loss: 92.846 +22400/69092 Loss: 93.398 +25600/69092 Loss: 94.351 +28800/69092 Loss: 93.776 +32000/69092 Loss: 94.067 +35200/69092 Loss: 94.017 +38400/69092 Loss: 93.039 +41600/69092 Loss: 93.570 +44800/69092 Loss: 94.194 +48000/69092 Loss: 94.586 +51200/69092 Loss: 95.369 +54400/69092 Loss: 93.713 +57600/69092 Loss: 93.555 +60800/69092 Loss: 93.583 +64000/69092 Loss: 94.926 +67200/69092 Loss: 94.087 +Training time 0:07:04.719803 +Epoch: 19 Average loss: 94.00 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 250) +0/69092 Loss: 91.810 +3200/69092 Loss: 93.853 +6400/69092 Loss: 93.550 +9600/69092 Loss: 94.183 +12800/69092 Loss: 94.360 +16000/69092 Loss: 95.047 +19200/69092 Loss: 92.740 +22400/69092 Loss: 93.590 +25600/69092 Loss: 93.564 +28800/69092 Loss: 93.260 +32000/69092 Loss: 94.822 +35200/69092 Loss: 93.956 +38400/69092 Loss: 94.448 +41600/69092 Loss: 94.269 +44800/69092 Loss: 92.516 +48000/69092 Loss: 94.475 +51200/69092 Loss: 95.246 +54400/69092 Loss: 93.080 +57600/69092 Loss: 93.654 +60800/69092 Loss: 93.237 +64000/69092 Loss: 91.936 +67200/69092 Loss: 95.163 +Training time 0:06:56.653846 +Epoch: 20 Average loss: 93.88 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 251) +0/69092 Loss: 91.851 +3200/69092 Loss: 93.537 +6400/69092 Loss: 94.055 +9600/69092 Loss: 93.282 +12800/69092 Loss: 92.024 +16000/69092 Loss: 93.725 +19200/69092 Loss: 92.303 +22400/69092 Loss: 93.757 +25600/69092 Loss: 94.509 +28800/69092 Loss: 95.277 +32000/69092 Loss: 92.789 +35200/69092 Loss: 93.005 +38400/69092 Loss: 94.144 +41600/69092 Loss: 94.650 +44800/69092 Loss: 93.839 +48000/69092 Loss: 93.994 +51200/69092 Loss: 92.768 +54400/69092 Loss: 95.125 +57600/69092 Loss: 93.862 +60800/69092 Loss: 93.129 +64000/69092 Loss: 93.729 +67200/69092 Loss: 95.086 +Training time 0:06:56.394174 +Epoch: 21 Average loss: 93.79 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 252) +0/69092 Loss: 92.413 +3200/69092 Loss: 92.976 +6400/69092 Loss: 93.642 +9600/69092 Loss: 93.068 +12800/69092 Loss: 95.438 +16000/69092 Loss: 93.382 +19200/69092 Loss: 93.966 +22400/69092 Loss: 92.135 +25600/69092 Loss: 94.228 +28800/69092 Loss: 92.834 +32000/69092 Loss: 93.703 +35200/69092 Loss: 95.286 +38400/69092 Loss: 94.118 +41600/69092 Loss: 93.763 +44800/69092 Loss: 94.233 +48000/69092 Loss: 94.543 +51200/69092 Loss: 95.497 +54400/69092 Loss: 92.438 +57600/69092 Loss: 93.232 +60800/69092 Loss: 94.590 +64000/69092 Loss: 93.731 +67200/69092 Loss: 93.416 +Training time 0:07:04.982861 +Epoch: 22 Average loss: 93.80 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 253) +0/69092 Loss: 98.044 +3200/69092 Loss: 94.214 +6400/69092 Loss: 94.709 +9600/69092 Loss: 92.819 +12800/69092 Loss: 94.416 +16000/69092 Loss: 92.884 +19200/69092 Loss: 93.711 +22400/69092 Loss: 94.797 +25600/69092 Loss: 93.724 +28800/69092 Loss: 91.769 +32000/69092 Loss: 94.743 +35200/69092 Loss: 93.565 +38400/69092 Loss: 94.076 +41600/69092 Loss: 95.068 +44800/69092 Loss: 93.737 +48000/69092 Loss: 94.523 +51200/69092 Loss: 94.158 +54400/69092 Loss: 93.901 +57600/69092 Loss: 93.605 +60800/69092 Loss: 94.148 +64000/69092 Loss: 94.700 +67200/69092 Loss: 93.275 +Training time 0:07:01.573592 +Epoch: 23 Average loss: 93.94 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 254) +0/69092 Loss: 91.237 +3200/69092 Loss: 94.671 +6400/69092 Loss: 93.024 +9600/69092 Loss: 94.478 +12800/69092 Loss: 93.864 +16000/69092 Loss: 93.331 +19200/69092 Loss: 92.649 +22400/69092 Loss: 94.202 +25600/69092 Loss: 95.006 +28800/69092 Loss: 94.293 +32000/69092 Loss: 95.058 +35200/69092 Loss: 94.177 +38400/69092 Loss: 93.300 +41600/69092 Loss: 95.435 +44800/69092 Loss: 94.899 +48000/69092 Loss: 92.462 +51200/69092 Loss: 93.426 +54400/69092 Loss: 92.618 +57600/69092 Loss: 92.562 +60800/69092 Loss: 93.392 +64000/69092 Loss: 93.695 +67200/69092 Loss: 94.188 +Training time 0:07:04.783166 +Epoch: 24 Average loss: 93.82 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 255) +0/69092 Loss: 93.984 +3200/69092 Loss: 92.581 +6400/69092 Loss: 94.216 +9600/69092 Loss: 93.562 +12800/69092 Loss: 93.957 +16000/69092 Loss: 95.314 +19200/69092 Loss: 93.697 +22400/69092 Loss: 94.224 +25600/69092 Loss: 93.041 +28800/69092 Loss: 94.212 +32000/69092 Loss: 93.382 +35200/69092 Loss: 94.075 +38400/69092 Loss: 93.619 +41600/69092 Loss: 92.547 +44800/69092 Loss: 95.178 +48000/69092 Loss: 95.023 +51200/69092 Loss: 94.718 +54400/69092 Loss: 93.548 +57600/69092 Loss: 94.401 +60800/69092 Loss: 92.869 +64000/69092 Loss: 94.009 +67200/69092 Loss: 92.415 +Training time 0:07:06.273061 +Epoch: 25 Average loss: 93.83 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 256) +0/69092 Loss: 96.946 +3200/69092 Loss: 93.795 +6400/69092 Loss: 94.650 +9600/69092 Loss: 93.176 +12800/69092 Loss: 94.644 +16000/69092 Loss: 94.119 +19200/69092 Loss: 93.164 +22400/69092 Loss: 93.211 +25600/69092 Loss: 95.226 +28800/69092 Loss: 93.395 +32000/69092 Loss: 93.292 +35200/69092 Loss: 95.615 +38400/69092 Loss: 93.261 +41600/69092 Loss: 94.393 +44800/69092 Loss: 92.232 +48000/69092 Loss: 93.041 +51200/69092 Loss: 92.723 +54400/69092 Loss: 95.004 +57600/69092 Loss: 94.116 +60800/69092 Loss: 94.583 +64000/69092 Loss: 92.882 +67200/69092 Loss: 93.180 +Training time 0:07:00.351770 +Epoch: 26 Average loss: 93.85 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 257) +0/69092 Loss: 96.043 +3200/69092 Loss: 93.766 +6400/69092 Loss: 94.551 +9600/69092 Loss: 92.584 +12800/69092 Loss: 94.161 +16000/69092 Loss: 93.527 +19200/69092 Loss: 92.728 +22400/69092 Loss: 94.186 +25600/69092 Loss: 93.800 +28800/69092 Loss: 93.250 +32000/69092 Loss: 95.470 +35200/69092 Loss: 92.976 +38400/69092 Loss: 95.473 +41600/69092 Loss: 93.914 +44800/69092 Loss: 94.419 +48000/69092 Loss: 92.285 +51200/69092 Loss: 94.074 +54400/69092 Loss: 93.707 +57600/69092 Loss: 93.070 +60800/69092 Loss: 93.295 +64000/69092 Loss: 94.360 +67200/69092 Loss: 94.939 +Training time 0:06:57.667458 +Epoch: 27 Average loss: 93.83 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 258) +0/69092 Loss: 87.967 +3200/69092 Loss: 93.106 +6400/69092 Loss: 94.031 +9600/69092 Loss: 92.841 +12800/69092 Loss: 94.685 +16000/69092 Loss: 93.360 +19200/69092 Loss: 94.858 +22400/69092 Loss: 93.427 +25600/69092 Loss: 93.627 +28800/69092 Loss: 94.235 +32000/69092 Loss: 94.440 +35200/69092 Loss: 94.991 +38400/69092 Loss: 91.689 +41600/69092 Loss: 92.641 +44800/69092 Loss: 93.826 +48000/69092 Loss: 95.376 +51200/69092 Loss: 91.718 +54400/69092 Loss: 94.108 +57600/69092 Loss: 93.994 +60800/69092 Loss: 94.924 +64000/69092 Loss: 94.465 +67200/69092 Loss: 92.952 +Training time 0:07:07.501697 +Epoch: 28 Average loss: 93.82 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 259) +0/69092 Loss: 96.866 +3200/69092 Loss: 93.988 +6400/69092 Loss: 93.443 +9600/69092 Loss: 92.457 +12800/69092 Loss: 94.463 +16000/69092 Loss: 93.752 +19200/69092 Loss: 93.315 +22400/69092 Loss: 95.187 +25600/69092 Loss: 92.730 +28800/69092 Loss: 93.596 +32000/69092 Loss: 94.775 +35200/69092 Loss: 93.785 +38400/69092 Loss: 94.027 +41600/69092 Loss: 94.259 +44800/69092 Loss: 93.648 +48000/69092 Loss: 92.778 +51200/69092 Loss: 93.170 +54400/69092 Loss: 92.939 +57600/69092 Loss: 93.916 +60800/69092 Loss: 92.893 +64000/69092 Loss: 94.399 +67200/69092 Loss: 95.745 +Training time 0:07:01.176009 +Epoch: 29 Average loss: 93.75 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 260) +0/69092 Loss: 88.893 +3200/69092 Loss: 93.371 +6400/69092 Loss: 93.672 +9600/69092 Loss: 92.901 +12800/69092 Loss: 94.890 +16000/69092 Loss: 93.264 +19200/69092 Loss: 92.714 +22400/69092 Loss: 93.444 +25600/69092 Loss: 94.636 +28800/69092 Loss: 93.342 +32000/69092 Loss: 93.919 +35200/69092 Loss: 93.856 +38400/69092 Loss: 94.341 +41600/69092 Loss: 92.829 +44800/69092 Loss: 94.299 +48000/69092 Loss: 92.778 +51200/69092 Loss: 94.102 +54400/69092 Loss: 93.205 +57600/69092 Loss: 94.638 +60800/69092 Loss: 93.439 +64000/69092 Loss: 94.108 +67200/69092 Loss: 94.949 +Training time 0:07:01.388608 +Epoch: 30 Average loss: 93.79 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 261) +0/69092 Loss: 91.465 +3200/69092 Loss: 93.457 +6400/69092 Loss: 93.957 +9600/69092 Loss: 93.198 +12800/69092 Loss: 92.597 +16000/69092 Loss: 94.662 +19200/69092 Loss: 93.855 +22400/69092 Loss: 92.564 +25600/69092 Loss: 93.856 +28800/69092 Loss: 94.671 +32000/69092 Loss: 93.587 +35200/69092 Loss: 93.843 +38400/69092 Loss: 93.311 +41600/69092 Loss: 94.148 +44800/69092 Loss: 94.342 +48000/69092 Loss: 93.594 +51200/69092 Loss: 93.388 +54400/69092 Loss: 94.381 +57600/69092 Loss: 94.078 +60800/69092 Loss: 95.167 +64000/69092 Loss: 93.073 +67200/69092 Loss: 94.303 +Training time 0:06:54.184314 +Epoch: 31 Average loss: 93.74 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 262) +0/69092 Loss: 93.965 +3200/69092 Loss: 92.997 +6400/69092 Loss: 94.673 +9600/69092 Loss: 93.646 +12800/69092 Loss: 92.445 +16000/69092 Loss: 93.038 +19200/69092 Loss: 94.505 +22400/69092 Loss: 94.032 +25600/69092 Loss: 92.595 +28800/69092 Loss: 93.496 +32000/69092 Loss: 92.535 +35200/69092 Loss: 93.531 +38400/69092 Loss: 95.391 +41600/69092 Loss: 93.826 +44800/69092 Loss: 94.712 +48000/69092 Loss: 94.795 +51200/69092 Loss: 93.624 +54400/69092 Loss: 94.756 +57600/69092 Loss: 92.694 +60800/69092 Loss: 93.902 +64000/69092 Loss: 93.025 +67200/69092 Loss: 92.939 +Training time 0:07:05.999554 +Epoch: 32 Average loss: 93.67 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 263) +0/69092 Loss: 80.107 +3200/69092 Loss: 93.373 +6400/69092 Loss: 93.693 +9600/69092 Loss: 94.507 +12800/69092 Loss: 90.907 +16000/69092 Loss: 94.202 +19200/69092 Loss: 93.587 +22400/69092 Loss: 94.077 +25600/69092 Loss: 94.347 +28800/69092 Loss: 93.079 +32000/69092 Loss: 93.517 +35200/69092 Loss: 93.432 +38400/69092 Loss: 94.089 +41600/69092 Loss: 94.129 +44800/69092 Loss: 93.881 +48000/69092 Loss: 95.019 +51200/69092 Loss: 93.879 +54400/69092 Loss: 92.711 +57600/69092 Loss: 93.385 +60800/69092 Loss: 94.130 +64000/69092 Loss: 93.788 +67200/69092 Loss: 93.357 +Training time 0:06:58.006102 +Epoch: 33 Average loss: 93.69 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 264) +0/69092 Loss: 100.435 +3200/69092 Loss: 95.487 +6400/69092 Loss: 95.220 +9600/69092 Loss: 92.782 +12800/69092 Loss: 92.154 +16000/69092 Loss: 93.941 +19200/69092 Loss: 92.487 +22400/69092 Loss: 93.238 +25600/69092 Loss: 93.125 +28800/69092 Loss: 95.396 +32000/69092 Loss: 93.285 +35200/69092 Loss: 93.284 +38400/69092 Loss: 93.904 +41600/69092 Loss: 93.350 +44800/69092 Loss: 94.356 +48000/69092 Loss: 92.680 +51200/69092 Loss: 92.969 +54400/69092 Loss: 93.093 +57600/69092 Loss: 94.034 +60800/69092 Loss: 93.292 +64000/69092 Loss: 92.719 +67200/69092 Loss: 93.829 +Training time 0:07:06.735766 +Epoch: 34 Average loss: 93.59 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 265) +0/69092 Loss: 93.811 +3200/69092 Loss: 92.820 +6400/69092 Loss: 94.973 +9600/69092 Loss: 93.743 +12800/69092 Loss: 93.008 +16000/69092 Loss: 93.329 +19200/69092 Loss: 93.017 +22400/69092 Loss: 94.531 +25600/69092 Loss: 94.180 +28800/69092 Loss: 92.846 +32000/69092 Loss: 94.095 +35200/69092 Loss: 93.935 +38400/69092 Loss: 92.765 +41600/69092 Loss: 94.828 +44800/69092 Loss: 92.949 +48000/69092 Loss: 93.175 +51200/69092 Loss: 95.265 +54400/69092 Loss: 93.193 +57600/69092 Loss: 92.640 +60800/69092 Loss: 93.445 +64000/69092 Loss: 92.821 +67200/69092 Loss: 94.815 +Training time 0:07:03.410689 +Epoch: 35 Average loss: 93.62 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 266) +0/69092 Loss: 83.360 +3200/69092 Loss: 94.565 +6400/69092 Loss: 95.067 +9600/69092 Loss: 91.926 +12800/69092 Loss: 93.163 +16000/69092 Loss: 93.349 +19200/69092 Loss: 94.452 +22400/69092 Loss: 93.721 +25600/69092 Loss: 93.591 +28800/69092 Loss: 95.295 +32000/69092 Loss: 93.322 +35200/69092 Loss: 93.582 +38400/69092 Loss: 93.383 +41600/69092 Loss: 92.727 +44800/69092 Loss: 92.520 +48000/69092 Loss: 94.412 +51200/69092 Loss: 94.572 +54400/69092 Loss: 92.589 +57600/69092 Loss: 94.587 +60800/69092 Loss: 93.354 +64000/69092 Loss: 93.367 +67200/69092 Loss: 92.784 +Training time 0:07:04.671140 +Epoch: 36 Average loss: 93.64 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 267) +0/69092 Loss: 100.371 +3200/69092 Loss: 94.071 +6400/69092 Loss: 93.271 +9600/69092 Loss: 94.163 +12800/69092 Loss: 91.935 +16000/69092 Loss: 93.439 +19200/69092 Loss: 92.337 +22400/69092 Loss: 93.676 +25600/69092 Loss: 94.121 +28800/69092 Loss: 92.181 +32000/69092 Loss: 92.042 +35200/69092 Loss: 93.660 +38400/69092 Loss: 92.445 +41600/69092 Loss: 93.822 +44800/69092 Loss: 93.543 +48000/69092 Loss: 93.959 +51200/69092 Loss: 93.435 +54400/69092 Loss: 95.076 +57600/69092 Loss: 93.718 +60800/69092 Loss: 93.814 +64000/69092 Loss: 92.744 +67200/69092 Loss: 93.017 +Training time 0:06:59.907736 +Epoch: 37 Average loss: 93.33 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 268) +0/69092 Loss: 100.740 +3200/69092 Loss: 92.019 +6400/69092 Loss: 94.097 +9600/69092 Loss: 92.550 +12800/69092 Loss: 93.252 +16000/69092 Loss: 94.385 +19200/69092 Loss: 93.504 +22400/69092 Loss: 92.266 +25600/69092 Loss: 93.742 +28800/69092 Loss: 94.089 +32000/69092 Loss: 94.235 +35200/69092 Loss: 93.887 +38400/69092 Loss: 93.994 +41600/69092 Loss: 93.715 +44800/69092 Loss: 93.188 +48000/69092 Loss: 93.574 +51200/69092 Loss: 92.766 +54400/69092 Loss: 93.564 +57600/69092 Loss: 92.800 +60800/69092 Loss: 93.783 +64000/69092 Loss: 94.058 +67200/69092 Loss: 94.499 +Training time 0:07:05.589520 +Epoch: 38 Average loss: 93.49 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 269) +0/69092 Loss: 88.844 +3200/69092 Loss: 93.925 +6400/69092 Loss: 90.867 +9600/69092 Loss: 94.601 +12800/69092 Loss: 93.800 +16000/69092 Loss: 93.144 +19200/69092 Loss: 93.385 +22400/69092 Loss: 93.795 +25600/69092 Loss: 93.682 +28800/69092 Loss: 93.326 +32000/69092 Loss: 93.157 +35200/69092 Loss: 93.875 +38400/69092 Loss: 93.727 +41600/69092 Loss: 93.252 +44800/69092 Loss: 94.059 +48000/69092 Loss: 92.622 +51200/69092 Loss: 93.057 +54400/69092 Loss: 93.775 +57600/69092 Loss: 94.975 +60800/69092 Loss: 92.577 +64000/69092 Loss: 93.395 +67200/69092 Loss: 95.323 +Training time 0:07:01.926352 +Epoch: 39 Average loss: 93.54 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 270) +0/69092 Loss: 87.129 +3200/69092 Loss: 92.901 +6400/69092 Loss: 93.095 +9600/69092 Loss: 93.067 +12800/69092 Loss: 93.046 +16000/69092 Loss: 93.275 +19200/69092 Loss: 93.726 +22400/69092 Loss: 92.880 +25600/69092 Loss: 94.158 +28800/69092 Loss: 94.366 +32000/69092 Loss: 94.610 +35200/69092 Loss: 92.425 +38400/69092 Loss: 92.927 +41600/69092 Loss: 94.287 +44800/69092 Loss: 94.346 +48000/69092 Loss: 94.566 +51200/69092 Loss: 93.048 +54400/69092 Loss: 94.151 +57600/69092 Loss: 92.607 +60800/69092 Loss: 92.808 +64000/69092 Loss: 92.593 +67200/69092 Loss: 93.874 +Training time 0:06:54.876161 +Epoch: 40 Average loss: 93.51 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 271) +0/69092 Loss: 92.415 +3200/69092 Loss: 92.357 +6400/69092 Loss: 93.147 +9600/69092 Loss: 93.883 +12800/69092 Loss: 93.691 +16000/69092 Loss: 92.528 +19200/69092 Loss: 93.745 +22400/69092 Loss: 94.083 +25600/69092 Loss: 94.158 +28800/69092 Loss: 92.199 +32000/69092 Loss: 93.931 +35200/69092 Loss: 91.717 +38400/69092 Loss: 95.522 +41600/69092 Loss: 93.916 +44800/69092 Loss: 92.727 +48000/69092 Loss: 93.924 +51200/69092 Loss: 93.853 +54400/69092 Loss: 93.795 +57600/69092 Loss: 94.596 +60800/69092 Loss: 94.001 +64000/69092 Loss: 93.537 +67200/69092 Loss: 93.377 +Training time 0:07:04.623070 +Epoch: 41 Average loss: 93.55 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 272) +0/69092 Loss: 84.335 +3200/69092 Loss: 93.574 +6400/69092 Loss: 93.742 +9600/69092 Loss: 91.718 +12800/69092 Loss: 93.692 +16000/69092 Loss: 93.585 +19200/69092 Loss: 93.553 +22400/69092 Loss: 94.403 +25600/69092 Loss: 92.876 +28800/69092 Loss: 94.815 +32000/69092 Loss: 92.975 +35200/69092 Loss: 93.024 +38400/69092 Loss: 93.580 +41600/69092 Loss: 92.611 +44800/69092 Loss: 92.174 +48000/69092 Loss: 94.334 +51200/69092 Loss: 93.557 +54400/69092 Loss: 93.344 +57600/69092 Loss: 92.551 +60800/69092 Loss: 92.518 +64000/69092 Loss: 95.062 +67200/69092 Loss: 93.588 +Training time 0:06:58.053124 +Epoch: 42 Average loss: 93.41 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 273) +0/69092 Loss: 86.378 +3200/69092 Loss: 94.490 +6400/69092 Loss: 93.240 +9600/69092 Loss: 94.708 +12800/69092 Loss: 92.909 +16000/69092 Loss: 92.741 +19200/69092 Loss: 92.189 +22400/69092 Loss: 93.218 +25600/69092 Loss: 94.175 +28800/69092 Loss: 94.273 +32000/69092 Loss: 93.785 +35200/69092 Loss: 93.442 +38400/69092 Loss: 94.381 +41600/69092 Loss: 94.249 +44800/69092 Loss: 93.853 +48000/69092 Loss: 92.900 +51200/69092 Loss: 94.007 +54400/69092 Loss: 92.366 +57600/69092 Loss: 93.261 +60800/69092 Loss: 94.317 +64000/69092 Loss: 93.246 +67200/69092 Loss: 93.206 +Training time 0:07:11.713550 +Epoch: 43 Average loss: 93.57 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last' (iter 274) +0/69092 Loss: 87.270 +3200/69092 Loss: 93.622 +6400/69092 Loss: 93.504 +9600/69092 Loss: 92.500 +12800/69092 Loss: 93.778 +16000/69092 Loss: 93.100 +19200/69092 Loss: 93.562 +22400/69092 Loss: 93.398 +25600/69092 Loss: 94.634 +28800/69092 Loss: 93.390 +32000/69092 Loss: 93.942 +35200/69092 Loss: 93.197 +38400/69092 Loss: 92.071 +41600/69092 Loss: 93.950 +44800/69092 Loss: 92.994 +48000/69092 Loss: 94.056 diff --git a/OAR.2071928.stderr b/OAR.2071928.stderr new file mode 100644 index 0000000000000000000000000000000000000000..b94539781871578b0885599ebe7813e2c98e69c9 --- /dev/null +++ b/OAR.2071928.stderr @@ -0,0 +1,10 @@ +Traceback (most recent call last): + File "main.py", line 154, in <module> + main(args) + File "main.py", line 87, in main + beta=args.beta) + File "/data1/data/expes/julien.dejasmin/Thesis/Pytorch_CNN_mixt_representation/utils/training.py", line 75, in __init__ + self.load_checkpoint(self.ckpt_name) + File "/data1/data/expes/julien.dejasmin/Thesis/Pytorch_CNN_mixt_representation/utils/training.py", line 394, in load_checkpoint + self.mean_epoch_loss = checkpoint['loss'] +KeyError: 'loss' diff --git a/OAR.2071928.stdout b/OAR.2071928.stdout new file mode 100644 index 0000000000000000000000000000000000000000..1ef17703fa18f384f1248c43bb9c00235d24f317 --- /dev/null +++ b/OAR.2071928.stdout @@ -0,0 +1,46 @@ +Namespace(batch_size=256, beta=4, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='beta_VAE_bs_256', gpu_devices=[0, 1], is_beta_VAE=True, latent_spec_cont=10, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +GeForce RTX 2080 Ti +GeForce RTX 2080 Ti +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=20, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=10, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 765335 +don't use continuous capacity diff --git a/OAR.2071929.stderr b/OAR.2071929.stderr new file mode 100644 index 0000000000000000000000000000000000000000..a9c628a29c58ebc65df754dd5dfbbd99e2282b5f --- /dev/null +++ b/OAR.2071929.stderr @@ -0,0 +1,2 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071929.stdout b/OAR.2071929.stdout new file mode 100644 index 0000000000000000000000000000000000000000..b21b9064a1cdc3ef62c372457a830d7be6f66708 --- /dev/null +++ b/OAR.2071929.stdout @@ -0,0 +1,784 @@ +Namespace(batch_size=64, beta=4, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='beta_VAE_bs_64', gpu_devices=[0, 1], is_beta_VAE=True, latent_spec_cont=10, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +GeForce RTX 2080 Ti +GeForce RTX 2080 Ti +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=20, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=10, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 765335 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last (iter 768)' +0/69092 Loss: 151.940 +3200/69092 Loss: 151.123 +6400/69092 Loss: 152.240 +9600/69092 Loss: 154.456 +12800/69092 Loss: 152.151 +16000/69092 Loss: 150.843 +19200/69092 Loss: 151.704 +22400/69092 Loss: 151.679 +25600/69092 Loss: 150.622 +28800/69092 Loss: 153.092 +32000/69092 Loss: 153.267 +35200/69092 Loss: 150.582 +38400/69092 Loss: 153.799 +41600/69092 Loss: 151.305 +44800/69092 Loss: 151.560 +48000/69092 Loss: 150.741 +51200/69092 Loss: 151.885 +54400/69092 Loss: 150.161 +57600/69092 Loss: 152.044 +60800/69092 Loss: 150.668 +64000/69092 Loss: 150.622 +67200/69092 Loss: 150.227 +Training time 0:10:39.669089 +Epoch: 1 Average loss: 151.68 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 769) +0/69092 Loss: 159.826 +3200/69092 Loss: 155.528 +6400/69092 Loss: 153.084 +9600/69092 Loss: 152.096 +12800/69092 Loss: 152.363 +16000/69092 Loss: 150.556 +19200/69092 Loss: 148.058 +22400/69092 Loss: 152.518 +25600/69092 Loss: 151.159 +28800/69092 Loss: 153.739 +32000/69092 Loss: 149.939 +35200/69092 Loss: 152.543 +38400/69092 Loss: 151.002 +41600/69092 Loss: 151.378 +44800/69092 Loss: 149.524 +48000/69092 Loss: 152.165 +51200/69092 Loss: 151.533 +54400/69092 Loss: 148.544 +57600/69092 Loss: 152.467 +60800/69092 Loss: 153.726 +64000/69092 Loss: 151.344 +67200/69092 Loss: 154.598 +Training time 0:10:34.220042 +Epoch: 2 Average loss: 151.83 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 770) +0/69092 Loss: 155.574 +3200/69092 Loss: 151.047 +6400/69092 Loss: 149.378 +9600/69092 Loss: 151.613 +12800/69092 Loss: 152.097 +16000/69092 Loss: 152.221 +19200/69092 Loss: 151.504 +22400/69092 Loss: 152.222 +25600/69092 Loss: 152.531 +28800/69092 Loss: 150.282 +32000/69092 Loss: 151.100 +35200/69092 Loss: 153.495 +38400/69092 Loss: 149.776 +41600/69092 Loss: 150.461 +44800/69092 Loss: 152.533 +48000/69092 Loss: 153.336 +51200/69092 Loss: 153.742 +54400/69092 Loss: 152.341 +57600/69092 Loss: 153.011 +60800/69092 Loss: 153.359 +64000/69092 Loss: 151.765 +67200/69092 Loss: 149.833 +Training time 0:10:18.548337 +Epoch: 3 Average loss: 151.74 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 771) +0/69092 Loss: 149.541 +3200/69092 Loss: 150.722 +6400/69092 Loss: 152.327 +9600/69092 Loss: 148.762 +12800/69092 Loss: 150.084 +16000/69092 Loss: 151.317 +19200/69092 Loss: 151.855 +22400/69092 Loss: 151.256 +25600/69092 Loss: 153.280 +28800/69092 Loss: 152.067 +32000/69092 Loss: 154.260 +35200/69092 Loss: 147.713 +38400/69092 Loss: 152.622 +41600/69092 Loss: 152.811 +44800/69092 Loss: 151.757 +48000/69092 Loss: 152.970 +51200/69092 Loss: 150.973 +54400/69092 Loss: 150.648 +57600/69092 Loss: 152.005 +60800/69092 Loss: 152.170 +64000/69092 Loss: 152.880 +67200/69092 Loss: 152.945 +Training time 0:10:46.082361 +Epoch: 4 Average loss: 151.68 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 772) +0/69092 Loss: 128.234 +3200/69092 Loss: 151.711 +6400/69092 Loss: 151.919 +9600/69092 Loss: 150.830 +12800/69092 Loss: 153.890 +16000/69092 Loss: 150.860 +19200/69092 Loss: 153.058 +22400/69092 Loss: 153.962 +25600/69092 Loss: 151.795 +28800/69092 Loss: 150.298 +32000/69092 Loss: 153.955 +35200/69092 Loss: 151.321 +38400/69092 Loss: 150.578 +41600/69092 Loss: 151.991 +44800/69092 Loss: 151.477 +48000/69092 Loss: 150.079 +51200/69092 Loss: 151.311 +54400/69092 Loss: 150.190 +57600/69092 Loss: 153.374 +60800/69092 Loss: 154.199 +64000/69092 Loss: 152.083 +67200/69092 Loss: 152.334 +Training time 0:10:26.629045 +Epoch: 5 Average loss: 151.90 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 773) +0/69092 Loss: 151.981 +3200/69092 Loss: 152.328 +6400/69092 Loss: 153.386 +9600/69092 Loss: 150.931 +12800/69092 Loss: 152.916 +16000/69092 Loss: 150.853 +19200/69092 Loss: 155.195 +22400/69092 Loss: 150.530 +25600/69092 Loss: 150.215 +28800/69092 Loss: 149.994 +32000/69092 Loss: 151.922 +35200/69092 Loss: 150.617 +38400/69092 Loss: 148.687 +41600/69092 Loss: 149.949 +44800/69092 Loss: 150.374 +48000/69092 Loss: 153.431 +51200/69092 Loss: 152.256 +54400/69092 Loss: 154.912 +57600/69092 Loss: 151.775 +60800/69092 Loss: 148.809 +64000/69092 Loss: 152.993 +67200/69092 Loss: 152.113 +Training time 0:10:55.747236 +Epoch: 6 Average loss: 151.71 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 774) +0/69092 Loss: 161.032 +3200/69092 Loss: 150.489 +6400/69092 Loss: 151.788 +9600/69092 Loss: 151.782 +12800/69092 Loss: 150.235 +16000/69092 Loss: 154.594 +19200/69092 Loss: 150.676 +22400/69092 Loss: 150.407 +25600/69092 Loss: 151.105 +28800/69092 Loss: 152.824 +32000/69092 Loss: 152.237 +35200/69092 Loss: 153.190 +38400/69092 Loss: 150.788 +41600/69092 Loss: 149.924 +44800/69092 Loss: 152.878 +48000/69092 Loss: 151.703 +51200/69092 Loss: 150.422 +54400/69092 Loss: 151.719 +57600/69092 Loss: 152.224 +60800/69092 Loss: 153.809 +64000/69092 Loss: 149.802 +67200/69092 Loss: 150.667 +Training time 0:10:53.402046 +Epoch: 7 Average loss: 151.65 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 775) +0/69092 Loss: 150.086 +3200/69092 Loss: 153.433 +6400/69092 Loss: 150.415 +9600/69092 Loss: 153.438 +12800/69092 Loss: 149.193 +16000/69092 Loss: 151.045 +19200/69092 Loss: 148.747 +22400/69092 Loss: 151.117 +25600/69092 Loss: 150.434 +28800/69092 Loss: 150.123 +32000/69092 Loss: 152.996 +35200/69092 Loss: 152.099 +38400/69092 Loss: 151.190 +41600/69092 Loss: 152.928 +44800/69092 Loss: 152.553 +48000/69092 Loss: 151.283 +51200/69092 Loss: 152.039 +54400/69092 Loss: 151.448 +57600/69092 Loss: 151.806 +60800/69092 Loss: 150.245 +64000/69092 Loss: 152.300 +67200/69092 Loss: 152.545 +Training time 0:10:16.941117 +Epoch: 8 Average loss: 151.50 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 776) +0/69092 Loss: 142.755 +3200/69092 Loss: 150.657 +6400/69092 Loss: 151.619 +9600/69092 Loss: 152.357 +12800/69092 Loss: 151.392 +16000/69092 Loss: 151.836 +19200/69092 Loss: 152.021 +22400/69092 Loss: 153.780 +25600/69092 Loss: 151.759 +28800/69092 Loss: 151.016 +32000/69092 Loss: 150.397 +35200/69092 Loss: 151.324 +38400/69092 Loss: 152.198 +41600/69092 Loss: 153.828 +44800/69092 Loss: 150.664 +48000/69092 Loss: 150.837 +51200/69092 Loss: 154.898 +54400/69092 Loss: 151.182 +57600/69092 Loss: 148.820 +60800/69092 Loss: 152.543 +64000/69092 Loss: 151.735 +67200/69092 Loss: 148.089 +Training time 0:10:28.361734 +Epoch: 9 Average loss: 151.61 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 777) +0/69092 Loss: 142.051 +3200/69092 Loss: 148.852 +6400/69092 Loss: 148.513 +9600/69092 Loss: 152.910 +12800/69092 Loss: 150.108 +16000/69092 Loss: 154.028 +19200/69092 Loss: 150.451 +22400/69092 Loss: 153.521 +25600/69092 Loss: 147.976 +28800/69092 Loss: 148.766 +32000/69092 Loss: 149.970 +35200/69092 Loss: 150.618 +38400/69092 Loss: 150.971 +41600/69092 Loss: 150.817 +44800/69092 Loss: 154.553 +48000/69092 Loss: 155.033 +51200/69092 Loss: 149.239 +54400/69092 Loss: 152.046 +57600/69092 Loss: 153.721 +60800/69092 Loss: 151.871 +64000/69092 Loss: 154.558 +67200/69092 Loss: 150.223 +Training time 0:10:23.290199 +Epoch: 10 Average loss: 151.46 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 778) +0/69092 Loss: 149.057 +3200/69092 Loss: 150.138 +6400/69092 Loss: 150.666 +9600/69092 Loss: 151.855 +12800/69092 Loss: 151.778 +16000/69092 Loss: 151.908 +19200/69092 Loss: 151.032 +22400/69092 Loss: 152.961 +25600/69092 Loss: 154.912 +28800/69092 Loss: 151.573 +32000/69092 Loss: 154.543 +35200/69092 Loss: 151.433 +38400/69092 Loss: 149.579 +41600/69092 Loss: 150.587 +44800/69092 Loss: 152.174 +48000/69092 Loss: 149.528 +51200/69092 Loss: 150.743 +54400/69092 Loss: 152.121 +57600/69092 Loss: 151.739 +60800/69092 Loss: 149.955 +64000/69092 Loss: 150.658 +67200/69092 Loss: 155.248 +Training time 0:10:02.301303 +Epoch: 11 Average loss: 151.73 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 779) +0/69092 Loss: 152.842 +3200/69092 Loss: 152.468 +6400/69092 Loss: 149.136 +9600/69092 Loss: 152.618 +12800/69092 Loss: 150.399 +16000/69092 Loss: 153.064 +19200/69092 Loss: 151.608 +22400/69092 Loss: 152.280 +25600/69092 Loss: 150.410 +28800/69092 Loss: 152.462 +32000/69092 Loss: 149.632 +35200/69092 Loss: 151.900 +38400/69092 Loss: 148.866 +41600/69092 Loss: 153.946 +44800/69092 Loss: 153.180 +48000/69092 Loss: 152.256 +51200/69092 Loss: 150.869 +54400/69092 Loss: 151.987 +57600/69092 Loss: 151.927 +60800/69092 Loss: 150.315 +64000/69092 Loss: 152.272 +67200/69092 Loss: 150.027 +Training time 0:10:35.921149 +Epoch: 12 Average loss: 151.59 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 780) +0/69092 Loss: 155.203 +3200/69092 Loss: 149.290 +6400/69092 Loss: 150.722 +9600/69092 Loss: 150.173 +12800/69092 Loss: 151.060 +16000/69092 Loss: 152.570 +19200/69092 Loss: 156.369 +22400/69092 Loss: 151.513 +25600/69092 Loss: 150.945 +28800/69092 Loss: 150.378 +32000/69092 Loss: 153.038 +35200/69092 Loss: 154.230 +38400/69092 Loss: 153.176 +41600/69092 Loss: 148.850 +44800/69092 Loss: 154.538 +48000/69092 Loss: 151.359 +51200/69092 Loss: 151.464 +54400/69092 Loss: 151.170 +57600/69092 Loss: 149.252 +60800/69092 Loss: 151.042 +64000/69092 Loss: 153.423 +67200/69092 Loss: 149.422 +Training time 0:10:31.361427 +Epoch: 13 Average loss: 151.71 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 781) +0/69092 Loss: 146.776 +3200/69092 Loss: 152.727 +6400/69092 Loss: 154.581 +9600/69092 Loss: 151.684 +12800/69092 Loss: 149.094 +16000/69092 Loss: 151.217 +19200/69092 Loss: 150.166 +22400/69092 Loss: 152.380 +25600/69092 Loss: 151.874 +28800/69092 Loss: 153.713 +32000/69092 Loss: 152.922 +35200/69092 Loss: 154.464 +38400/69092 Loss: 150.433 +41600/69092 Loss: 149.945 +44800/69092 Loss: 152.313 +48000/69092 Loss: 152.333 +51200/69092 Loss: 153.386 +54400/69092 Loss: 152.386 +57600/69092 Loss: 150.411 +60800/69092 Loss: 151.260 +64000/69092 Loss: 153.093 +67200/69092 Loss: 147.063 +Training time 0:10:44.034995 +Epoch: 14 Average loss: 151.70 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 782) +0/69092 Loss: 133.995 +3200/69092 Loss: 153.631 +6400/69092 Loss: 152.677 +9600/69092 Loss: 148.565 +12800/69092 Loss: 152.216 +16000/69092 Loss: 153.812 +19200/69092 Loss: 153.412 +22400/69092 Loss: 149.740 +25600/69092 Loss: 152.386 +28800/69092 Loss: 152.186 +32000/69092 Loss: 153.078 +35200/69092 Loss: 151.331 +38400/69092 Loss: 151.224 +41600/69092 Loss: 151.745 +44800/69092 Loss: 150.595 +48000/69092 Loss: 149.903 +51200/69092 Loss: 151.151 +54400/69092 Loss: 152.719 +57600/69092 Loss: 147.748 +60800/69092 Loss: 147.822 +64000/69092 Loss: 151.649 +67200/69092 Loss: 152.350 +Training time 0:10:26.490651 +Epoch: 15 Average loss: 151.37 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 783) +0/69092 Loss: 165.291 +3200/69092 Loss: 149.069 +6400/69092 Loss: 151.144 +9600/69092 Loss: 149.362 +12800/69092 Loss: 152.472 +16000/69092 Loss: 150.844 +19200/69092 Loss: 150.975 +22400/69092 Loss: 150.666 +25600/69092 Loss: 149.360 +28800/69092 Loss: 150.892 +32000/69092 Loss: 151.124 +35200/69092 Loss: 152.196 +38400/69092 Loss: 153.774 +41600/69092 Loss: 152.521 +44800/69092 Loss: 151.684 +48000/69092 Loss: 153.329 +51200/69092 Loss: 150.415 +54400/69092 Loss: 151.445 +57600/69092 Loss: 154.007 +60800/69092 Loss: 150.657 +64000/69092 Loss: 152.308 +67200/69092 Loss: 150.726 +Training time 0:11:06.137906 +Epoch: 16 Average loss: 151.42 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 784) +0/69092 Loss: 144.484 +3200/69092 Loss: 151.458 +6400/69092 Loss: 152.609 +9600/69092 Loss: 152.940 +12800/69092 Loss: 153.378 +16000/69092 Loss: 152.672 +19200/69092 Loss: 152.431 +22400/69092 Loss: 151.115 +25600/69092 Loss: 151.894 +28800/69092 Loss: 150.751 +32000/69092 Loss: 152.050 +35200/69092 Loss: 150.673 +38400/69092 Loss: 151.820 +41600/69092 Loss: 152.699 +44800/69092 Loss: 150.842 +48000/69092 Loss: 152.601 +51200/69092 Loss: 148.367 +54400/69092 Loss: 151.450 +57600/69092 Loss: 151.025 +60800/69092 Loss: 150.256 +64000/69092 Loss: 153.945 +67200/69092 Loss: 150.735 +Training time 0:10:30.921815 +Epoch: 17 Average loss: 151.67 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 785) +0/69092 Loss: 149.105 +3200/69092 Loss: 154.167 +6400/69092 Loss: 150.111 +9600/69092 Loss: 150.985 +12800/69092 Loss: 151.978 +16000/69092 Loss: 151.739 +19200/69092 Loss: 153.227 +22400/69092 Loss: 150.762 +25600/69092 Loss: 153.572 +28800/69092 Loss: 151.868 +32000/69092 Loss: 152.932 +35200/69092 Loss: 150.313 +38400/69092 Loss: 149.599 +41600/69092 Loss: 151.857 +44800/69092 Loss: 152.196 +48000/69092 Loss: 151.910 +51200/69092 Loss: 149.218 +54400/69092 Loss: 151.897 +57600/69092 Loss: 152.954 +60800/69092 Loss: 149.504 +64000/69092 Loss: 152.771 +67200/69092 Loss: 151.289 +Training time 0:10:48.111194 +Epoch: 18 Average loss: 151.65 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 786) +0/69092 Loss: 156.353 +3200/69092 Loss: 151.484 +6400/69092 Loss: 152.173 +9600/69092 Loss: 150.284 +12800/69092 Loss: 152.109 +16000/69092 Loss: 153.112 +19200/69092 Loss: 155.199 +22400/69092 Loss: 150.800 +25600/69092 Loss: 149.949 +28800/69092 Loss: 151.230 +32000/69092 Loss: 149.613 +35200/69092 Loss: 152.457 +38400/69092 Loss: 152.412 +41600/69092 Loss: 151.116 +44800/69092 Loss: 153.088 +48000/69092 Loss: 151.320 +51200/69092 Loss: 149.919 +54400/69092 Loss: 150.432 +57600/69092 Loss: 153.685 +60800/69092 Loss: 151.287 +64000/69092 Loss: 150.028 +67200/69092 Loss: 150.473 +Training time 0:10:12.069793 +Epoch: 19 Average loss: 151.59 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 787) +0/69092 Loss: 139.116 +3200/69092 Loss: 152.967 +6400/69092 Loss: 148.803 +9600/69092 Loss: 151.478 +12800/69092 Loss: 151.373 +16000/69092 Loss: 150.007 +19200/69092 Loss: 153.504 +22400/69092 Loss: 151.458 +25600/69092 Loss: 149.068 +28800/69092 Loss: 153.966 +32000/69092 Loss: 153.562 +35200/69092 Loss: 148.962 +38400/69092 Loss: 153.138 +41600/69092 Loss: 154.136 +44800/69092 Loss: 149.912 +48000/69092 Loss: 151.040 +51200/69092 Loss: 150.974 +54400/69092 Loss: 152.708 +57600/69092 Loss: 150.486 +60800/69092 Loss: 153.213 +64000/69092 Loss: 151.913 +67200/69092 Loss: 152.005 +Training time 0:10:20.811275 +Epoch: 20 Average loss: 151.61 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 788) +0/69092 Loss: 147.916 +3200/69092 Loss: 149.391 +6400/69092 Loss: 150.723 +9600/69092 Loss: 152.174 +12800/69092 Loss: 151.629 +16000/69092 Loss: 153.353 +19200/69092 Loss: 151.887 +22400/69092 Loss: 150.459 +25600/69092 Loss: 152.292 +28800/69092 Loss: 151.080 +32000/69092 Loss: 154.926 +35200/69092 Loss: 150.486 +38400/69092 Loss: 151.631 +41600/69092 Loss: 150.604 +44800/69092 Loss: 151.841 +48000/69092 Loss: 151.894 +51200/69092 Loss: 149.205 +54400/69092 Loss: 151.854 +57600/69092 Loss: 150.251 +60800/69092 Loss: 149.714 +64000/69092 Loss: 152.077 +67200/69092 Loss: 152.032 +Training time 0:10:11.770670 +Epoch: 21 Average loss: 151.42 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 789) +0/69092 Loss: 164.078 +3200/69092 Loss: 151.525 +6400/69092 Loss: 153.988 +9600/69092 Loss: 149.142 +12800/69092 Loss: 151.592 +16000/69092 Loss: 151.993 +19200/69092 Loss: 150.118 +22400/69092 Loss: 150.498 +25600/69092 Loss: 151.343 +28800/69092 Loss: 153.764 +32000/69092 Loss: 150.852 +35200/69092 Loss: 152.486 +38400/69092 Loss: 152.394 +41600/69092 Loss: 150.734 +44800/69092 Loss: 151.284 +48000/69092 Loss: 152.597 +51200/69092 Loss: 150.589 +54400/69092 Loss: 150.206 +57600/69092 Loss: 151.417 +60800/69092 Loss: 151.005 +64000/69092 Loss: 153.224 +67200/69092 Loss: 153.169 +Training time 0:10:33.472371 +Epoch: 22 Average loss: 151.71 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 790) +0/69092 Loss: 145.559 +3200/69092 Loss: 150.164 +6400/69092 Loss: 151.514 +9600/69092 Loss: 151.173 +12800/69092 Loss: 152.271 +16000/69092 Loss: 153.139 +19200/69092 Loss: 150.887 +22400/69092 Loss: 150.752 +25600/69092 Loss: 152.485 +28800/69092 Loss: 151.779 +32000/69092 Loss: 151.846 +35200/69092 Loss: 152.138 +38400/69092 Loss: 154.723 +41600/69092 Loss: 152.015 +44800/69092 Loss: 152.515 +48000/69092 Loss: 151.063 +51200/69092 Loss: 154.114 +54400/69092 Loss: 151.833 +57600/69092 Loss: 148.895 +60800/69092 Loss: 149.349 +64000/69092 Loss: 151.584 +67200/69092 Loss: 148.678 +Training time 0:09:53.632891 +Epoch: 23 Average loss: 151.62 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 791) +0/69092 Loss: 135.151 +3200/69092 Loss: 150.026 +6400/69092 Loss: 149.487 +9600/69092 Loss: 152.590 +12800/69092 Loss: 150.973 +16000/69092 Loss: 150.229 +19200/69092 Loss: 148.999 +22400/69092 Loss: 147.100 +25600/69092 Loss: 153.667 +28800/69092 Loss: 152.520 +32000/69092 Loss: 152.234 +35200/69092 Loss: 153.836 +38400/69092 Loss: 150.599 +41600/69092 Loss: 153.598 +44800/69092 Loss: 153.442 +48000/69092 Loss: 151.034 +51200/69092 Loss: 151.636 +54400/69092 Loss: 150.366 +57600/69092 Loss: 152.079 +60800/69092 Loss: 155.366 +64000/69092 Loss: 152.208 +67200/69092 Loss: 154.034 +Training time 0:11:00.856151 +Epoch: 24 Average loss: 151.67 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 792) +0/69092 Loss: 148.839 +3200/69092 Loss: 150.547 +6400/69092 Loss: 152.331 +9600/69092 Loss: 150.041 +12800/69092 Loss: 149.418 +16000/69092 Loss: 148.857 +19200/69092 Loss: 151.414 +22400/69092 Loss: 152.656 +25600/69092 Loss: 150.780 +28800/69092 Loss: 151.046 +32000/69092 Loss: 152.391 +35200/69092 Loss: 152.347 +38400/69092 Loss: 151.635 +41600/69092 Loss: 151.890 +44800/69092 Loss: 154.596 +48000/69092 Loss: 149.873 +51200/69092 Loss: 150.918 +54400/69092 Loss: 151.772 +57600/69092 Loss: 152.508 +60800/69092 Loss: 151.179 +64000/69092 Loss: 152.463 +67200/69092 Loss: 152.157 +Training time 0:11:03.146948 +Epoch: 25 Average loss: 151.51 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 793) +0/69092 Loss: 161.401 +3200/69092 Loss: 149.583 +6400/69092 Loss: 153.927 +9600/69092 Loss: 151.978 +12800/69092 Loss: 148.130 +16000/69092 Loss: 150.961 +19200/69092 Loss: 152.851 +22400/69092 Loss: 152.051 +25600/69092 Loss: 151.571 +28800/69092 Loss: 154.100 +32000/69092 Loss: 151.094 +35200/69092 Loss: 155.622 +38400/69092 Loss: 149.063 +41600/69092 Loss: 149.194 +44800/69092 Loss: 152.503 +48000/69092 Loss: 151.265 +51200/69092 Loss: 150.613 +54400/69092 Loss: 150.716 +57600/69092 Loss: 151.582 +60800/69092 Loss: 152.828 +64000/69092 Loss: 152.323 +67200/69092 Loss: 151.812 +Training time 0:10:32.774337 +Epoch: 26 Average loss: 151.67 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 794) +0/69092 Loss: 161.924 +3200/69092 Loss: 150.709 +6400/69092 Loss: 148.974 +9600/69092 Loss: 150.561 +12800/69092 Loss: 151.155 +16000/69092 Loss: 153.755 +19200/69092 Loss: 153.586 +22400/69092 Loss: 152.655 +25600/69092 Loss: 152.570 +28800/69092 Loss: 153.050 +32000/69092 Loss: 152.711 +35200/69092 Loss: 151.796 +38400/69092 Loss: 151.566 +41600/69092 Loss: 151.215 +44800/69092 Loss: 151.070 +48000/69092 Loss: 152.138 +51200/69092 Loss: 151.314 +54400/69092 Loss: 151.994 +57600/69092 Loss: 149.981 +60800/69092 Loss: 149.437 +64000/69092 Loss: 149.455 +67200/69092 Loss: 154.203 +Training time 0:10:02.543376 +Epoch: 27 Average loss: 151.57 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 795) +0/69092 Loss: 144.700 +3200/69092 Loss: 150.546 +6400/69092 Loss: 151.705 +9600/69092 Loss: 151.077 +12800/69092 Loss: 151.864 +16000/69092 Loss: 151.946 +19200/69092 Loss: 150.677 +22400/69092 Loss: 150.169 +25600/69092 Loss: 148.889 +28800/69092 Loss: 154.708 +32000/69092 Loss: 151.943 +35200/69092 Loss: 152.121 +38400/69092 Loss: 152.201 +41600/69092 Loss: 152.669 +44800/69092 Loss: 152.839 +48000/69092 Loss: 154.190 +51200/69092 Loss: 150.046 +54400/69092 Loss: 150.808 +57600/69092 Loss: 153.924 +60800/69092 Loss: 149.571 +64000/69092 Loss: 151.539 +67200/69092 Loss: 149.816 +Training time 0:10:12.554327 +Epoch: 28 Average loss: 151.63 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 796) +0/69092 Loss: 159.629 +3200/69092 Loss: 151.190 +6400/69092 Loss: 152.045 +9600/69092 Loss: 151.655 +12800/69092 Loss: 149.766 +16000/69092 Loss: 150.414 +19200/69092 Loss: 151.565 +22400/69092 Loss: 149.433 +25600/69092 Loss: 154.225 +28800/69092 Loss: 153.969 +32000/69092 Loss: 148.928 +35200/69092 Loss: 150.995 +38400/69092 Loss: 150.903 +41600/69092 Loss: 149.831 +44800/69092 Loss: 152.827 +48000/69092 Loss: 150.649 +51200/69092 Loss: 152.855 +54400/69092 Loss: 152.803 +57600/69092 Loss: 151.228 +60800/69092 Loss: 151.632 +64000/69092 Loss: 151.694 +67200/69092 Loss: 151.580 +Training time 0:10:29.073350 +Epoch: 29 Average loss: 151.49 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last' (iter 797) +0/69092 Loss: 153.443 +3200/69092 Loss: 152.324 +6400/69092 Loss: 150.465 +9600/69092 Loss: 149.903 +12800/69092 Loss: 152.710 +16000/69092 Loss: 149.294 +19200/69092 Loss: 150.507 +22400/69092 Loss: 150.007 +25600/69092 Loss: 151.076 +28800/69092 Loss: 150.543 +32000/69092 Loss: 152.231 +35200/69092 Loss: 151.692 diff --git a/OAR.2071930.stderr b/OAR.2071930.stderr new file mode 100644 index 0000000000000000000000000000000000000000..a9c628a29c58ebc65df754dd5dfbbd99e2282b5f --- /dev/null +++ b/OAR.2071930.stderr @@ -0,0 +1,2 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071930.stdout b/OAR.2071930.stdout new file mode 100644 index 0000000000000000000000000000000000000000..46e50419e9ff8f4dd8a795e2e0621850c0216a68 --- /dev/null +++ b/OAR.2071930.stdout @@ -0,0 +1,319 @@ +Namespace(batch_size=256, beta=None, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='VAE_bs_256', gpu_devices=[0, 1], is_beta_VAE=False, latent_spec_cont=10, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +GeForce RTX 2080 Ti +GeForce RTX 2080 Ti +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=20, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=10, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 765335 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last (iter 698)' +0/69092 Loss: 108.257 +12800/69092 Loss: 111.037 +25600/69092 Loss: 111.303 +38400/69092 Loss: 111.113 +51200/69092 Loss: 110.736 +64000/69092 Loss: 111.199 +Training time 0:10:21.698606 +Epoch: 1 Average loss: 111.13 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 699) +0/69092 Loss: 104.603 +12800/69092 Loss: 111.150 +25600/69092 Loss: 110.834 +38400/69092 Loss: 111.260 +51200/69092 Loss: 111.194 +64000/69092 Loss: 111.373 +Training time 0:10:20.675258 +Epoch: 2 Average loss: 111.24 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 700) +0/69092 Loss: 111.066 +12800/69092 Loss: 111.171 +25600/69092 Loss: 111.355 +38400/69092 Loss: 111.289 +51200/69092 Loss: 110.907 +64000/69092 Loss: 111.152 +Training time 0:09:57.137305 +Epoch: 3 Average loss: 111.25 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 701) +0/69092 Loss: 112.474 +12800/69092 Loss: 110.678 +25600/69092 Loss: 111.427 +38400/69092 Loss: 111.644 +51200/69092 Loss: 111.016 +64000/69092 Loss: 111.316 +Training time 0:10:35.748850 +Epoch: 4 Average loss: 111.19 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 702) +0/69092 Loss: 109.716 +12800/69092 Loss: 111.310 +25600/69092 Loss: 111.895 +38400/69092 Loss: 110.686 +51200/69092 Loss: 111.364 +64000/69092 Loss: 111.271 +Training time 0:10:36.029432 +Epoch: 5 Average loss: 111.26 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 703) +0/69092 Loss: 112.889 +12800/69092 Loss: 111.197 +25600/69092 Loss: 110.554 +38400/69092 Loss: 111.645 +51200/69092 Loss: 110.559 +64000/69092 Loss: 111.748 +Training time 0:10:28.903556 +Epoch: 6 Average loss: 111.14 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 704) +0/69092 Loss: 109.053 +12800/69092 Loss: 111.012 +25600/69092 Loss: 110.963 +38400/69092 Loss: 111.352 +51200/69092 Loss: 112.067 +64000/69092 Loss: 110.935 +Training time 0:10:12.876278 +Epoch: 7 Average loss: 111.26 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 705) +0/69092 Loss: 105.232 +12800/69092 Loss: 110.625 +25600/69092 Loss: 111.185 +38400/69092 Loss: 111.544 +51200/69092 Loss: 110.688 +64000/69092 Loss: 111.145 +Training time 0:10:17.070023 +Epoch: 8 Average loss: 111.12 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 706) +0/69092 Loss: 110.348 +12800/69092 Loss: 110.485 +25600/69092 Loss: 111.852 +38400/69092 Loss: 111.109 +51200/69092 Loss: 111.071 +64000/69092 Loss: 111.193 +Training time 0:10:13.666167 +Epoch: 9 Average loss: 111.15 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 707) +0/69092 Loss: 111.819 +12800/69092 Loss: 110.953 +25600/69092 Loss: 111.249 +38400/69092 Loss: 111.333 +51200/69092 Loss: 110.671 +64000/69092 Loss: 111.599 +Training time 0:10:25.411552 +Epoch: 10 Average loss: 111.27 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 708) +0/69092 Loss: 109.143 +12800/69092 Loss: 111.175 +25600/69092 Loss: 111.717 +38400/69092 Loss: 110.971 +51200/69092 Loss: 111.052 +64000/69092 Loss: 111.495 +Training time 0:09:47.203189 +Epoch: 11 Average loss: 111.26 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 709) +0/69092 Loss: 109.594 +12800/69092 Loss: 111.242 +25600/69092 Loss: 111.338 +38400/69092 Loss: 111.486 +51200/69092 Loss: 111.300 +64000/69092 Loss: 110.529 +Training time 0:10:34.029615 +Epoch: 12 Average loss: 111.11 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 710) +0/69092 Loss: 112.388 +12800/69092 Loss: 111.593 +25600/69092 Loss: 111.454 +38400/69092 Loss: 110.729 +51200/69092 Loss: 111.548 +64000/69092 Loss: 110.343 +Training time 0:09:57.117715 +Epoch: 13 Average loss: 111.17 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 711) +0/69092 Loss: 108.358 +12800/69092 Loss: 110.723 +25600/69092 Loss: 110.354 +38400/69092 Loss: 111.101 +51200/69092 Loss: 111.659 +64000/69092 Loss: 112.200 +Training time 0:10:32.154893 +Epoch: 14 Average loss: 111.21 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 712) +0/69092 Loss: 115.446 +12800/69092 Loss: 110.808 +25600/69092 Loss: 111.401 +38400/69092 Loss: 111.777 +51200/69092 Loss: 111.487 +64000/69092 Loss: 110.340 +Training time 0:10:10.584737 +Epoch: 15 Average loss: 111.17 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 713) +0/69092 Loss: 108.508 +12800/69092 Loss: 111.041 +25600/69092 Loss: 111.330 +38400/69092 Loss: 111.807 +51200/69092 Loss: 110.823 +64000/69092 Loss: 111.151 +Training time 0:10:14.803333 +Epoch: 16 Average loss: 111.21 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 714) +0/69092 Loss: 112.184 +12800/69092 Loss: 109.998 +25600/69092 Loss: 110.801 +38400/69092 Loss: 111.722 +51200/69092 Loss: 111.301 +64000/69092 Loss: 111.610 +Training time 0:10:35.918923 +Epoch: 17 Average loss: 111.08 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 715) +0/69092 Loss: 111.794 +12800/69092 Loss: 111.958 +25600/69092 Loss: 110.631 +38400/69092 Loss: 110.767 +51200/69092 Loss: 110.908 +64000/69092 Loss: 111.484 +Training time 0:10:23.342858 +Epoch: 18 Average loss: 111.11 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 716) +0/69092 Loss: 111.656 +12800/69092 Loss: 110.986 +25600/69092 Loss: 111.361 +38400/69092 Loss: 110.774 +51200/69092 Loss: 111.714 +64000/69092 Loss: 110.844 +Training time 0:10:23.680008 +Epoch: 19 Average loss: 111.10 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 717) +0/69092 Loss: 111.053 +12800/69092 Loss: 110.972 +25600/69092 Loss: 110.889 +38400/69092 Loss: 110.627 +51200/69092 Loss: 110.883 +64000/69092 Loss: 111.855 +Training time 0:10:05.521272 +Epoch: 20 Average loss: 111.04 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 718) +0/69092 Loss: 114.731 +12800/69092 Loss: 110.582 +25600/69092 Loss: 110.897 +38400/69092 Loss: 111.680 +51200/69092 Loss: 111.049 +64000/69092 Loss: 110.636 +Training time 0:09:57.470885 +Epoch: 21 Average loss: 110.99 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 719) +0/69092 Loss: 107.406 +12800/69092 Loss: 111.142 +25600/69092 Loss: 110.856 +38400/69092 Loss: 110.898 +51200/69092 Loss: 111.567 +64000/69092 Loss: 110.952 +Training time 0:10:21.494861 +Epoch: 22 Average loss: 111.03 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 720) +0/69092 Loss: 110.198 +12800/69092 Loss: 111.221 +25600/69092 Loss: 110.645 +38400/69092 Loss: 111.540 +51200/69092 Loss: 110.118 +64000/69092 Loss: 111.931 +Training time 0:10:36.660048 +Epoch: 23 Average loss: 111.12 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 721) +0/69092 Loss: 111.275 +12800/69092 Loss: 111.169 +25600/69092 Loss: 110.754 +38400/69092 Loss: 110.518 +51200/69092 Loss: 110.996 +64000/69092 Loss: 110.942 +Training time 0:10:24.377103 +Epoch: 24 Average loss: 111.00 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 722) +0/69092 Loss: 109.891 +12800/69092 Loss: 111.340 +25600/69092 Loss: 110.879 +38400/69092 Loss: 110.899 +51200/69092 Loss: 111.343 +64000/69092 Loss: 110.808 +Training time 0:10:52.849173 +Epoch: 25 Average loss: 111.10 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 723) +0/69092 Loss: 114.979 +12800/69092 Loss: 110.284 +25600/69092 Loss: 111.689 +38400/69092 Loss: 110.894 +51200/69092 Loss: 110.399 +64000/69092 Loss: 111.512 +Training time 0:10:33.774797 +Epoch: 26 Average loss: 111.04 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 724) +0/69092 Loss: 109.348 +12800/69092 Loss: 111.616 +25600/69092 Loss: 111.608 +38400/69092 Loss: 110.426 +51200/69092 Loss: 111.261 +64000/69092 Loss: 110.918 +Training time 0:10:14.889610 +Epoch: 27 Average loss: 111.19 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 725) +0/69092 Loss: 102.225 +12800/69092 Loss: 111.153 +25600/69092 Loss: 110.742 +38400/69092 Loss: 111.073 +51200/69092 Loss: 111.250 +64000/69092 Loss: 111.164 +Training time 0:10:20.088056 +Epoch: 28 Average loss: 111.08 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 726) +0/69092 Loss: 109.719 +12800/69092 Loss: 110.633 +25600/69092 Loss: 111.668 +38400/69092 Loss: 110.447 +51200/69092 Loss: 110.795 +64000/69092 Loss: 110.865 +Training time 0:10:19.927331 +Epoch: 29 Average loss: 110.89 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 727) +0/69092 Loss: 106.510 +12800/69092 Loss: 110.479 +25600/69092 Loss: 111.096 +38400/69092 Loss: 111.419 +51200/69092 Loss: 111.554 +64000/69092 Loss: 110.890 +Training time 0:10:14.306920 +Epoch: 30 Average loss: 111.07 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_256/checkpoints/last' (iter 728) +0/69092 Loss: 109.131 +12800/69092 Loss: 111.285 diff --git a/OAR.2071931.stderr b/OAR.2071931.stderr new file mode 100644 index 0000000000000000000000000000000000000000..a9c628a29c58ebc65df754dd5dfbbd99e2282b5f --- /dev/null +++ b/OAR.2071931.stderr @@ -0,0 +1,2 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071931.stdout b/OAR.2071931.stdout new file mode 100644 index 0000000000000000000000000000000000000000..a18f71b5dc40dd79af6548131a4043a3fc664f68 --- /dev/null +++ b/OAR.2071931.stdout @@ -0,0 +1,965 @@ +Namespace(batch_size=64, beta=None, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='VAE_bs_64', gpu_devices=[0, 1], is_beta_VAE=False, latent_spec_cont=10, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +GeForce RTX 2080 Ti +GeForce RTX 2080 Ti +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=20, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=10, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 765335 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last (iter 893)' +0/69092 Loss: 96.506 +3200/69092 Loss: 107.976 +6400/69092 Loss: 111.118 +9600/69092 Loss: 109.679 +12800/69092 Loss: 111.490 +16000/69092 Loss: 110.653 +19200/69092 Loss: 110.753 +22400/69092 Loss: 110.464 +25600/69092 Loss: 108.793 +28800/69092 Loss: 111.036 +32000/69092 Loss: 109.287 +35200/69092 Loss: 111.332 +38400/69092 Loss: 110.710 +41600/69092 Loss: 110.470 +44800/69092 Loss: 108.910 +48000/69092 Loss: 109.653 +51200/69092 Loss: 110.298 +54400/69092 Loss: 111.237 +57600/69092 Loss: 108.088 +60800/69092 Loss: 108.456 +64000/69092 Loss: 108.938 +67200/69092 Loss: 110.837 +Training time 0:08:08.135205 +Epoch: 1 Average loss: 110.03 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 894) +0/69092 Loss: 102.102 +3200/69092 Loss: 109.706 +6400/69092 Loss: 112.744 +9600/69092 Loss: 110.252 +12800/69092 Loss: 109.895 +16000/69092 Loss: 109.431 +19200/69092 Loss: 109.181 +22400/69092 Loss: 109.455 +25600/69092 Loss: 109.177 +28800/69092 Loss: 109.263 +32000/69092 Loss: 110.940 +35200/69092 Loss: 111.500 +38400/69092 Loss: 110.274 +41600/69092 Loss: 109.270 +44800/69092 Loss: 109.507 +48000/69092 Loss: 109.601 +51200/69092 Loss: 108.849 +54400/69092 Loss: 110.087 +57600/69092 Loss: 110.105 +60800/69092 Loss: 109.624 +64000/69092 Loss: 109.683 +67200/69092 Loss: 108.176 +Training time 0:08:38.014367 +Epoch: 2 Average loss: 109.91 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 895) +0/69092 Loss: 110.342 +3200/69092 Loss: 110.021 +6400/69092 Loss: 109.821 +9600/69092 Loss: 110.290 +12800/69092 Loss: 109.858 +16000/69092 Loss: 109.090 +19200/69092 Loss: 111.522 +22400/69092 Loss: 108.451 +25600/69092 Loss: 109.246 +28800/69092 Loss: 111.043 +32000/69092 Loss: 112.258 +35200/69092 Loss: 109.115 +38400/69092 Loss: 110.949 +41600/69092 Loss: 109.815 +44800/69092 Loss: 108.342 +48000/69092 Loss: 110.874 +51200/69092 Loss: 110.267 +54400/69092 Loss: 110.616 +57600/69092 Loss: 109.410 +60800/69092 Loss: 109.178 +64000/69092 Loss: 108.993 +67200/69092 Loss: 108.393 +Training time 0:08:26.658550 +Epoch: 3 Average loss: 109.83 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 896) +0/69092 Loss: 106.450 +3200/69092 Loss: 108.478 +6400/69092 Loss: 109.268 +9600/69092 Loss: 107.994 +12800/69092 Loss: 109.262 +16000/69092 Loss: 110.297 +19200/69092 Loss: 109.548 +22400/69092 Loss: 110.925 +25600/69092 Loss: 110.579 +28800/69092 Loss: 109.089 +32000/69092 Loss: 111.086 +35200/69092 Loss: 110.345 +38400/69092 Loss: 108.975 +41600/69092 Loss: 108.031 +44800/69092 Loss: 110.949 +48000/69092 Loss: 108.343 +51200/69092 Loss: 110.290 +54400/69092 Loss: 112.152 +57600/69092 Loss: 111.389 +60800/69092 Loss: 108.413 +64000/69092 Loss: 109.156 +67200/69092 Loss: 110.323 +Training time 0:08:14.276140 +Epoch: 4 Average loss: 109.80 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 897) +0/69092 Loss: 111.066 +3200/69092 Loss: 108.904 +6400/69092 Loss: 111.266 +9600/69092 Loss: 110.323 +12800/69092 Loss: 109.788 +16000/69092 Loss: 110.000 +19200/69092 Loss: 109.390 +22400/69092 Loss: 111.585 +25600/69092 Loss: 107.813 +28800/69092 Loss: 109.511 +32000/69092 Loss: 109.187 +35200/69092 Loss: 108.621 +38400/69092 Loss: 108.665 +41600/69092 Loss: 110.370 +44800/69092 Loss: 109.704 +48000/69092 Loss: 109.402 +51200/69092 Loss: 108.475 +54400/69092 Loss: 111.507 +57600/69092 Loss: 109.633 +60800/69092 Loss: 111.263 +64000/69092 Loss: 110.222 +67200/69092 Loss: 109.318 +Training time 0:08:35.101683 +Epoch: 5 Average loss: 109.82 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 898) +0/69092 Loss: 102.434 +3200/69092 Loss: 108.549 +6400/69092 Loss: 110.515 +9600/69092 Loss: 109.571 +12800/69092 Loss: 108.974 +16000/69092 Loss: 108.870 +19200/69092 Loss: 110.333 +22400/69092 Loss: 108.688 +25600/69092 Loss: 109.629 +28800/69092 Loss: 109.930 +32000/69092 Loss: 109.784 +35200/69092 Loss: 110.607 +38400/69092 Loss: 109.851 +41600/69092 Loss: 110.287 +44800/69092 Loss: 109.118 +48000/69092 Loss: 111.187 +51200/69092 Loss: 110.033 +54400/69092 Loss: 109.707 +57600/69092 Loss: 111.196 +60800/69092 Loss: 112.121 +64000/69092 Loss: 111.448 +67200/69092 Loss: 110.298 +Training time 0:08:24.229024 +Epoch: 6 Average loss: 110.07 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 899) +0/69092 Loss: 106.925 +3200/69092 Loss: 110.441 +6400/69092 Loss: 110.143 +9600/69092 Loss: 108.793 +12800/69092 Loss: 110.355 +16000/69092 Loss: 110.902 +19200/69092 Loss: 111.763 +22400/69092 Loss: 110.550 +25600/69092 Loss: 110.683 +28800/69092 Loss: 110.057 +32000/69092 Loss: 110.056 +35200/69092 Loss: 109.016 +38400/69092 Loss: 108.872 +41600/69092 Loss: 111.028 +44800/69092 Loss: 108.918 +48000/69092 Loss: 109.556 +51200/69092 Loss: 110.492 +54400/69092 Loss: 109.169 +57600/69092 Loss: 109.681 +60800/69092 Loss: 110.108 +64000/69092 Loss: 106.451 +67200/69092 Loss: 111.016 +Training time 0:08:12.867429 +Epoch: 7 Average loss: 109.87 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 900) +0/69092 Loss: 104.499 +3200/69092 Loss: 110.522 +6400/69092 Loss: 109.600 +9600/69092 Loss: 109.528 +12800/69092 Loss: 109.834 +16000/69092 Loss: 110.322 +19200/69092 Loss: 110.263 +22400/69092 Loss: 110.173 +25600/69092 Loss: 111.851 +28800/69092 Loss: 108.172 +32000/69092 Loss: 108.699 +35200/69092 Loss: 108.834 +38400/69092 Loss: 109.039 +41600/69092 Loss: 110.192 +44800/69092 Loss: 110.667 +48000/69092 Loss: 109.423 +51200/69092 Loss: 109.373 +54400/69092 Loss: 109.808 +57600/69092 Loss: 109.878 +60800/69092 Loss: 109.106 +64000/69092 Loss: 109.970 +67200/69092 Loss: 111.754 +Training time 0:08:15.733421 +Epoch: 8 Average loss: 109.86 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 901) +0/69092 Loss: 109.801 +3200/69092 Loss: 107.471 +6400/69092 Loss: 110.665 +9600/69092 Loss: 109.222 +12800/69092 Loss: 109.381 +16000/69092 Loss: 109.497 +19200/69092 Loss: 109.513 +22400/69092 Loss: 110.913 +25600/69092 Loss: 110.735 +28800/69092 Loss: 108.249 +32000/69092 Loss: 109.736 +35200/69092 Loss: 108.231 +38400/69092 Loss: 110.026 +41600/69092 Loss: 108.559 +44800/69092 Loss: 109.407 +48000/69092 Loss: 109.920 +51200/69092 Loss: 111.791 +54400/69092 Loss: 109.308 +57600/69092 Loss: 111.891 +60800/69092 Loss: 109.453 +64000/69092 Loss: 110.515 +67200/69092 Loss: 109.339 +Training time 0:08:38.954356 +Epoch: 9 Average loss: 109.79 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 902) +0/69092 Loss: 96.886 +3200/69092 Loss: 108.678 +6400/69092 Loss: 109.940 +9600/69092 Loss: 109.740 +12800/69092 Loss: 109.167 +16000/69092 Loss: 109.458 +19200/69092 Loss: 111.670 +22400/69092 Loss: 110.005 +25600/69092 Loss: 108.490 +28800/69092 Loss: 110.023 +32000/69092 Loss: 111.877 +35200/69092 Loss: 110.076 +38400/69092 Loss: 108.383 +41600/69092 Loss: 110.857 +44800/69092 Loss: 108.288 +48000/69092 Loss: 111.406 +51200/69092 Loss: 112.466 +54400/69092 Loss: 109.424 +57600/69092 Loss: 108.588 +60800/69092 Loss: 110.404 +64000/69092 Loss: 111.132 +67200/69092 Loss: 110.336 +Training time 0:08:27.222467 +Epoch: 10 Average loss: 109.99 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 903) +0/69092 Loss: 117.604 +3200/69092 Loss: 109.267 +6400/69092 Loss: 109.354 +9600/69092 Loss: 108.803 +12800/69092 Loss: 110.667 +16000/69092 Loss: 109.150 +19200/69092 Loss: 109.900 +22400/69092 Loss: 110.840 +25600/69092 Loss: 108.930 +28800/69092 Loss: 109.228 +32000/69092 Loss: 109.978 +35200/69092 Loss: 108.748 +38400/69092 Loss: 108.769 +41600/69092 Loss: 111.524 +44800/69092 Loss: 110.138 +48000/69092 Loss: 111.239 +51200/69092 Loss: 108.361 +54400/69092 Loss: 110.076 +57600/69092 Loss: 112.542 +60800/69092 Loss: 109.636 +64000/69092 Loss: 108.758 +67200/69092 Loss: 108.637 +Training time 0:08:13.154020 +Epoch: 11 Average loss: 109.78 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 904) +0/69092 Loss: 108.394 +3200/69092 Loss: 109.591 +6400/69092 Loss: 109.260 +9600/69092 Loss: 109.824 +12800/69092 Loss: 108.291 +16000/69092 Loss: 109.754 +19200/69092 Loss: 109.808 +22400/69092 Loss: 110.037 +25600/69092 Loss: 110.108 +28800/69092 Loss: 109.019 +32000/69092 Loss: 109.431 +35200/69092 Loss: 110.360 +38400/69092 Loss: 109.336 +41600/69092 Loss: 108.839 +44800/69092 Loss: 111.310 +48000/69092 Loss: 110.200 +51200/69092 Loss: 109.521 +54400/69092 Loss: 111.582 +57600/69092 Loss: 110.712 +60800/69092 Loss: 109.611 +64000/69092 Loss: 109.852 +67200/69092 Loss: 107.889 +Training time 0:08:30.258220 +Epoch: 12 Average loss: 109.75 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 905) +0/69092 Loss: 108.899 +3200/69092 Loss: 110.286 +6400/69092 Loss: 110.807 +9600/69092 Loss: 109.053 +12800/69092 Loss: 110.819 +16000/69092 Loss: 109.535 +19200/69092 Loss: 109.243 +22400/69092 Loss: 111.400 +25600/69092 Loss: 109.584 +28800/69092 Loss: 109.922 +32000/69092 Loss: 109.843 +35200/69092 Loss: 110.877 +38400/69092 Loss: 109.381 +41600/69092 Loss: 108.467 +44800/69092 Loss: 110.268 +48000/69092 Loss: 108.943 +51200/69092 Loss: 107.386 +54400/69092 Loss: 110.501 +57600/69092 Loss: 110.936 +60800/69092 Loss: 107.051 +64000/69092 Loss: 109.743 +67200/69092 Loss: 110.801 +Training time 0:08:34.919019 +Epoch: 13 Average loss: 109.75 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 906) +0/69092 Loss: 106.480 +3200/69092 Loss: 108.637 +6400/69092 Loss: 111.363 +9600/69092 Loss: 108.151 +12800/69092 Loss: 111.287 +16000/69092 Loss: 109.215 +19200/69092 Loss: 110.570 +22400/69092 Loss: 109.414 +25600/69092 Loss: 110.231 +28800/69092 Loss: 108.371 +32000/69092 Loss: 110.356 +35200/69092 Loss: 108.899 +38400/69092 Loss: 111.158 +41600/69092 Loss: 109.454 +44800/69092 Loss: 109.332 +48000/69092 Loss: 110.420 +51200/69092 Loss: 109.121 +54400/69092 Loss: 110.583 +57600/69092 Loss: 111.224 +60800/69092 Loss: 111.062 +64000/69092 Loss: 107.604 +67200/69092 Loss: 109.295 +Training time 0:08:12.988694 +Epoch: 14 Average loss: 109.80 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 907) +0/69092 Loss: 113.552 +3200/69092 Loss: 109.272 +6400/69092 Loss: 109.035 +9600/69092 Loss: 110.843 +12800/69092 Loss: 108.581 +16000/69092 Loss: 110.015 +19200/69092 Loss: 109.440 +22400/69092 Loss: 108.619 +25600/69092 Loss: 108.504 +28800/69092 Loss: 110.964 +32000/69092 Loss: 110.685 +35200/69092 Loss: 110.124 +38400/69092 Loss: 109.331 +41600/69092 Loss: 109.444 +44800/69092 Loss: 110.493 +48000/69092 Loss: 109.717 +51200/69092 Loss: 110.052 +54400/69092 Loss: 109.734 +57600/69092 Loss: 110.113 +60800/69092 Loss: 110.238 +64000/69092 Loss: 110.355 +67200/69092 Loss: 110.088 +Training time 0:08:17.855916 +Epoch: 15 Average loss: 109.79 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 908) +0/69092 Loss: 115.582 +3200/69092 Loss: 111.049 +6400/69092 Loss: 108.889 +9600/69092 Loss: 110.467 +12800/69092 Loss: 109.753 +16000/69092 Loss: 109.980 +19200/69092 Loss: 110.101 +22400/69092 Loss: 108.659 +25600/69092 Loss: 109.954 +28800/69092 Loss: 110.273 +32000/69092 Loss: 111.276 +35200/69092 Loss: 110.227 +38400/69092 Loss: 109.020 +41600/69092 Loss: 109.429 +44800/69092 Loss: 109.395 +48000/69092 Loss: 108.223 +51200/69092 Loss: 111.319 +54400/69092 Loss: 109.162 +57600/69092 Loss: 107.795 +60800/69092 Loss: 109.384 +64000/69092 Loss: 109.929 +67200/69092 Loss: 110.769 +Training time 0:08:22.153288 +Epoch: 16 Average loss: 109.70 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 909) +0/69092 Loss: 110.604 +3200/69092 Loss: 109.816 +6400/69092 Loss: 110.551 +9600/69092 Loss: 108.674 +12800/69092 Loss: 109.234 +16000/69092 Loss: 110.181 +19200/69092 Loss: 109.667 +22400/69092 Loss: 110.132 +25600/69092 Loss: 109.124 +28800/69092 Loss: 109.338 +32000/69092 Loss: 110.111 +35200/69092 Loss: 110.614 +38400/69092 Loss: 110.279 +41600/69092 Loss: 108.266 +44800/69092 Loss: 109.889 +48000/69092 Loss: 111.837 +51200/69092 Loss: 110.700 +54400/69092 Loss: 109.877 +57600/69092 Loss: 110.370 +60800/69092 Loss: 109.595 +64000/69092 Loss: 107.575 +67200/69092 Loss: 107.701 +Training time 0:08:12.541597 +Epoch: 17 Average loss: 109.74 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 910) +0/69092 Loss: 113.704 +3200/69092 Loss: 108.796 +6400/69092 Loss: 110.130 +9600/69092 Loss: 109.383 +12800/69092 Loss: 109.466 +16000/69092 Loss: 110.090 +19200/69092 Loss: 109.447 +22400/69092 Loss: 110.780 +25600/69092 Loss: 109.133 +28800/69092 Loss: 109.416 +32000/69092 Loss: 108.995 +35200/69092 Loss: 109.165 +38400/69092 Loss: 110.280 +41600/69092 Loss: 110.049 +44800/69092 Loss: 111.023 +48000/69092 Loss: 110.316 +51200/69092 Loss: 109.759 +54400/69092 Loss: 110.034 +57600/69092 Loss: 109.265 +60800/69092 Loss: 109.184 +64000/69092 Loss: 109.522 +67200/69092 Loss: 110.263 +Training time 0:08:14.583313 +Epoch: 18 Average loss: 109.77 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 911) +0/69092 Loss: 108.895 +3200/69092 Loss: 109.986 +6400/69092 Loss: 109.630 +9600/69092 Loss: 111.963 +12800/69092 Loss: 111.758 +16000/69092 Loss: 108.648 +19200/69092 Loss: 109.151 +22400/69092 Loss: 109.271 +25600/69092 Loss: 109.333 +28800/69092 Loss: 110.728 +32000/69092 Loss: 110.444 +35200/69092 Loss: 109.595 +38400/69092 Loss: 109.363 +41600/69092 Loss: 110.533 +44800/69092 Loss: 108.189 +48000/69092 Loss: 108.910 +51200/69092 Loss: 110.093 +54400/69092 Loss: 108.096 +57600/69092 Loss: 109.775 +60800/69092 Loss: 109.771 +64000/69092 Loss: 111.250 +67200/69092 Loss: 109.601 +Training time 0:08:31.719230 +Epoch: 19 Average loss: 109.81 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 912) +0/69092 Loss: 120.258 +3200/69092 Loss: 109.774 +6400/69092 Loss: 109.994 +9600/69092 Loss: 109.507 +12800/69092 Loss: 109.003 +16000/69092 Loss: 108.244 +19200/69092 Loss: 108.104 +22400/69092 Loss: 108.867 +25600/69092 Loss: 110.178 +28800/69092 Loss: 110.568 +32000/69092 Loss: 110.986 +35200/69092 Loss: 109.078 +38400/69092 Loss: 110.728 +41600/69092 Loss: 109.283 +44800/69092 Loss: 108.889 +48000/69092 Loss: 110.796 +51200/69092 Loss: 109.935 +54400/69092 Loss: 111.437 +57600/69092 Loss: 108.444 +60800/69092 Loss: 111.464 +64000/69092 Loss: 110.165 +67200/69092 Loss: 111.627 +Training time 0:08:23.221328 +Epoch: 20 Average loss: 109.89 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 913) +0/69092 Loss: 108.357 +3200/69092 Loss: 111.646 +6400/69092 Loss: 108.523 +9600/69092 Loss: 110.985 +12800/69092 Loss: 110.621 +16000/69092 Loss: 109.441 +19200/69092 Loss: 110.316 +22400/69092 Loss: 109.373 +25600/69092 Loss: 108.827 +28800/69092 Loss: 109.140 +32000/69092 Loss: 107.578 +35200/69092 Loss: 111.282 +38400/69092 Loss: 109.706 +41600/69092 Loss: 108.787 +44800/69092 Loss: 110.955 +48000/69092 Loss: 110.700 +51200/69092 Loss: 109.682 +54400/69092 Loss: 110.392 +57600/69092 Loss: 109.227 +60800/69092 Loss: 109.500 +64000/69092 Loss: 108.587 +67200/69092 Loss: 108.766 +Training time 0:08:15.513935 +Epoch: 21 Average loss: 109.75 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 914) +0/69092 Loss: 108.012 +3200/69092 Loss: 109.785 +6400/69092 Loss: 108.768 +9600/69092 Loss: 107.534 +12800/69092 Loss: 108.122 +16000/69092 Loss: 108.332 +19200/69092 Loss: 110.009 +22400/69092 Loss: 109.827 +25600/69092 Loss: 109.433 +28800/69092 Loss: 110.671 +32000/69092 Loss: 108.734 +35200/69092 Loss: 108.370 +38400/69092 Loss: 110.615 +41600/69092 Loss: 111.236 +44800/69092 Loss: 110.378 +48000/69092 Loss: 110.068 +51200/69092 Loss: 110.537 +54400/69092 Loss: 109.612 +57600/69092 Loss: 108.842 +60800/69092 Loss: 109.957 +64000/69092 Loss: 110.722 +67200/69092 Loss: 109.085 +Training time 0:08:29.323857 +Epoch: 22 Average loss: 109.61 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 915) +0/69092 Loss: 98.606 +3200/69092 Loss: 111.806 +6400/69092 Loss: 108.681 +9600/69092 Loss: 110.218 +12800/69092 Loss: 111.459 +16000/69092 Loss: 110.256 +19200/69092 Loss: 109.856 +22400/69092 Loss: 108.792 +25600/69092 Loss: 109.350 +28800/69092 Loss: 109.909 +32000/69092 Loss: 109.159 +35200/69092 Loss: 110.690 +38400/69092 Loss: 108.486 +41600/69092 Loss: 109.467 +44800/69092 Loss: 108.747 +48000/69092 Loss: 111.261 +51200/69092 Loss: 109.100 +54400/69092 Loss: 109.166 +57600/69092 Loss: 110.897 +60800/69092 Loss: 109.595 +64000/69092 Loss: 110.913 +67200/69092 Loss: 110.218 +Training time 0:08:43.886611 +Epoch: 23 Average loss: 109.86 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 916) +0/69092 Loss: 102.628 +3200/69092 Loss: 109.790 +6400/69092 Loss: 110.513 +9600/69092 Loss: 109.577 +12800/69092 Loss: 110.755 +16000/69092 Loss: 109.130 +19200/69092 Loss: 109.957 +22400/69092 Loss: 108.203 +25600/69092 Loss: 111.515 +28800/69092 Loss: 108.747 +32000/69092 Loss: 109.503 +35200/69092 Loss: 110.292 +38400/69092 Loss: 111.143 +41600/69092 Loss: 108.800 +44800/69092 Loss: 111.631 +48000/69092 Loss: 109.205 +51200/69092 Loss: 109.387 +54400/69092 Loss: 109.282 +57600/69092 Loss: 111.000 +60800/69092 Loss: 109.470 +64000/69092 Loss: 108.622 +67200/69092 Loss: 108.642 +Training time 0:08:17.169422 +Epoch: 24 Average loss: 109.83 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 917) +0/69092 Loss: 112.609 +3200/69092 Loss: 109.214 +6400/69092 Loss: 110.045 +9600/69092 Loss: 110.177 +12800/69092 Loss: 110.006 +16000/69092 Loss: 110.071 +19200/69092 Loss: 110.067 +22400/69092 Loss: 111.286 +25600/69092 Loss: 109.371 +28800/69092 Loss: 108.579 +32000/69092 Loss: 109.009 +35200/69092 Loss: 109.586 +38400/69092 Loss: 111.596 +41600/69092 Loss: 109.845 +44800/69092 Loss: 108.395 +48000/69092 Loss: 109.349 +51200/69092 Loss: 110.376 +54400/69092 Loss: 110.299 +57600/69092 Loss: 108.934 +60800/69092 Loss: 109.750 +64000/69092 Loss: 109.232 +67200/69092 Loss: 109.902 +Training time 0:08:24.376567 +Epoch: 25 Average loss: 109.82 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 918) +0/69092 Loss: 101.586 +3200/69092 Loss: 109.017 +6400/69092 Loss: 110.772 +9600/69092 Loss: 110.224 +12800/69092 Loss: 108.636 +16000/69092 Loss: 109.983 +19200/69092 Loss: 110.476 +22400/69092 Loss: 110.637 +25600/69092 Loss: 109.178 +28800/69092 Loss: 110.094 +32000/69092 Loss: 109.600 +35200/69092 Loss: 108.152 +38400/69092 Loss: 109.064 +41600/69092 Loss: 110.805 +44800/69092 Loss: 109.737 +48000/69092 Loss: 109.623 +51200/69092 Loss: 108.737 +54400/69092 Loss: 110.510 +57600/69092 Loss: 108.234 +60800/69092 Loss: 109.183 +64000/69092 Loss: 111.722 +67200/69092 Loss: 108.152 +Training time 0:08:37.100633 +Epoch: 26 Average loss: 109.63 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 919) +0/69092 Loss: 106.414 +3200/69092 Loss: 109.265 +6400/69092 Loss: 109.205 +9600/69092 Loss: 109.474 +12800/69092 Loss: 109.728 +16000/69092 Loss: 108.176 +19200/69092 Loss: 110.381 +22400/69092 Loss: 110.187 +25600/69092 Loss: 110.399 +28800/69092 Loss: 110.505 +32000/69092 Loss: 107.933 +35200/69092 Loss: 109.721 +38400/69092 Loss: 109.225 +41600/69092 Loss: 112.266 +44800/69092 Loss: 108.616 +48000/69092 Loss: 110.297 +51200/69092 Loss: 110.420 +54400/69092 Loss: 111.526 +57600/69092 Loss: 108.996 +60800/69092 Loss: 109.643 +64000/69092 Loss: 109.631 +67200/69092 Loss: 109.490 +Training time 0:08:26.801735 +Epoch: 27 Average loss: 109.75 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 920) +0/69092 Loss: 115.588 +3200/69092 Loss: 108.981 +6400/69092 Loss: 111.154 +9600/69092 Loss: 110.157 +12800/69092 Loss: 109.135 +16000/69092 Loss: 110.038 +19200/69092 Loss: 109.615 +22400/69092 Loss: 109.590 +25600/69092 Loss: 110.713 +28800/69092 Loss: 107.865 +32000/69092 Loss: 109.370 +35200/69092 Loss: 108.198 +38400/69092 Loss: 108.758 +41600/69092 Loss: 111.193 +44800/69092 Loss: 108.665 +48000/69092 Loss: 108.259 +51200/69092 Loss: 110.544 +54400/69092 Loss: 110.299 +57600/69092 Loss: 110.564 +60800/69092 Loss: 108.609 +64000/69092 Loss: 108.380 +67200/69092 Loss: 111.460 +Training time 0:08:10.084292 +Epoch: 28 Average loss: 109.60 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 921) +0/69092 Loss: 115.302 +3200/69092 Loss: 108.736 +6400/69092 Loss: 109.284 +9600/69092 Loss: 109.514 +12800/69092 Loss: 110.094 +16000/69092 Loss: 108.533 +19200/69092 Loss: 109.507 +22400/69092 Loss: 109.770 +25600/69092 Loss: 110.592 +28800/69092 Loss: 110.646 +32000/69092 Loss: 109.315 +35200/69092 Loss: 109.352 +38400/69092 Loss: 108.755 +41600/69092 Loss: 111.472 +44800/69092 Loss: 110.371 +48000/69092 Loss: 109.554 +51200/69092 Loss: 108.482 +54400/69092 Loss: 108.490 +57600/69092 Loss: 110.220 +60800/69092 Loss: 109.001 +64000/69092 Loss: 110.251 +67200/69092 Loss: 110.991 +Training time 0:08:33.553547 +Epoch: 29 Average loss: 109.66 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 922) +0/69092 Loss: 109.241 +3200/69092 Loss: 108.647 +6400/69092 Loss: 108.049 +9600/69092 Loss: 109.502 +12800/69092 Loss: 109.735 +16000/69092 Loss: 111.508 +19200/69092 Loss: 110.922 +22400/69092 Loss: 109.205 +25600/69092 Loss: 109.838 +28800/69092 Loss: 110.626 +32000/69092 Loss: 109.786 +35200/69092 Loss: 110.763 +38400/69092 Loss: 109.427 +41600/69092 Loss: 108.606 +44800/69092 Loss: 109.792 +48000/69092 Loss: 109.158 +51200/69092 Loss: 110.283 +54400/69092 Loss: 109.445 +57600/69092 Loss: 110.223 +60800/69092 Loss: 108.971 +64000/69092 Loss: 111.724 +67200/69092 Loss: 109.088 +Training time 0:08:27.126029 +Epoch: 30 Average loss: 109.77 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 923) +0/69092 Loss: 116.686 +3200/69092 Loss: 110.322 +6400/69092 Loss: 109.907 +9600/69092 Loss: 109.940 +12800/69092 Loss: 111.581 +16000/69092 Loss: 109.236 +19200/69092 Loss: 108.935 +22400/69092 Loss: 108.903 +25600/69092 Loss: 110.447 +28800/69092 Loss: 108.463 +32000/69092 Loss: 110.719 +35200/69092 Loss: 108.835 +38400/69092 Loss: 110.248 +41600/69092 Loss: 110.756 +44800/69092 Loss: 108.916 +48000/69092 Loss: 109.694 +51200/69092 Loss: 109.632 +54400/69092 Loss: 109.935 +57600/69092 Loss: 110.132 +60800/69092 Loss: 110.983 +64000/69092 Loss: 109.134 +67200/69092 Loss: 109.580 +Training time 0:08:20.480800 +Epoch: 31 Average loss: 109.86 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 924) +0/69092 Loss: 110.905 +3200/69092 Loss: 109.653 +6400/69092 Loss: 109.556 +9600/69092 Loss: 110.120 +12800/69092 Loss: 109.615 +16000/69092 Loss: 111.569 +19200/69092 Loss: 110.255 +22400/69092 Loss: 107.942 +25600/69092 Loss: 110.008 +28800/69092 Loss: 108.796 +32000/69092 Loss: 110.053 +35200/69092 Loss: 109.828 +38400/69092 Loss: 109.237 +41600/69092 Loss: 109.805 +44800/69092 Loss: 108.382 +48000/69092 Loss: 109.038 +51200/69092 Loss: 110.557 +54400/69092 Loss: 110.062 +57600/69092 Loss: 108.280 +60800/69092 Loss: 110.407 +64000/69092 Loss: 109.928 +67200/69092 Loss: 108.921 +Training time 0:08:20.143426 +Epoch: 32 Average loss: 109.68 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 925) +0/69092 Loss: 114.821 +3200/69092 Loss: 108.729 +6400/69092 Loss: 110.943 +9600/69092 Loss: 108.081 +12800/69092 Loss: 108.091 +16000/69092 Loss: 109.974 +19200/69092 Loss: 110.299 +22400/69092 Loss: 110.198 +25600/69092 Loss: 109.387 +28800/69092 Loss: 109.245 +32000/69092 Loss: 110.016 +35200/69092 Loss: 109.274 +38400/69092 Loss: 109.848 +41600/69092 Loss: 109.667 +44800/69092 Loss: 109.867 +48000/69092 Loss: 108.490 +51200/69092 Loss: 110.273 +54400/69092 Loss: 110.828 +57600/69092 Loss: 110.311 +60800/69092 Loss: 109.978 +64000/69092 Loss: 110.202 +67200/69092 Loss: 109.773 +Training time 0:08:29.862622 +Epoch: 33 Average loss: 109.76 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 926) +0/69092 Loss: 106.247 +3200/69092 Loss: 109.189 +6400/69092 Loss: 110.799 +9600/69092 Loss: 111.419 +12800/69092 Loss: 108.602 +16000/69092 Loss: 110.871 +19200/69092 Loss: 108.889 +22400/69092 Loss: 106.634 +25600/69092 Loss: 109.990 +28800/69092 Loss: 111.041 +32000/69092 Loss: 110.678 +35200/69092 Loss: 108.914 +38400/69092 Loss: 111.101 +41600/69092 Loss: 108.035 +44800/69092 Loss: 110.859 +48000/69092 Loss: 109.421 +51200/69092 Loss: 109.238 +54400/69092 Loss: 109.267 +57600/69092 Loss: 109.837 +60800/69092 Loss: 108.726 +64000/69092 Loss: 110.032 +67200/69092 Loss: 109.824 +Training time 0:08:21.746606 +Epoch: 34 Average loss: 109.76 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 927) +0/69092 Loss: 113.420 +3200/69092 Loss: 108.717 +6400/69092 Loss: 109.250 +9600/69092 Loss: 108.531 +12800/69092 Loss: 111.823 +16000/69092 Loss: 108.119 +19200/69092 Loss: 110.376 +22400/69092 Loss: 108.626 +25600/69092 Loss: 110.165 +28800/69092 Loss: 109.598 +32000/69092 Loss: 109.473 +35200/69092 Loss: 109.605 +38400/69092 Loss: 108.955 +41600/69092 Loss: 108.566 +44800/69092 Loss: 108.869 +48000/69092 Loss: 109.015 +51200/69092 Loss: 109.486 +54400/69092 Loss: 108.909 +57600/69092 Loss: 109.819 +60800/69092 Loss: 112.781 +64000/69092 Loss: 110.751 +67200/69092 Loss: 110.591 +Training time 0:08:09.939992 +Epoch: 35 Average loss: 109.62 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 928) +0/69092 Loss: 112.473 +3200/69092 Loss: 108.880 +6400/69092 Loss: 110.422 +9600/69092 Loss: 108.548 +12800/69092 Loss: 108.895 +16000/69092 Loss: 108.475 +19200/69092 Loss: 109.079 +22400/69092 Loss: 109.963 +25600/69092 Loss: 110.722 +28800/69092 Loss: 109.424 +32000/69092 Loss: 110.036 +35200/69092 Loss: 109.287 +38400/69092 Loss: 110.355 +41600/69092 Loss: 109.804 +44800/69092 Loss: 110.613 +48000/69092 Loss: 108.773 +51200/69092 Loss: 108.909 +54400/69092 Loss: 108.376 +57600/69092 Loss: 109.938 +60800/69092 Loss: 111.428 +64000/69092 Loss: 111.244 +67200/69092 Loss: 109.535 +Training time 0:08:34.925414 +Epoch: 36 Average loss: 109.65 +=> saved checkpoint 'trained_models/rendered_chairs/VAE_bs_64/checkpoints/last' (iter 929) +0/69092 Loss: 104.706 +3200/69092 Loss: 108.327 +6400/69092 Loss: 108.243 +9600/69092 Loss: 109.821 +12800/69092 Loss: 111.070 +16000/69092 Loss: 108.779 +19200/69092 Loss: 109.712 +22400/69092 Loss: 109.174 +25600/69092 Loss: 111.046 +28800/69092 Loss: 109.639 +32000/69092 Loss: 109.781 +35200/69092 Loss: 108.534 +38400/69092 Loss: 110.820 +41600/69092 Loss: 111.755 +44800/69092 Loss: 107.944 +48000/69092 Loss: 112.064 +51200/69092 Loss: 110.967 +54400/69092 Loss: 109.786 diff --git a/OAR.2071932.stderr b/OAR.2071932.stderr new file mode 100644 index 0000000000000000000000000000000000000000..a9c628a29c58ebc65df754dd5dfbbd99e2282b5f --- /dev/null +++ b/OAR.2071932.stderr @@ -0,0 +1,2 @@ +/data1/home/julien.dejasmin/anaconda3/envs/pytorch/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead. + warnings.warn(warning.format(ret)) diff --git a/OAR.2071932.stdout b/OAR.2071932.stdout new file mode 100644 index 0000000000000000000000000000000000000000..a8cc4bdf572decb852ca3060e55f111071b181f7 --- /dev/null +++ b/OAR.2071932.stdout @@ -0,0 +1,909 @@ +Namespace(batch_size=64, beta=4, ckpt_dir='checkpoints', ckpt_name='last', cont_capacity=None, dataset='rendered_chairs', disc_capacity=None, epochs=400, experiment_name='beta_VAE_bs_64_ls_15', gpu_devices=[0, 1], is_beta_VAE=True, latent_spec_cont=15, latent_spec_disc=None, load_expe_name='', load_model_checkpoint=True, lr=1e-05, nb_filter_conv1=32, nb_filter_conv2=32, nb_filter_conv3=64, nb_filter_conv4=64, num_worker=1, print_loss_every=50, record_loss_every=50, save_model=True, save_reconstruction_image=False, save_step=1, verbose=True) +CUDA Visible devices ! +load dataset: rendered_chairs, with: 69120 train images of shape: (3, 64, 64) +use 2 gpu who named: +GeForce RTX 2080 Ti +GeForce RTX 2080 Ti +DataParallel( + (module): VAE( + (img_to_last_conv): Sequential( + (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): Conv2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + ) + (last_conv_to_continuous_features): Sequential( + (0): Conv2d(64, 256, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + ) + (features_to_hidden_continue): Sequential( + (0): Linear(in_features=256, out_features=30, bias=True) + (1): ReLU() + ) + (latent_to_features): Sequential( + (0): Linear(in_features=15, out_features=256, bias=True) + (1): ReLU() + ) + (features_to_img): Sequential( + (0): ConvTranspose2d(256, 64, kernel_size=(4, 4), stride=(1, 1)) + (1): ReLU() + (2): ConvTranspose2d(64, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (3): ReLU() + (4): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (5): ReLU() + (6): ConvTranspose2d(32, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (7): ReLU() + (8): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1)) + (9): Sigmoid() + ) + ) +) +The number of parameters of model is 769185 +don't use continuous capacity +=> loaded checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last (iter 384)' +0/69092 Loss: 152.361 +3200/69092 Loss: 148.499 +6400/69092 Loss: 149.480 +9600/69092 Loss: 147.846 +12800/69092 Loss: 151.184 +16000/69092 Loss: 149.320 +19200/69092 Loss: 147.923 +22400/69092 Loss: 147.379 +25600/69092 Loss: 148.429 +28800/69092 Loss: 147.844 +32000/69092 Loss: 148.419 +35200/69092 Loss: 145.906 +38400/69092 Loss: 147.226 +41600/69092 Loss: 149.770 +44800/69092 Loss: 147.473 +48000/69092 Loss: 147.804 +51200/69092 Loss: 148.531 +54400/69092 Loss: 146.512 +57600/69092 Loss: 147.608 +60800/69092 Loss: 148.155 +64000/69092 Loss: 150.530 +67200/69092 Loss: 150.236 +Training time 0:08:48.207744 +Epoch: 1 Average loss: 148.49 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 385) +0/69092 Loss: 150.210 +3200/69092 Loss: 148.198 +6400/69092 Loss: 147.188 +9600/69092 Loss: 147.347 +12800/69092 Loss: 148.467 +16000/69092 Loss: 150.042 +19200/69092 Loss: 148.443 +22400/69092 Loss: 147.597 +25600/69092 Loss: 148.485 +28800/69092 Loss: 146.093 +32000/69092 Loss: 148.259 +35200/69092 Loss: 148.657 +38400/69092 Loss: 148.375 +41600/69092 Loss: 147.089 +44800/69092 Loss: 150.649 +48000/69092 Loss: 149.635 +51200/69092 Loss: 150.797 +54400/69092 Loss: 147.939 +57600/69092 Loss: 151.415 +60800/69092 Loss: 147.258 +64000/69092 Loss: 150.691 +67200/69092 Loss: 151.794 +Training time 0:08:57.550437 +Epoch: 2 Average loss: 148.75 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 386) +0/69092 Loss: 152.646 +3200/69092 Loss: 148.428 +6400/69092 Loss: 146.480 +9600/69092 Loss: 147.692 +12800/69092 Loss: 144.389 +16000/69092 Loss: 148.671 +19200/69092 Loss: 149.079 +22400/69092 Loss: 151.463 +25600/69092 Loss: 148.722 +28800/69092 Loss: 148.784 +32000/69092 Loss: 148.804 +35200/69092 Loss: 148.886 +38400/69092 Loss: 149.955 +41600/69092 Loss: 149.098 +44800/69092 Loss: 148.989 +48000/69092 Loss: 147.723 +51200/69092 Loss: 149.765 +54400/69092 Loss: 151.254 +57600/69092 Loss: 151.866 +60800/69092 Loss: 147.102 +64000/69092 Loss: 146.083 +67200/69092 Loss: 147.822 +Training time 0:08:51.299947 +Epoch: 3 Average loss: 148.75 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 387) +0/69092 Loss: 142.594 +3200/69092 Loss: 148.032 +6400/69092 Loss: 149.080 +9600/69092 Loss: 147.561 +12800/69092 Loss: 149.002 +16000/69092 Loss: 147.063 +19200/69092 Loss: 150.164 +22400/69092 Loss: 150.987 +25600/69092 Loss: 147.199 +28800/69092 Loss: 150.476 +32000/69092 Loss: 147.531 +35200/69092 Loss: 146.436 +38400/69092 Loss: 147.448 +41600/69092 Loss: 150.520 +44800/69092 Loss: 151.851 +48000/69092 Loss: 148.633 +51200/69092 Loss: 147.498 +54400/69092 Loss: 148.287 +57600/69092 Loss: 150.259 +60800/69092 Loss: 147.854 +64000/69092 Loss: 148.655 +67200/69092 Loss: 148.532 +Training time 0:09:12.031770 +Epoch: 4 Average loss: 148.74 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 388) +0/69092 Loss: 146.292 +3200/69092 Loss: 146.458 +6400/69092 Loss: 148.207 +9600/69092 Loss: 149.125 +12800/69092 Loss: 148.913 +16000/69092 Loss: 151.184 +19200/69092 Loss: 148.501 +22400/69092 Loss: 148.748 +25600/69092 Loss: 149.945 +28800/69092 Loss: 148.486 +32000/69092 Loss: 149.766 +35200/69092 Loss: 144.062 +38400/69092 Loss: 146.422 +41600/69092 Loss: 147.301 +44800/69092 Loss: 148.002 +48000/69092 Loss: 149.905 +51200/69092 Loss: 148.207 +54400/69092 Loss: 149.714 +57600/69092 Loss: 148.578 +60800/69092 Loss: 149.743 +64000/69092 Loss: 149.386 +67200/69092 Loss: 148.032 +Training time 0:09:00.591673 +Epoch: 5 Average loss: 148.59 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 389) +0/69092 Loss: 146.621 +3200/69092 Loss: 150.144 +6400/69092 Loss: 150.271 +9600/69092 Loss: 148.271 +12800/69092 Loss: 149.640 +16000/69092 Loss: 148.857 +19200/69092 Loss: 147.642 +22400/69092 Loss: 149.931 +25600/69092 Loss: 148.541 +28800/69092 Loss: 148.161 +32000/69092 Loss: 149.247 +35200/69092 Loss: 148.439 +38400/69092 Loss: 148.187 +41600/69092 Loss: 147.979 +44800/69092 Loss: 146.100 +48000/69092 Loss: 148.955 +51200/69092 Loss: 146.269 +54400/69092 Loss: 150.135 +57600/69092 Loss: 148.776 +60800/69092 Loss: 145.928 +64000/69092 Loss: 148.470 +67200/69092 Loss: 148.817 +Training time 0:08:44.529777 +Epoch: 6 Average loss: 148.57 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 390) +0/69092 Loss: 141.314 +3200/69092 Loss: 150.979 +6400/69092 Loss: 147.888 +9600/69092 Loss: 149.883 +12800/69092 Loss: 146.250 +16000/69092 Loss: 149.826 +19200/69092 Loss: 147.241 +22400/69092 Loss: 149.990 +25600/69092 Loss: 149.990 +28800/69092 Loss: 148.794 +32000/69092 Loss: 145.935 +35200/69092 Loss: 149.564 +38400/69092 Loss: 150.240 +41600/69092 Loss: 148.454 +44800/69092 Loss: 150.215 +48000/69092 Loss: 148.807 +51200/69092 Loss: 148.885 +54400/69092 Loss: 147.434 +57600/69092 Loss: 148.269 +60800/69092 Loss: 146.226 +64000/69092 Loss: 148.154 +67200/69092 Loss: 147.661 +Training time 0:08:52.399564 +Epoch: 7 Average loss: 148.59 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 391) +0/69092 Loss: 154.954 +3200/69092 Loss: 146.626 +6400/69092 Loss: 146.706 +9600/69092 Loss: 147.856 +12800/69092 Loss: 147.818 +16000/69092 Loss: 147.751 +19200/69092 Loss: 147.856 +22400/69092 Loss: 150.530 +25600/69092 Loss: 148.228 +28800/69092 Loss: 150.639 +32000/69092 Loss: 147.815 +35200/69092 Loss: 148.694 +38400/69092 Loss: 147.708 +41600/69092 Loss: 148.732 +44800/69092 Loss: 149.883 +48000/69092 Loss: 147.828 +51200/69092 Loss: 149.429 +54400/69092 Loss: 148.536 +57600/69092 Loss: 149.280 +60800/69092 Loss: 149.577 +64000/69092 Loss: 147.691 +67200/69092 Loss: 148.652 +Training time 0:09:05.264120 +Epoch: 8 Average loss: 148.52 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 392) +0/69092 Loss: 141.132 +3200/69092 Loss: 146.203 +6400/69092 Loss: 151.039 +9600/69092 Loss: 147.543 +12800/69092 Loss: 146.663 +16000/69092 Loss: 147.522 +19200/69092 Loss: 147.544 +22400/69092 Loss: 146.566 +25600/69092 Loss: 150.070 +28800/69092 Loss: 151.101 +32000/69092 Loss: 148.373 +35200/69092 Loss: 149.597 +38400/69092 Loss: 146.910 +41600/69092 Loss: 148.381 +44800/69092 Loss: 150.115 +48000/69092 Loss: 145.903 +51200/69092 Loss: 147.350 +54400/69092 Loss: 148.460 +57600/69092 Loss: 147.940 +60800/69092 Loss: 150.142 +64000/69092 Loss: 148.387 +67200/69092 Loss: 147.158 +Training time 0:09:03.795954 +Epoch: 9 Average loss: 148.23 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 393) +0/69092 Loss: 150.138 +3200/69092 Loss: 149.238 +6400/69092 Loss: 149.604 +9600/69092 Loss: 146.707 +12800/69092 Loss: 149.958 +16000/69092 Loss: 148.945 +19200/69092 Loss: 148.408 +22400/69092 Loss: 149.698 +25600/69092 Loss: 150.222 +28800/69092 Loss: 149.839 +32000/69092 Loss: 148.148 +35200/69092 Loss: 147.426 +38400/69092 Loss: 147.905 +41600/69092 Loss: 150.213 +44800/69092 Loss: 149.591 +48000/69092 Loss: 148.548 +51200/69092 Loss: 147.646 +54400/69092 Loss: 147.575 +57600/69092 Loss: 150.757 +60800/69092 Loss: 145.836 +64000/69092 Loss: 148.908 +67200/69092 Loss: 148.884 +Training time 0:09:06.431221 +Epoch: 10 Average loss: 148.72 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 394) +0/69092 Loss: 132.612 +3200/69092 Loss: 148.859 +6400/69092 Loss: 148.332 +9600/69092 Loss: 147.848 +12800/69092 Loss: 147.987 +16000/69092 Loss: 146.405 +19200/69092 Loss: 149.743 +22400/69092 Loss: 148.967 +25600/69092 Loss: 150.643 +28800/69092 Loss: 147.096 +32000/69092 Loss: 150.261 +35200/69092 Loss: 148.284 +38400/69092 Loss: 150.075 +41600/69092 Loss: 149.636 +44800/69092 Loss: 145.665 +48000/69092 Loss: 148.471 +51200/69092 Loss: 151.987 +54400/69092 Loss: 149.194 +57600/69092 Loss: 145.463 +60800/69092 Loss: 147.419 +64000/69092 Loss: 148.302 +67200/69092 Loss: 147.979 +Training time 0:08:55.323519 +Epoch: 11 Average loss: 148.51 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 395) +0/69092 Loss: 145.348 +3200/69092 Loss: 148.658 +6400/69092 Loss: 148.206 +9600/69092 Loss: 148.802 +12800/69092 Loss: 147.112 +16000/69092 Loss: 147.435 +19200/69092 Loss: 149.155 +22400/69092 Loss: 148.159 +25600/69092 Loss: 151.041 +28800/69092 Loss: 150.103 +32000/69092 Loss: 147.212 +35200/69092 Loss: 148.987 +38400/69092 Loss: 146.847 +41600/69092 Loss: 147.224 +44800/69092 Loss: 147.831 +48000/69092 Loss: 149.702 +51200/69092 Loss: 148.154 +54400/69092 Loss: 148.037 +57600/69092 Loss: 148.568 +60800/69092 Loss: 147.742 +64000/69092 Loss: 149.219 +67200/69092 Loss: 148.690 +Training time 0:09:02.812885 +Epoch: 12 Average loss: 148.42 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 396) +0/69092 Loss: 142.583 +3200/69092 Loss: 148.079 +6400/69092 Loss: 148.763 +9600/69092 Loss: 149.152 +12800/69092 Loss: 148.507 +16000/69092 Loss: 149.841 +19200/69092 Loss: 148.869 +22400/69092 Loss: 149.488 +25600/69092 Loss: 148.301 +28800/69092 Loss: 149.035 +32000/69092 Loss: 148.148 +35200/69092 Loss: 146.048 +38400/69092 Loss: 149.526 +41600/69092 Loss: 145.658 +44800/69092 Loss: 148.386 +48000/69092 Loss: 150.427 +51200/69092 Loss: 150.326 +54400/69092 Loss: 145.604 +57600/69092 Loss: 148.702 +60800/69092 Loss: 151.526 +64000/69092 Loss: 147.669 +67200/69092 Loss: 148.106 +Training time 0:09:00.687459 +Epoch: 13 Average loss: 148.63 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 397) +0/69092 Loss: 153.672 +3200/69092 Loss: 150.343 +6400/69092 Loss: 148.938 +9600/69092 Loss: 145.993 +12800/69092 Loss: 147.901 +16000/69092 Loss: 147.641 +19200/69092 Loss: 150.301 +22400/69092 Loss: 147.967 +25600/69092 Loss: 149.461 +28800/69092 Loss: 147.351 +32000/69092 Loss: 147.978 +35200/69092 Loss: 147.770 +38400/69092 Loss: 148.081 +41600/69092 Loss: 152.066 +44800/69092 Loss: 149.364 +48000/69092 Loss: 149.303 +51200/69092 Loss: 147.742 +54400/69092 Loss: 147.767 +57600/69092 Loss: 149.432 +60800/69092 Loss: 148.578 +64000/69092 Loss: 148.193 +67200/69092 Loss: 146.824 +Training time 0:09:07.147777 +Epoch: 14 Average loss: 148.55 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 398) +0/69092 Loss: 154.706 +3200/69092 Loss: 149.984 +6400/69092 Loss: 147.826 +9600/69092 Loss: 149.050 +12800/69092 Loss: 148.082 +16000/69092 Loss: 147.026 +19200/69092 Loss: 147.830 +22400/69092 Loss: 147.191 +25600/69092 Loss: 150.028 +28800/69092 Loss: 147.502 +32000/69092 Loss: 148.913 +35200/69092 Loss: 147.774 +38400/69092 Loss: 146.584 +41600/69092 Loss: 147.086 +44800/69092 Loss: 151.431 +48000/69092 Loss: 148.993 +51200/69092 Loss: 147.126 +54400/69092 Loss: 149.061 +57600/69092 Loss: 149.496 +60800/69092 Loss: 150.705 +64000/69092 Loss: 149.200 +67200/69092 Loss: 147.942 +Training time 0:08:47.208411 +Epoch: 15 Average loss: 148.55 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 399) +0/69092 Loss: 162.081 +3200/69092 Loss: 146.958 +6400/69092 Loss: 147.251 +9600/69092 Loss: 147.883 +12800/69092 Loss: 145.721 +16000/69092 Loss: 145.575 +19200/69092 Loss: 151.240 +22400/69092 Loss: 147.378 +25600/69092 Loss: 149.983 +28800/69092 Loss: 148.666 +32000/69092 Loss: 148.957 +35200/69092 Loss: 147.893 +38400/69092 Loss: 150.558 +41600/69092 Loss: 148.435 +44800/69092 Loss: 149.339 +48000/69092 Loss: 148.824 +51200/69092 Loss: 146.886 +54400/69092 Loss: 147.452 +57600/69092 Loss: 147.164 +60800/69092 Loss: 148.653 +64000/69092 Loss: 147.348 +67200/69092 Loss: 150.272 +Training time 0:08:54.783178 +Epoch: 16 Average loss: 148.19 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 400) +0/69092 Loss: 148.374 +3200/69092 Loss: 148.049 +6400/69092 Loss: 150.116 +9600/69092 Loss: 143.784 +12800/69092 Loss: 146.203 +16000/69092 Loss: 148.805 +19200/69092 Loss: 149.583 +22400/69092 Loss: 147.258 +25600/69092 Loss: 147.839 +28800/69092 Loss: 149.711 +32000/69092 Loss: 147.253 +35200/69092 Loss: 149.489 +38400/69092 Loss: 147.980 +41600/69092 Loss: 149.826 +44800/69092 Loss: 149.049 +48000/69092 Loss: 148.529 +51200/69092 Loss: 148.352 +54400/69092 Loss: 149.552 +57600/69092 Loss: 148.745 +60800/69092 Loss: 147.908 +64000/69092 Loss: 150.221 +67200/69092 Loss: 148.150 +Training time 0:08:46.326460 +Epoch: 17 Average loss: 148.46 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 401) +0/69092 Loss: 148.667 +3200/69092 Loss: 148.798 +6400/69092 Loss: 147.345 +9600/69092 Loss: 151.147 +12800/69092 Loss: 147.851 +16000/69092 Loss: 148.137 +19200/69092 Loss: 147.272 +22400/69092 Loss: 151.296 +25600/69092 Loss: 148.091 +28800/69092 Loss: 146.713 +32000/69092 Loss: 148.339 +35200/69092 Loss: 149.853 +38400/69092 Loss: 148.396 +41600/69092 Loss: 148.286 +44800/69092 Loss: 149.332 +48000/69092 Loss: 147.071 +51200/69092 Loss: 146.582 +54400/69092 Loss: 148.309 +57600/69092 Loss: 147.126 +60800/69092 Loss: 148.934 +64000/69092 Loss: 150.969 +67200/69092 Loss: 149.921 +Training time 0:08:52.783104 +Epoch: 18 Average loss: 148.62 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 402) +0/69092 Loss: 160.845 +3200/69092 Loss: 147.069 +6400/69092 Loss: 145.704 +9600/69092 Loss: 148.991 +12800/69092 Loss: 149.128 +16000/69092 Loss: 148.295 +19200/69092 Loss: 146.797 +22400/69092 Loss: 148.149 +25600/69092 Loss: 149.634 +28800/69092 Loss: 146.860 +32000/69092 Loss: 146.170 +35200/69092 Loss: 148.381 +38400/69092 Loss: 146.985 +41600/69092 Loss: 146.265 +44800/69092 Loss: 150.647 +48000/69092 Loss: 149.986 +51200/69092 Loss: 149.418 +54400/69092 Loss: 150.401 +57600/69092 Loss: 148.946 +60800/69092 Loss: 148.241 +64000/69092 Loss: 148.328 +67200/69092 Loss: 149.539 +Training time 0:08:58.246864 +Epoch: 19 Average loss: 148.39 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 403) +0/69092 Loss: 148.493 +3200/69092 Loss: 150.437 +6400/69092 Loss: 147.256 +9600/69092 Loss: 150.340 +12800/69092 Loss: 145.597 +16000/69092 Loss: 143.907 +19200/69092 Loss: 147.719 +22400/69092 Loss: 149.212 +25600/69092 Loss: 147.621 +28800/69092 Loss: 149.197 +32000/69092 Loss: 148.108 +35200/69092 Loss: 148.515 +38400/69092 Loss: 147.109 +41600/69092 Loss: 150.834 +44800/69092 Loss: 146.322 +48000/69092 Loss: 147.154 +51200/69092 Loss: 150.583 +54400/69092 Loss: 150.381 +57600/69092 Loss: 147.629 +60800/69092 Loss: 148.895 +64000/69092 Loss: 149.012 +67200/69092 Loss: 150.196 +Training time 0:09:09.768489 +Epoch: 20 Average loss: 148.40 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 404) +0/69092 Loss: 155.011 +3200/69092 Loss: 150.039 +6400/69092 Loss: 147.834 +9600/69092 Loss: 151.055 +12800/69092 Loss: 147.030 +16000/69092 Loss: 149.360 +19200/69092 Loss: 147.986 +22400/69092 Loss: 149.366 +25600/69092 Loss: 148.121 +28800/69092 Loss: 148.877 +32000/69092 Loss: 149.134 +35200/69092 Loss: 149.375 +38400/69092 Loss: 147.610 +41600/69092 Loss: 147.797 +44800/69092 Loss: 150.285 +48000/69092 Loss: 146.658 +51200/69092 Loss: 145.238 +54400/69092 Loss: 146.441 +57600/69092 Loss: 149.190 +60800/69092 Loss: 148.107 +64000/69092 Loss: 150.174 +67200/69092 Loss: 147.560 +Training time 0:09:02.290506 +Epoch: 21 Average loss: 148.49 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 405) +0/69092 Loss: 136.954 +3200/69092 Loss: 148.606 +6400/69092 Loss: 147.706 +9600/69092 Loss: 146.021 +12800/69092 Loss: 147.506 +16000/69092 Loss: 148.935 +19200/69092 Loss: 148.701 +22400/69092 Loss: 149.310 +25600/69092 Loss: 147.547 +28800/69092 Loss: 148.972 +32000/69092 Loss: 148.618 +35200/69092 Loss: 147.854 +38400/69092 Loss: 147.932 +41600/69092 Loss: 146.946 +44800/69092 Loss: 149.427 +48000/69092 Loss: 148.953 +51200/69092 Loss: 149.170 +54400/69092 Loss: 147.848 +57600/69092 Loss: 147.531 +60800/69092 Loss: 145.401 +64000/69092 Loss: 149.425 +67200/69092 Loss: 149.049 +Training time 0:09:02.930326 +Epoch: 22 Average loss: 148.21 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 406) +0/69092 Loss: 132.888 +3200/69092 Loss: 146.944 +6400/69092 Loss: 147.659 +9600/69092 Loss: 148.153 +12800/69092 Loss: 146.067 +16000/69092 Loss: 146.874 +19200/69092 Loss: 146.669 +22400/69092 Loss: 149.614 +25600/69092 Loss: 146.883 +28800/69092 Loss: 149.405 +32000/69092 Loss: 146.970 +35200/69092 Loss: 146.448 +38400/69092 Loss: 151.269 +41600/69092 Loss: 149.954 +44800/69092 Loss: 149.437 +48000/69092 Loss: 149.159 +51200/69092 Loss: 146.190 +54400/69092 Loss: 145.881 +57600/69092 Loss: 147.482 +60800/69092 Loss: 151.322 +64000/69092 Loss: 149.133 +67200/69092 Loss: 147.746 +Training time 0:08:47.872983 +Epoch: 23 Average loss: 148.16 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 407) +0/69092 Loss: 134.402 +3200/69092 Loss: 149.188 +6400/69092 Loss: 148.388 +9600/69092 Loss: 148.458 +12800/69092 Loss: 145.928 +16000/69092 Loss: 147.602 +19200/69092 Loss: 146.478 +22400/69092 Loss: 148.272 +25600/69092 Loss: 148.289 +28800/69092 Loss: 148.099 +32000/69092 Loss: 148.887 +35200/69092 Loss: 150.344 +38400/69092 Loss: 148.626 +41600/69092 Loss: 148.378 +44800/69092 Loss: 149.421 +48000/69092 Loss: 148.677 +51200/69092 Loss: 148.167 +54400/69092 Loss: 148.458 +57600/69092 Loss: 149.659 +60800/69092 Loss: 149.415 +64000/69092 Loss: 147.444 +67200/69092 Loss: 146.036 +Training time 0:08:50.696391 +Epoch: 24 Average loss: 148.35 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 408) +0/69092 Loss: 157.593 +3200/69092 Loss: 150.009 +6400/69092 Loss: 149.978 +9600/69092 Loss: 148.384 +12800/69092 Loss: 148.417 +16000/69092 Loss: 147.030 +19200/69092 Loss: 148.650 +22400/69092 Loss: 145.753 +25600/69092 Loss: 150.395 +28800/69092 Loss: 147.850 +32000/69092 Loss: 148.134 +35200/69092 Loss: 148.396 +38400/69092 Loss: 147.508 +41600/69092 Loss: 145.220 +44800/69092 Loss: 150.131 +48000/69092 Loss: 149.278 +51200/69092 Loss: 146.565 +54400/69092 Loss: 148.079 +57600/69092 Loss: 151.236 +60800/69092 Loss: 149.145 +64000/69092 Loss: 147.249 +67200/69092 Loss: 148.559 +Training time 0:09:07.761578 +Epoch: 25 Average loss: 148.42 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 409) +0/69092 Loss: 157.040 +3200/69092 Loss: 149.437 +6400/69092 Loss: 148.512 +9600/69092 Loss: 147.398 +12800/69092 Loss: 149.228 +16000/69092 Loss: 148.261 +19200/69092 Loss: 151.086 +22400/69092 Loss: 149.001 +25600/69092 Loss: 150.663 +28800/69092 Loss: 148.152 +32000/69092 Loss: 146.417 +35200/69092 Loss: 148.666 +38400/69092 Loss: 148.263 +41600/69092 Loss: 149.578 +44800/69092 Loss: 146.636 +48000/69092 Loss: 147.623 +51200/69092 Loss: 146.347 +54400/69092 Loss: 147.707 +57600/69092 Loss: 148.565 +60800/69092 Loss: 148.173 +64000/69092 Loss: 148.738 +67200/69092 Loss: 148.175 +Training time 0:09:09.511901 +Epoch: 26 Average loss: 148.35 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 410) +0/69092 Loss: 150.704 +3200/69092 Loss: 147.565 +6400/69092 Loss: 147.792 +9600/69092 Loss: 146.466 +12800/69092 Loss: 147.434 +16000/69092 Loss: 150.431 +19200/69092 Loss: 149.472 +22400/69092 Loss: 147.425 +25600/69092 Loss: 147.409 +28800/69092 Loss: 149.671 +32000/69092 Loss: 147.043 +35200/69092 Loss: 148.956 +38400/69092 Loss: 148.086 +41600/69092 Loss: 149.742 +44800/69092 Loss: 147.087 +48000/69092 Loss: 146.434 +51200/69092 Loss: 147.783 +54400/69092 Loss: 148.881 +57600/69092 Loss: 148.384 +60800/69092 Loss: 150.013 +64000/69092 Loss: 151.416 +67200/69092 Loss: 148.161 +Training time 0:09:05.992895 +Epoch: 27 Average loss: 148.42 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 411) +0/69092 Loss: 145.985 +3200/69092 Loss: 148.648 +6400/69092 Loss: 147.020 +9600/69092 Loss: 146.654 +12800/69092 Loss: 146.435 +16000/69092 Loss: 146.639 +19200/69092 Loss: 149.927 +22400/69092 Loss: 148.067 +25600/69092 Loss: 148.626 +28800/69092 Loss: 148.934 +32000/69092 Loss: 149.611 +35200/69092 Loss: 147.454 +38400/69092 Loss: 146.692 +41600/69092 Loss: 147.571 +44800/69092 Loss: 148.395 +48000/69092 Loss: 149.005 +51200/69092 Loss: 149.284 +54400/69092 Loss: 150.009 +57600/69092 Loss: 148.238 +60800/69092 Loss: 148.752 +64000/69092 Loss: 152.488 +67200/69092 Loss: 148.458 +Training time 0:08:53.110724 +Epoch: 28 Average loss: 148.42 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 412) +0/69092 Loss: 140.530 +3200/69092 Loss: 149.617 +6400/69092 Loss: 148.875 +9600/69092 Loss: 148.385 +12800/69092 Loss: 145.872 +16000/69092 Loss: 147.475 +19200/69092 Loss: 148.305 +22400/69092 Loss: 150.160 +25600/69092 Loss: 147.275 +28800/69092 Loss: 149.528 +32000/69092 Loss: 147.095 +35200/69092 Loss: 145.753 +38400/69092 Loss: 151.182 +41600/69092 Loss: 148.270 +44800/69092 Loss: 149.513 +48000/69092 Loss: 147.346 +51200/69092 Loss: 147.973 +54400/69092 Loss: 148.995 +57600/69092 Loss: 149.471 +60800/69092 Loss: 146.983 +64000/69092 Loss: 147.300 +67200/69092 Loss: 150.169 +Training time 0:08:59.751902 +Epoch: 29 Average loss: 148.39 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 413) +0/69092 Loss: 154.834 +3200/69092 Loss: 148.602 +6400/69092 Loss: 147.281 +9600/69092 Loss: 148.257 +12800/69092 Loss: 148.066 +16000/69092 Loss: 149.303 +19200/69092 Loss: 147.593 +22400/69092 Loss: 150.067 +25600/69092 Loss: 148.943 +28800/69092 Loss: 149.007 +32000/69092 Loss: 149.917 +35200/69092 Loss: 148.033 +38400/69092 Loss: 147.071 +41600/69092 Loss: 145.994 +44800/69092 Loss: 149.236 +48000/69092 Loss: 150.653 +51200/69092 Loss: 147.954 +54400/69092 Loss: 148.355 +57600/69092 Loss: 148.669 +60800/69092 Loss: 148.295 +64000/69092 Loss: 145.584 +67200/69092 Loss: 148.381 +Training time 0:08:53.192705 +Epoch: 30 Average loss: 148.28 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 414) +0/69092 Loss: 147.329 +3200/69092 Loss: 150.836 +6400/69092 Loss: 148.322 +9600/69092 Loss: 147.041 +12800/69092 Loss: 150.053 +16000/69092 Loss: 148.353 +19200/69092 Loss: 149.621 +22400/69092 Loss: 149.111 +25600/69092 Loss: 146.548 +28800/69092 Loss: 148.025 +32000/69092 Loss: 146.271 +35200/69092 Loss: 146.669 +38400/69092 Loss: 148.178 +41600/69092 Loss: 147.295 +44800/69092 Loss: 150.026 +48000/69092 Loss: 147.352 +51200/69092 Loss: 148.345 +54400/69092 Loss: 149.542 +57600/69092 Loss: 149.092 +60800/69092 Loss: 145.997 +64000/69092 Loss: 149.676 +67200/69092 Loss: 148.665 +Training time 0:09:05.739956 +Epoch: 31 Average loss: 148.36 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 415) +0/69092 Loss: 151.837 +3200/69092 Loss: 147.201 +6400/69092 Loss: 149.816 +9600/69092 Loss: 149.556 +12800/69092 Loss: 148.716 +16000/69092 Loss: 147.223 +19200/69092 Loss: 146.076 +22400/69092 Loss: 148.112 +25600/69092 Loss: 148.333 +28800/69092 Loss: 149.023 +32000/69092 Loss: 148.763 +35200/69092 Loss: 148.995 +38400/69092 Loss: 145.106 +41600/69092 Loss: 148.213 +44800/69092 Loss: 149.859 +48000/69092 Loss: 147.339 +51200/69092 Loss: 146.453 +54400/69092 Loss: 150.950 +57600/69092 Loss: 147.687 +60800/69092 Loss: 147.135 +64000/69092 Loss: 151.026 +67200/69092 Loss: 149.801 +Training time 0:08:51.114166 +Epoch: 32 Average loss: 148.36 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 416) +0/69092 Loss: 156.896 +3200/69092 Loss: 148.975 +6400/69092 Loss: 147.950 +9600/69092 Loss: 149.030 +12800/69092 Loss: 145.228 +16000/69092 Loss: 147.176 +19200/69092 Loss: 148.334 +22400/69092 Loss: 149.624 +25600/69092 Loss: 152.305 +28800/69092 Loss: 147.235 +32000/69092 Loss: 146.033 +35200/69092 Loss: 147.091 +38400/69092 Loss: 148.412 +41600/69092 Loss: 148.229 +44800/69092 Loss: 148.240 +48000/69092 Loss: 147.240 +51200/69092 Loss: 148.851 +54400/69092 Loss: 147.703 +57600/69092 Loss: 148.065 +60800/69092 Loss: 148.718 +64000/69092 Loss: 148.478 +67200/69092 Loss: 148.794 +Training time 0:09:00.180181 +Epoch: 33 Average loss: 148.26 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 417) +0/69092 Loss: 148.454 +3200/69092 Loss: 148.839 +6400/69092 Loss: 149.033 +9600/69092 Loss: 148.163 +12800/69092 Loss: 149.427 +16000/69092 Loss: 149.431 +19200/69092 Loss: 151.534 +22400/69092 Loss: 149.696 +25600/69092 Loss: 149.770 +28800/69092 Loss: 147.351 +32000/69092 Loss: 148.725 +35200/69092 Loss: 146.890 +38400/69092 Loss: 147.490 +41600/69092 Loss: 147.314 +44800/69092 Loss: 146.463 +48000/69092 Loss: 146.713 +51200/69092 Loss: 147.816 +54400/69092 Loss: 147.123 +57600/69092 Loss: 148.996 +60800/69092 Loss: 148.508 +64000/69092 Loss: 147.167 +67200/69092 Loss: 146.828 +Training time 0:08:44.694787 +Epoch: 34 Average loss: 148.25 +=> saved checkpoint 'trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last' (iter 418) +0/69092 Loss: 154.200 +3200/69092 Loss: 146.082 +6400/69092 Loss: 147.593 +9600/69092 Loss: 147.783 +12800/69092 Loss: 146.372 +16000/69092 Loss: 149.915 +19200/69092 Loss: 147.554 +22400/69092 Loss: 149.566 +25600/69092 Loss: 150.731 +28800/69092 Loss: 147.846 +32000/69092 Loss: 148.889 +35200/69092 Loss: 148.217 diff --git a/trained_models/rendered_chairs/VAE_bs_256/checkpoints/last b/trained_models/rendered_chairs/VAE_bs_256/checkpoints/last index d1a87d6f3c35884bfe06323480b84a9506780b70..cb177b2a09cbbb5ec3855f67263df44ec114a168 100644 Binary files a/trained_models/rendered_chairs/VAE_bs_256/checkpoints/last and b/trained_models/rendered_chairs/VAE_bs_256/checkpoints/last differ diff --git a/trained_models/rendered_chairs/VAE_bs_64/checkpoints/last b/trained_models/rendered_chairs/VAE_bs_64/checkpoints/last index e8fc305ece11b771a8d803775d22e57629e170e3..4f2642ad341e29f5407b4424c4b4e2123c640da4 100644 Binary files a/trained_models/rendered_chairs/VAE_bs_64/checkpoints/last and b/trained_models/rendered_chairs/VAE_bs_64/checkpoints/last differ diff --git a/trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last b/trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last index 9b35309aa4578f2de59bce458f3fdc47ab49254a..72b2573a48d08d3e56c5496b1c132eabd5434fd4 100644 Binary files a/trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last and b/trained_models/rendered_chairs/VAE_bs_64_ls_30/checkpoints/last differ diff --git a/trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last b/trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last index 466b0762595ac5ab58c360abf843691dd1c4e6ed..ec60f6afe1e8e7f9ad62860e82424b6b1328e366 100644 Binary files a/trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last and b/trained_models/rendered_chairs/VAE_bs_64_ls_40/checkpoints/last differ diff --git a/trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last b/trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last index 4c186ac9f144c21e647e14546867ddb581b2e061..2b42d43dc2fecf682658e2bb42014de3bb47d9a5 100644 Binary files a/trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last and b/trained_models/rendered_chairs/VAE_bs_64_ls_50/checkpoints/last differ diff --git a/trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last b/trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last index bc31e8decd836bb7a49e930b3a502eb5219e0368..bd97a1596ce25bc80b9d7219543de89c95eb8da9 100644 Binary files a/trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last and b/trained_models/rendered_chairs/beta_VAE_bs_64/checkpoints/last differ diff --git a/trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last b/trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last index ca6484f67e2c8ffeb890aaa408b77ffb742e3a09..754683cd89ef435818991bf29b93f403a60bbb1c 100644 Binary files a/trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last and b/trained_models/rendered_chairs/beta_VAE_bs_64_ls_15/checkpoints/last differ