.ipynb_checkpoints/left_min_experiment-5-epochs-valhistory-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.utils.np_utils import to_categorical\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from keras.datasets import mnist\n",
    "import itertools\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Randomly select a defined number of image paires from a given dataset\n",
    "def get_left_min_image_combinations(number_image_pairs_selected, image_data, image_labels):\n",
    "    \n",
    "    num_images_total = len(image_labels)\n",
    "    \n",
    "    image_combination_data = []\n",
    "    image_combination_labels = []\n",
    "    \n",
    "    for i in range(number_image_pairs_selected):\n",
    "        \n",
    "        # Draw two image indices from a uniform random distribution\n",
    "        random_index_A = random.randint(0, num_images_total-1)\n",
    "        random_index_B = random.randint(0, num_images_total-1)\n",
    "        \n",
    "        # Randomly choose two images from the dataset\n",
    "        image_A = image_data[random_index_A]\n",
    "        image_B = image_data[random_index_B]\n",
    "        \n",
    "        # Merge images TODO: this could be a function with several options\n",
    "        combined_images = np.hstack((image_A, image_B))\n",
    "        \n",
    "        # Find the minimum between the two labels\n",
    "        label_A = image_labels[random_index_A]\n",
    "        label_B = image_labels[random_index_B]\n",
    "        minimum_label = min(label_A, label_B)\n",
    "        \n",
    "        if label_A < label_B:\n",
    "            # Merge images TODO: this could be a function with several options\n",
    "            combined_images = np.hstack((image_A, image_B))\n",
    "        elif label_A > label_B: \n",
    "            # Merge images TODO: this could be a function with several options\n",
    "            combined_images = np.hstack((image_B, image_A))\n",
    "        # Here we want to exclude the images where they are the same on both sides\n",
    "            \n",
    "   \n",
    "        # Append newly generated image combination and minimum label to list\n",
    "        image_combination_data.append(combined_images)\n",
    "        image_combination_labels.append(minimum_label)\n",
    "        \n",
    "    # Convert image data and labels lists to numpy arrays\n",
    "    image_combination_data = np.array(image_combination_data)\n",
    "    image_combination_labels = np.array(image_combination_labels)\n",
    "    \n",
    "    return image_combination_data, image_combination_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Randomly select a defined number of image paires from a given dataset\n",
    "def get_right_min_image_combinations(number_image_pairs_selected, image_data, image_labels):\n",
    "    \n",
    "    num_images_total = len(image_labels)\n",
    "    \n",
    "    image_combination_data = []\n",
    "    image_combination_labels = []\n",
    "    \n",
    "    for i in range(number_image_pairs_selected):\n",
    "        \n",
    "        # Draw two image indices from a uniform random distribution\n",
    "        random_index_A = random.randint(0, num_images_total-1)\n",
    "        random_index_B = random.randint(0, num_images_total-1)\n",
    "        \n",
    "        # Randomly choose two images from the dataset\n",
    "        image_A = image_data[random_index_A]\n",
    "        image_B = image_data[random_index_B]\n",
    "        \n",
    "        # Merge images TODO: this could be a function with several options\n",
    "        combined_images = np.hstack((image_A, image_B))\n",
    "        \n",
    "        # Find the minimum between the two labels\n",
    "        label_A = image_labels[random_index_A]\n",
    "        label_B = image_labels[random_index_B]\n",
    "        minimum_label = min(label_A, label_B)\n",
    "        \n",
    "        if label_A < label_B:\n",
    "            # Merge images TODO: this could be a function with several options\n",
    "            combined_images = np.hstack((image_B, image_A))\n",
    "        elif label_A > label_B: \n",
    "            # Merge images TODO: this could be a function with several options\n",
    "            combined_images = np.hstack((image_A, image_B))\n",
    "        # Here we want to exclude the images where they are the same on both sides\n",
    "            \n",
    "   \n",
    "        # Append newly generated image combination and minimum label to list\n",
    "        image_combination_data.append(combined_images)\n",
    "        image_combination_labels.append(minimum_label)\n",
    "        \n",
    "    # Convert image data and labels lists to numpy arrays\n",
    "    image_combination_data = np.array(image_combination_data)\n",
    "    image_combination_labels = np.array(image_combination_labels)\n",
    "    \n",
    "    return image_combination_data, image_combination_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data and split into training and test\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split training set further into training and validation sets\n",
    "sample_size = 40000\n",
    "X_train, X_val, y_train, y_val = train_images[:sample_size], train_images[sample_size:sample_size*2],  train_labels[:sample_size], train_labels[sample_size:sample_size*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "combo_train_X, combo_train_y = get_left_min_image_combinations(40000, X_train, y_train)\n",
    "combo_val_X, combo_val_y = get_right_min_image_combinations(20000, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape the data so they can be fed into the network, one-hot-encode the labels\n",
    "\n",
    "train_images = combo_train_X.reshape((combo_train_X.shape[0], 28, 56, 1))\n",
    "train_images = train_images/ 255\n",
    "\n",
    "val_images = combo_val_X.reshape((combo_val_X.shape[0], 28, 56, 1))\n",
    "val_images = val_images/255\n",
    "\n",
    "train_labels = to_categorical(combo_train_y)\n",
    "val_labels = to_categorical(combo_val_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model architecture\n",
    "def build_CNN():\n",
    "\n",
    "    # Convolutional NN\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(layers.Conv2D(32, (3,3), activation = 'relu', input_shape = (28,56,1)))\n",
    "    model.add(layers.MaxPooling2D(2,2))\n",
    "    model.add(layers.Conv2D(64, (3,3), activation = 'relu'))\n",
    "    model.add(layers.MaxPooling2D(2,2))\n",
    "    model.add(layers.Conv2D(64, (3,3), activation = 'relu'))\n",
    "    # Adding a NN Classifier\n",
    "    model.add(layers.Flatten())\n",
    "    model.add(layers.Dense(64, activation = 'relu'))\n",
    "    model.add(layers.Dense(10, activation = 'softmax'))\n",
    "\n",
    "    model.compile(optimizer='rmsprop', \n",
    "                loss='categorical_crossentropy', \n",
    "                metrics=['accuracy', 'mae'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 26, 54, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 13, 27, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 11, 25, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 5, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 3, 10, 64)         36928     \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 1920)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 64)                122944    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 179,338\n",
      "Trainable params: 179,338\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "CNN_model = build_CNN()\n",
    "CNN_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "625/625 [==============================] - 58s 88ms/step - loss: 0.2058 - accuracy: 0.9366 - mae: 0.0194\n",
      "Epoch 2/5\n",
      "625/625 [==============================] - 71s 114ms/step - loss: 0.0382 - accuracy: 0.9881 - mae: 0.0037\n",
      "Epoch 3/5\n",
      "625/625 [==============================] - 63s 102ms/step - loss: 0.0207 - accuracy: 0.9937 - mae: 0.0019\n",
      "Epoch 4/5\n",
      "625/625 [==============================] - 66s 105ms/step - loss: 0.0139 - accuracy: 0.9953 - mae: 0.0013\n",
      "Epoch 5/5\n",
      "625/625 [==============================] - 63s 101ms/step - loss: 0.0101 - accuracy: 0.9970 - mae: 8.6650e-04\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x114252a30>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit model\n",
    "CNN_model.fit(train_images, \n",
    "          train_labels, \n",
    "          epochs = 5, \n",
    "          batch_size = 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "625/625 [==============================] - 7s 11ms/step - loss: 16.1712 - accuracy: 0.1176 - mae: 0.1763\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[16.171188354492188, 0.11755000054836273, 0.1762557476758957]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print loss, mae and accuracy\n",
    "CNN_eval_model = CNN_model.evaluate(val_images, val_labels)\n",
    "CNN_eval_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model architecture\n",
    "def build_DNN():\n",
    "\n",
    "    # Convolutional NN\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(layers.Conv2D(32, (3,3), activation = 'relu', input_shape = (28,56,1)))\n",
    "    model.add(layers.MaxPooling2D(2,2))\n",
    "    model.add(layers.Conv2D(64, (3,3), activation = 'relu'))\n",
    "    model.add(layers.MaxPooling2D(2,2))\n",
    "    model.add(layers.Conv2D(64, (3,3), activation = 'relu'))\n",
    "    # Adding a NN Classifier\n",
    "    model.add(layers.Flatten())\n",
    "    model.add(layers.Dropout(0.5))\n",
    "    model.add(layers.Dense(64, activation = 'relu'))\n",
    "    model.add(layers.Dense(10, activation = 'softmax'))\n",
    "\n",
    "    # Compile model\n",
    "    model.compile(optimizer='rmsprop', \n",
    "                loss='categorical_crossentropy', \n",
    "                metrics=['accuracy', 'mae'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_3 (Conv2D)            (None, 26, 54, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 13, 27, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 11, 25, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 5, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 3, 10, 64)         36928     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 1920)              0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1920)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 64)                122944    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 179,338\n",
      "Trainable params: 179,338\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "DNN_model = build_DNN()\n",
    "DNN_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "625/625 [==============================] - 65s 100ms/step - loss: 0.2164 - accuracy: 0.9326 - mae: 0.0207\n",
      "Epoch 2/5\n",
      "625/625 [==============================] - 66s 105ms/step - loss: 0.0597 - accuracy: 0.9821 - mae: 0.0056\n",
      "Epoch 3/5\n",
      "625/625 [==============================] - 62s 99ms/step - loss: 0.0377 - accuracy: 0.9874 - mae: 0.0036\n",
      "Epoch 4/5\n",
      "625/625 [==============================] - 60s 97ms/step - loss: 0.0299 - accuracy: 0.9910 - mae: 0.0027\n",
      "Epoch 5/5\n",
      "625/625 [==============================] - 61s 98ms/step - loss: 0.0223 - accuracy: 0.9926 - mae: 0.0021\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x14ce83670>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit model\n",
    "DNN_model.fit(train_images, \n",
    "          train_labels, \n",
    "          epochs = 5, \n",
    "          batch_size = 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "625/625 [==============================] - 7s 10ms/step - loss: 14.6404 - accuracy: 0.1159 - mae: 0.1764\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[14.64042854309082, 0.11590000241994858, 0.17643354833126068]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print loss, mae and accuracy\n",
    "DNN_eval_model = DNN_model.evaluate(val_images, val_labels)\n",
    "DNN_eval_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "625/625 [==============================] - 47s 76ms/step - loss: 0.0203 - accuracy: 0.9939 - mae: 0.0018\n",
      "Epoch 2/10\n",
      "625/625 [==============================] - 49s 78ms/step - loss: 0.0167 - accuracy: 0.9945 - mae: 0.0015\n",
      "Epoch 3/10\n",
      "625/625 [==============================] - 49s 79ms/step - loss: 0.0137 - accuracy: 0.9955 - mae: 0.0012\n",
      "Epoch 4/10\n",
      "625/625 [==============================] - 46s 73ms/step - loss: 0.0133 - accuracy: 0.9960 - mae: 0.0011\n",
      "Epoch 5/10\n",
      "625/625 [==============================] - 46s 74ms/step - loss: 0.0113 - accuracy: 0.9962 - mae: 9.8396e-04\n",
      "Epoch 6/10\n",
      "625/625 [==============================] - 47s 75ms/step - loss: 0.0118 - accuracy: 0.9963 - mae: 9.3090e-04\n",
      "Epoch 7/10\n",
      "625/625 [==============================] - 48s 77ms/step - loss: 0.0110 - accuracy: 0.9963 - mae: 9.2957e-04\n",
      "Epoch 8/10\n",
      "625/625 [==============================] - 50s 81ms/step - loss: 0.0098 - accuracy: 0.9969 - mae: 7.9370e-04\n",
      "Epoch 9/10\n",
      "625/625 [==============================] - 50s 79ms/step - loss: 0.0089 - accuracy: 0.9974 - mae: 6.7800e-04\n",
      "Epoch 10/10\n",
      "625/625 [==============================] - 52s 83ms/step - loss: 0.0075 - accuracy: 0.9980 - mae: 5.9168e-04\n",
      "625/625 [==============================] - 5s 8ms/step - loss: 22.6241 - accuracy: 0.1116 - mae: 0.1776\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[22.624094009399414, 0.11155000329017639, 0.17761020362377167]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit model\n",
    "DNN_model.fit(train_images, \n",
    "          train_labels, \n",
    "          epochs = 10, \n",
    "          batch_size = 64)\n",
    "\n",
    "# Print loss, mae and accuracy\n",
    "DNN_eval_model = DNN_model.evaluate(val_images, val_labels)\n",
    "DNN_eval_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "general_ML",
   "language": "python",
   "name": "general_ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}