.ipynb_checkpoints/FindingMino-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import numpy as np\n",
    "from keras.utils.np_utils import to_categorical\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "plt.style.use('dark_background')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding the minimum of two hand drawn digits without intermediate labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = fetch_openml('mnist_784', version = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'frame', 'categories', 'feature_names', 'target_names', 'DESCR', 'details', 'url'])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### About the data\n",
    "\n",
    "- DESC: describes the dataset\n",
    "- data: contains one row per instance, one columnn per feature\n",
    "- target: contains an array of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mnist[\"data\"], mnist[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>pixel9</th>\n",
       "      <th>pixel10</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "      <th>pixel784</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 784 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  pixel9  \\\n",
       "0     0.0     0.0     0.0     0.0     0.0     0.0     0.0     0.0     0.0   \n",
       "1     0.0     0.0     0.0     0.0     0.0     0.0     0.0     0.0     0.0   \n",
       "2     0.0     0.0     0.0     0.0     0.0     0.0     0.0     0.0     0.0   \n",
       "\n",
       "   pixel10  ...  pixel775  pixel776  pixel777  pixel778  pixel779  pixel780  \\\n",
       "0      0.0  ...       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "1      0.0  ...       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "2      0.0  ...       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   pixel781  pixel782  pixel783  pixel784  \n",
       "0       0.0       0.0       0.0       0.0  \n",
       "1       0.0       0.0       0.0       0.0  \n",
       "2       0.0       0.0       0.0       0.0  \n",
       "\n",
       "[3 rows x 784 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pixel1      0.0\n",
       "pixel2      0.0\n",
       "pixel3      0.0\n",
       "pixel4      0.0\n",
       "pixel5      0.0\n",
       "           ... \n",
       "pixel780    0.0\n",
       "pixel781    0.0\n",
       "pixel782    0.0\n",
       "pixel783    0.0\n",
       "pixel784    0.0\n",
       "Name: 0, Length: 784, dtype: float64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_digit = X.loc[0,:]\n",
    "example_digit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split out test set\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature engineering using Gonzalo's proposed method\n",
    "\n",
    "- Create pairs of inputs a and b\n",
    "    - Find a set of possible combination of a and b indices. We probably do not need the entire set of combinations (60000C2 is 1799970000!)\n",
    "    - We'll start with a more manageable amount, say 2000 images (2000 C 2 is 1999000) for training and testing\n",
    "\n",
    "- Label them as min(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "mini_size = 500\n",
    "\n",
    "mini_X_train, mini_X_test, mini_y_train, mini_y_test = X_train[:mini_size], X_train[mini_size:mini_size*2],  y_train[:mini_size], y_train[mini_size:mini_size*2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500    3\n",
       "501    9\n",
       "502    5\n",
       "503    8\n",
       "504    5\n",
       "      ..\n",
       "995    7\n",
       "996    6\n",
       "997    0\n",
       "998    3\n",
       "999    6\n",
       "Name: class, Length: 500, dtype: category\n",
       "Categories (10, object): ['0', '1', '2', '3', ..., '6', '7', '8', '9']"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mini_y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Training data label distribution')"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAccUlEQVR4nO3de1BU58EG8IfLquAFkI0QQEEdsOg4EZIFrRdGxHghEZo4RE0iKsVJGmNstUrtJGKiqab5ojbJaIsEtyo3RSJWR6HoqHGibgIoKCooooCwrFy84W15vz+sWxBkF2RZ3+b5zZwZ99zeZw/4sJyzZ7ECIEBERNKxtnQAIiLqGBY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOC/APv27cPs2bM7fd1nJYTA4MGDn6uxPD09IYSAjY1Nu8do77YJCQn47LPPAABjxozBuXPn2j3m0zT9OkZGRuLo0aOdtu9Zs2bhwIEDnbY/ejaC0/M33bx50zDp9Xpx584dw+NZs2ZZPF9nTEIIMXjwYKPreXp6CiGEsLGxea7Hau+2CQkJ4rPPPmvXGCtWrBBbt25t1zaRkZHi6NGjHTpunXHsOZlvsgU9l3r37m34d0lJCX77298iOzu7xXo2NjbQ6/VdGY2ec1ZWVhBCWDoGdQGeQpFMUFAQrl69iqVLl+LatWtISEiAo6Mj9uzZA61Wi5qaGuzZswfu7u6GbQ4dOoSoqCgA//11+q9//Stqampw6dIlTJ48uUPrenl54fDhw7hx4waysrLwzTffYOvWrU/NvmTJElRUVKC8vBxz585ttmzq1KnIyclBfX09rly5ghUrVhiWHTlyBABQV1eHmzdvYuTIkRg0aBCys7Oh0+lQXV2Nbdu2wcHBwaRj2NZYj82bNw/l5eWoqKjA4sWLDfOtrKywbNkyFBcXQ6fTISUlBU5OTiaNO2LECPz888+4ceMGkpOT0aNHD8Oyx1/Xx5YuXYqysjLcuHED586dQ3BwMCZNmoTly5fjrbfews2bN5GXlwfg0dds1apV+OGHH3Dnzh0MGjSo2dfxce6vv/4adXV1KCwsRHBwsGFZSUkJJkyYYHi8YsUKw9extWP/5CmZUaNG4eTJk6irq8PJkycxatQow7JDhw7h008/xQ8//IAbN27gwIEDcHZ2Nul4kXEscAm5urqib9++8PT0xPz582FtbY2EhAR4enpiwIABaGhowDfffPPU7QMDA3H+/HkolUp88cUXiI+P79C6iYmJOHnyJJydnREbG4t33333qfuZNGkSlixZgokTJ8Lb2xshISHNlt++fRuzZ8+Go6MjQkND8f777yMsLAwAMG7cOACAo6MjevfujePHj8PKygp/+ctf4ObmBl9fX/Tv3x+xsbGmHL42x3ps/Pjx8Pb2xquvvoply5YZCu7DDz9EeHg4goKC4ObmhtraWnz77bdGx1QoFPj++++xdetW9O3bFzt27MCbb77Z6ro+Pj5YsGABVCoV+vTpg0mTJuHy5cs4cOAAPv/8c6SkpKB3794YMWKEYZt3330X8+fPR+/evVFaWtpin4GBgbh48SKUSiVWrFiBXbt2mfSDp7Vj35STkxP27t2Lv/3tb3B2dsZXX32FvXv3om/fvoZ1Zs2ahblz56Jfv37o1q0blixZYnRcMg0LXEKNjY1YsWIF7t+/j7t376Kmpga7du1CQ0MDbt26hdWrVyMoKOip25eWlmLz5s1obGyEWq2Gm5sbXFxc2rVu//79oVKp8Mknn+DBgwc4duwYMjIynjpmREQEEhIScObMGdy5c6dF2R4+fBgFBQUQQiA/Px9JSUltPoeLFy/i3//+N+7fvw+dToevvvqqzfXbO9bKlStx584dFBQUICEhATNnzgQAvPfee/jzn/+M8vJy3L9/H7GxsZg+fbrRC5cjR46EQqHA+vXr8fDhQ6SlpUGj0bS6rl6vR/fu3TF06FDY2tqitLQUly5danP/W7ZswdmzZ6HX6/Hw4cMWy7VarWHs1NRUnD9/HqGhoW3u0xShoaEoKirCtm3boNfrkZycjHPnzuH11183rJOQkICioiLcvXsXqampzX7w0LNhgUuouroa9+7dMzy2s7PDpk2bcPnyZdTX1+PIkSNwcnKCtXXrX97KykrDvxsaGgAAvXr1ate6bm5uqKmpMcwD0OwUwJPc3NyaLX/yVWJAQAAOHjwIrVaLuro6vPfee1AqlU/dX79+/ZCUlISysjLU19dj27Ztba7f3rGezOrm5gbg0TtN0tPTUVtbi9raWhQWFkKv1z/1B2DT519eXt5sXmuvlIFHP5wWLVqE2NhYaLVaJCUl4cUXX2xz/20dewCtjv34OT0LNze3Fs+jtLS02Sm8pt9Dd+7ceer3GrUfC1xCT16gWrx4MYYMGYLAwEA4ODgYfu21srIyW4Zr166hb9++sLOzM8zr379/m+s3XT5gwIBmyxMTE5GRkYH+/fvD0dERmzZtMuRv7YLc559/DiEEhg8fDgcHB7zzzjsmP9+2xmrtuQwYMAAVFRUAHhXllClT4OTkZJjs7OwMy9t6/k1LrbVj0FRSUhLGjh1reGvi2rVrAbR+LNqa/1hrYz/OfPv2bdjb2xuWubq6mrzfiooKeHp6ttj3kz8wyDxY4P8DevfujYaGBtTV1cHJyanVi3Kd7cqVK/jpp58QGxsLhUKBkSNHNvu1+UmpqamYM2cOfH19YWdn1yJj7969UVNTg3v37kGlUmHWrFmGZdXV1dDr9Rg0aFCz9W/duoX6+nq4ubnhj3/8o8nZ2xrrsY8//hh2dnYYOnQo5s6di5SUFADApk2bsHr1akP5KpVKTJs2zeiYP/74Ix4+fIiFCxfC1tYWv/nNbxAQENDquj4+Phg/fjy6deuGu3fvoqGhAY2NjQCAqqoqeHl5tfuHc79+/QxjT58+Hb6+vti3bx8AIC8vDzNmzICtrS1efvllTJ8+3bBda8e+qX379sHHxwczZ86EjY0NIiIiMHToUPzrX/9qVz7qGBb4/4D169fDzs4OOp0Ox48fx/79+7tk3LfffhujRo3C9evXsWrVKqSkpDQ7tdPU/v37sX79ehw8eBDFxcU4ePBgs+W/+93v8Omnn+LGjRv45JNPkJqaaljW0NCA1atX49ixY6itrUVgYCBWrlwJf39/1NfXY+/evdi1a5fJudsa67HDhw+juLgY2dnZ+PLLL5GVlQUA2LBhAzIyMpCZmYkbN27g+PHjCAwMNDrmgwcP8MYbb2DOnDmoqanBW2+99dTM3bt3x5o1a6DT6VBZWYl+/frhT3/6EwBgx44dAIDr16/j559/Nvk5nzhxAt7e3tDpdFi9ejWmT5+OmpoaAI9+WA0ePBi1tbVYuXIlEhMTDdu1duybqqmpwWuvvYbFixfj+vXrWLp0KV577TVcv37d5GzUcVZ49IZwomf2+AKWqe8GIaJnw1fg1GGvvPIKBg0aBCsrK0yaNAlhYWH4/vvvLR2L6BeDd2JSh7m6umLXrl1wdnZGWVkZ3n//fcPNJURkfjyFQkQkKZ5CISKSVJeeQtFqtU+9eYGIiFrn6emJfv36tZjfpQVeWloKlUrVlUMSEUnvaR+7wFMoRESSYoETEUmKBU5EJCkWOBGRpFjgRESSYoETEUmKBU5EJCkWOBGRpFjgRESS4qcRUgv/l/+jxcZePHyUxcYmkg1fgRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKZMK3MHBATt27EBhYSHOnj2LkSNHwsnJCZmZmbhw4QIyMzPh6Oho5qhERNSUSQW+YcMG7N+/H76+vnjppZdQWFiImJgYZGdnw8fHB9nZ2YiJiTF3ViIiasJogffp0wfjxo1DfHw8AODBgweor69HWFgY1Go1AECtViM8PNysQYmIqDmjBT5w4EBUV1cjISEBOTk5iIuLg729PVxcXFBZWQkAqKyshIuLi9nDEhHRfxktcFtbW/j7+2Pjxo3w9/fH7du3Wz1dIoRodfvo6GhoNBpoNBoolcpnT0xERABMKPCysjKUlZXh5MmTAICdO3fC398fVVVVcHV1BQC4urpCq9W2un1cXBxUKhVUKhV0Ol0nRici+mUzWuBVVVW4evUqfHx8AAATJkzA2bNnkZGRgcjISABAZGQkdu/ebd6kRETUjEl/lf7DDz/E9u3b0a1bN1y6dAlz586FtbU1UlNTERUVhdLSUkRERJg7KxERNWFSgZ86dQoqlarF/JCQkE4PREREpuGdmEREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJyqRb6Ym6yv/l/2iRcRcPH2WRcYmeBV+BExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJindiPscsdVfiL5Elj/Uv8S5Q3nHbOfgKnIhIUixwIiJJmXQKpaSkBDdv3oRer8fDhw+hUqng5OSElJQUeHl54fLly4iIiEBdXZ2Z4xIR0WMmvwIfP348/Pz8oFKpAAAxMTHIzs6Gj48PsrOzERMTY7aQRETUUodPoYSFhUGtVgMA1Go1wsPDOysTERGZwKRTKEIIZGZmQgiBv//974iLi4OLiwsqKysBAJWVlXBxcWl12+joaMyfPx8AoFQqOyl21+E7Qcjc+I4M6iiTCnzMmDGoqKjACy+8gKysLJw7d67FOkKIVreNi4tDXFwcAECj0TxDVCIiasqkUygVFRUAgOrqaqSnpyMgIABVVVVwdXUFALi6ukKr1ZovJRERtWC0wO3t7dGrVy/Dv1999VUUFBQgIyMDkZGRAIDIyEjs3r3bvEmJiKgZo6dQXFxckJ6e/mhlW1skJibiwIED0Gg0SE1NRVRUFEpLSxEREWH2sERE9F9GC7ykpAQjRoxoMb+mpgYhISHmyERERCbgnZhERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYp/lZ7oF4qfdS8/vgInIpIUC5yISFIscCIiSbHAiYgkxQInIpIUC5yISFIscCIiSbHAiYgkxQInIpIUC5yISFIscCIiSZlc4NbW1sjJycGePXsAAF5eXjh+/DiKioqQnJwMhUJhtpBERNSSyQX+0UcfobCw0PB47dq1WLduHby9vVFbW4uoqCizBCQiotaZVODu7u4IDQ3F5s2bDfOCg4Oxc+dOAIBarUZ4eLhZAhIRUetMKvD169dj6dKlaGxsBAA4Ozujrq4Oer0eAFBWVgZ3d/dWt42OjoZGo4FGo4FSqeyk2EREZLTAQ0NDodVqkZOT06EB4uLioFKpoFKpoNPpOrQPIiJqyegfdBg9ejSmTZuGqVOnokePHujTpw82bNgAR0dH2NjYQK/Xw8PDA+Xl5V2Rl4iI/sMKgDB15aCgICxZsgSvv/46UlNTkZaWhpSUFGzcuBGnT5/Gxo0b29xeo9FApVJ1KCj/eggRyWrx8FHPtP3TurPD7wNftmwZ/vCHP6CoqAjOzs6Ij49/poBERNQ+7fqbmIcPH8bhw4cBACUlJQgMDDRLKCIiMo53YhIRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkjBZ49+7dceLECeTl5aGgoACxsbEAAC8vLxw/fhxFRUVITk6GQqEwd1YiImrCaIHfu3cPwcHBGDFiBEaMGIHJkycjMDAQa9euxbp16+Dt7Y3a2lpERUV1RV4iIvoPk06h3L59GwCgUCigUCgghEBwcDB27twJAFCr1QgPDzdbSCIiasmkAre2tkZubi60Wi2ysrJw8eJF1NXVQa/XAwDKysrg7u7e6rbR0dHQaDTQaDRQKpWdl5yI6BfOpAJvbGyEn58fPDw8EBAQgF/96lcmDxAXFweVSgWVSgWdTtfhoERE1Fy73oVSX1+PQ4cOYdSoUXB0dISNjQ0AwMPDA+Xl5WYJSERErTNa4EqlEg4ODgCAHj16YOLEiSgsLMShQ4cwffp0AEBkZCR2795t3qRERNSMrbEVXnzxRajVatjY2MDa2hqpqanYu3cvzp49i+TkZKxatQq5ubmIj4/virxERPQfRgs8Pz8f/v7+LeaXlJQgMDDQLKGIiMg43olJRCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaSMFriHhwcOHjyIM2fOoKCgAAsXLgQAODk5ITMzExcuXEBmZiYcHR3NnZWIiJowWuAPHz7E4sWLMWzYMIwcORIffPABfH19ERMTg+zsbPj4+CA7OxsxMTFdkZeIiP7DaIFXVlYiNzcXAHDr1i0UFhbC3d0dYWFhUKvVAAC1Wo3w8HCzBiUiouZs27Oyp6cn/Pz8cOLECbi4uKCyshLAo5J3cXFpdZvo6GjMnz8fAKBUKp8xLhERPWbyRcyePXsiLS0NixYtws2bN1ssF0K0ul1cXBxUKhVUKhV0Ol3HkxIRUTMmFbitrS3S0tKwfft2pKenAwCqqqrg6uoKAHB1dYVWqzVfSiIiasGkAo+Pj0dhYSHWrVtnmJeRkYHIyEgAQGRkJHbv3m2ehERE1Cqj58BHjx6N2bNn4/Tp04aLmcuXL8eaNWuQmpqKqKgolJaWIiIiwuxhiYjov4wW+LFjx2BlZdXqspCQkE4PREREpuGdmEREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJymiBx8fHo6qqCvn5+YZ5Tk5OyMzMxIULF5CZmQlHR0dzZiQiolYYLfAtW7Zg8uTJzebFxMQgOzsbPj4+yM7ORkxMjNkCEhFR64wW+NGjR1FTU9NsXlhYGNRqNQBArVYjPDzcLOGIiOjpbDuykYuLCyorKwEAlZWVcHFxeeq60dHRmD9/PgBAqVR2ZDgiImpFp1zEFEI8dVlcXBxUKhVUKhV0Ol1nDEdEROhggVdVVcHV1RUA4OrqCq1W26mhiIjIuA4VeEZGBiIjIwEAkZGR2L17d6eGIiIi44wWeGJiIn788UcMGTIEV69exbx587BmzRpMnDgRFy5cQEhICNasWdMVWYmIqAmjFzFnzZrV6vyQkJBOD0NERKbjnZhERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYoFTkQkqWcq8EmTJuHcuXMoKirCsmXLOisTERGZoMMFbm1tjW+//RZTpkzB0KFDMXPmTPj6+nZmNiIiakOHCzwgIADFxcUoKSnBgwcPkJycjLCwsM7MRkREbbDt6Ibu7u64evWq4XFZWRkCAwNbrBcdHY358+cDAIYMGQKNRtOh8ZS9lNDpdB0L24mUSuZ4njIwB3PIkKOkpOSZMnh6ej51mejI9Oabb4q4uDjD43feeUd8/fXXHdqXKZNGozHbvplD3gzMwRwy5DBXhg6fQikvL0f//v0Njz08PFBeXt7R3RERUTt1uMA1Gg28vb3h5eUFhUKBGTNmICMjozOzERFRGzp8Dlyv12PBggU4cOAAbGxs8N133+Hs2bOdma2Zf/zjH2bbd3swx/OVAWCOJzFHc89DDnNlsMKjcylERCQZ3olJRCQpFjgRkaSkKPCSkhKcPn0aubm5HX4f+bN6nj42wNraGjk5OdizZ49Fxo+Pj0dVVRXy8/MtMn5TCxcuRH5+PgoKCvDRRx9ZJIOHhwcOHjyIM2fOoKCgAAsXLrRIju7du+PEiRPIy8tDQUEBYmNjLZLDwcEBO3bsQGFhIc6ePYuRI0d2eQYfHx/k5uYapvr6eot9fyxatAgFBQXIz89HYmIiunfv3qn7t/h7JI1NJSUlwtnZ2WLjW1tbi+LiYjFw4EChUChEXl6e8PX1tVie3//+92L79u1iz549Fhl/7Nixws/PT+Tn51v0+2LYsGEiPz9f2NnZCRsbG5GVlSUGDx7c5TlcXV2Fn5+fACB69eolzp8/b7Hvj549ewoAwtbWVhw/flwEBgZ2eYYtW7aIqKgoAUAoFArh4OBg0e8Ta2trce3aNTFgwIAuH9vNzU1cunRJ9OjRQwAQKSkpIjIysvOeG8io5+ljA9zd3REaGorNmzdbZHwAOHr0KGpqaiw2/mO+vr44ceIEGhoaoNfrcfjwYbzxxhtdnqOyshK5ubkAgFu3bqGwsBDu7u5dngMAbt++DQBQKBRQKBQQQnTp+H369MG4ceMQHx8PAHjw4AHq6+u7NMOTJkyYgIsXL+LKlSsWGd/W1hZ2dnawsbGBvb09KioqOm3fUhS4EAKZmZn46aefEB0d3eXjt/axAZb6D7p+/XosXboUjY2NFhn/eVJQUICxY8eib9++sLOzw9SpU5vdXGYJnp6e8PPzw4kTJywyvrW1NXJzc6HVapGVlYWTJ0926fgDBw5EdXU1EhISkJOTg7i4ONjb23dphifNmDEDSUlJFhm7oqICX375Ja5cuYJr166hvr4eWVlZnbZ/KQp8zJgxePnllzFlyhR88MEHGDt2rKUjWURoaCi0Wi1ycnIsHeW5cO7cOaxduxaZmZnYv38/8vLyoNfrLZanZ8+eSEtLw6JFi3Dz5k2LZGhsbISfnx88PDwQEBCAYcOGden4tra28Pf3x8aNG+Hv74/bt28jJiamSzM0pVAoMG3aNOzYscMi4zs6OiIsLAwDBw6Em5sbevbsibfffrvT9i9FgT/+laO6uhrp6ekICAjo0vGfl48NGD16NKZNm4aSkhIkJycjODgYW7du7fIcz5PvvvsOr7zyCoKCglBbW4sLFy5YJIetrS3S0tKwfft2pKenWyRDU/X19Th06BAmT57cpeOWlZWhrKzM8Mp/586d8Pf379IMTU2ZMgU5OTnQarUWGT8kJMTwQVYPHz7Erl278Otf/7pTx7DoBQZjk729vejVq5fh38eOHROTJk3q0gw2Njbi4sWLwsvLy3ARc+jQoRY9LkFBQRa7iAlAeHp6WvwiJgDxwgsvCACif//+orCw0GIXzNRqtVi3bp1Fj4VSqTQ8/x49eogjR46I0NDQLs9x5MgR4ePjIwCIFStWiC+++MJixyQpKUnMmTPHYuMHBASIgoICYWdnJ4BHF3gXLFjQmWNY5omZOg0cOFDk5eWJvLw8UVBQIJYvX26RHFOmTBHnz58XxcXFFsvQdLJkgScmJoqKigpx//59cfXqVTFv3jyLHYcjR46IM2fOiLy8PBEcHGyRDKNHjxZCCHHq1CmRm5srcnNzxZQpU7o8x/Dhw0VOTo44deqUyM/PFx9//LFFjsdLL70kNBqNOHXqlEhPTxeOjo4WyWFvby90Op3o06ePRcZ/PMXGxorCwkKRn58v/vnPf4pu3bp12r55Kz0RkaSkOAdOREQtscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAiciktT/A8WY0L1X2NnjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Images should be shuffled in MNIST, but just to check we'll plot a histogram\n",
    "plt.hist(mini_y_train)\n",
    "plt.title('Training data label distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Training data label distribution')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcY0lEQVR4nO3deVCTd+IG8IcjKngAkgrlENARF51OhTag6zUiFpGusK1j1Vqjsjjt1lp3dZV1pwVb7Wq3v6rbduosUsyqiHhQcXUUFh21TlUsYEHAE1FACJHDC6/w/f3hkkpBE5AkfNvnM/POkPf6PknwSXjfvNEGgAAREUnH1toBiIioY1jgRESSYoETEUmKBU5EJCkWOBGRpFjgRESSYoH/Cuzbtw+zZs3q9HWflRACAwcO7FJj+fj4QAgBOzu7do/R3m2Tk5Px8ccfAwBGjRqFkpKSdo/5JI8/j2q1GkePHu20fc+YMQMHDhzotP3RsxGcut508+ZNw6TX68WdO3cMt2fMmGH1fJ0xCSHEwIEDja7n4+MjhBDCzs6uS4/V3m2Tk5PFxx9/3K4x4uPjxaZNm9q1jVqtFkePHu3Q49YZjz0n8032oC6pd+/ehp9LS0vxhz/8AdnZ2a3Ws7Ozg16vt2Q06uJsbGwghLB2DLIAHkKRzNixY3H16lUsWbIE165dQ3JyMpydnbFnzx5otVrU1tZiz5498PT0NGxz6NAhxMTEAPjpz+l//OMfqK2txaVLlzBx4sQOrevr64vDhw/jxo0byMrKwpdffolNmzY9MfvixYtRWVmJiooKzJkzp8WySZMmITc3Fw0NDbhy5Qri4+MNy44cOQIAqK+vx82bNzF8+HAMGDAA2dnZ0Ol0qKmpwebNm+Hk5GTSY/i0sZrNnTsXFRUVqKysxKJFiwzzbWxssHTpUly4cAE6nQ7btm2Di4uLSeMOGzYMP/zwA27cuIHU1FT06NHDsKz5eW22ZMkSlJeX48aNGygpKUFoaCjCw8OxbNkyvPHGG7h58yby8/MBPHrOVqxYge+++w537tzBgAEDWjyPzbm/+OIL1NfXo7i4GKGhoYZlpaWlGD9+vOF2fHy84Xls67H/+SGZESNG4OTJk6ivr8fJkycxYsQIw7JDhw7ho48+wnfffYcbN27gwIEDcHV1NenxIuNY4BJyd3dH37594ePjg3nz5sHW1hbJycnw8fFB//790djYiC+//PKJ24eEhODs2bNQKpX49NNPkZSU1KF1U1JScPLkSbi6uiIhIQFvvfXWE/cTHh6OxYsXY8KECRg0aBDCwsJaLL99+zZmzZoFZ2dnREZG4p133kFUVBQAYMyYMQAAZ2dn9O7dG8ePH4eNjQ3+/ve/w8PDAwEBAfD29kZCQoIpD99Tx2o2btw4DBo0CK+88gqWLl1qKLj33nsP0dHRGDt2LDw8PFBXV4evvvrK6JgKhQLffvstNm3ahL59+2L79u14/fXX21zX398f8+fPh0qlQp8+fRAeHo7Lly/jwIED+OSTT7Bt2zb07t0bw4YNM2zz1ltvYd68eejduzfKyspa7TMkJAQXL16EUqlEfHw8du3aZdILT1uP/eNcXFywd+9e/POf/4Srqys+//xz7N27F3379jWsM2PGDMyZMwf9+vVDt27dsHjxYqPjkmlY4BJqampCfHw87t+/j7t376K2tha7du1CY2Mjbt26hZUrV2Ls2LFP3L6srAwbNmxAU1MTNBoNPDw84Obm1q51vb29oVKp8OGHH+LBgwc4duwYMjIynjjm1KlTkZycjDNnzuDOnTutyvbw4cMoLCyEEAIFBQXYunXrU+/DxYsX8d///hf379+HTqfD559//tT12zvW8uXLcefOHRQWFiI5ORnTp08HALz99tv429/+hoqKCty/fx8JCQmYMmWK0ROXw4cPh0KhwNq1a/Hw4UPs3LkTOTk5ba6r1+vRvXt3DBkyBPb29igrK8OlS5eeuv+NGzeiqKgIer0eDx8+bLVcq9Uaxk5LS8PZs2cRGRn51H2aIjIyEufPn8fmzZuh1+uRmpqKkpIS/O53vzOsk5ycjPPnz+Pu3btIS0tr8cJDz4YFLqGamhrcu3fPcNvBwQHr16/H5cuX0dDQgCNHjsDFxQW2tm0/vVVVVYafGxsbAQC9evVq17oeHh6ora01zAPQ4hDAz3l4eLRY/vN3icHBwTh48CC0Wi3q6+vx9ttvQ6lUPnF//fr1w9atW1FeXo6GhgZs3rz5qeu3d6yfZ/Xw8ADw6JMm6enpqKurQ11dHYqLi6HX65/4Avj4/a+oqGgxr613ysCjF6eFCxciISEBWq0WW7duxfPPP//U/T/tsQfQ5tjN9+lZeHh4tLofZWVlLQ7hPf47dOfOnSf+rlH7scAl9PMTVIsWLcLgwYMREhICJycnw5+9NjY2Zstw7do19O3bFw4ODoZ53t7eT13/8eX9+/dvsTwlJQUZGRnw9vaGs7Mz1q9fb8jf1gm5Tz75BEIIvPDCC3BycsLMmTNNvr9PG6ut+9K/f39UVlYCeFSUERERcHFxMUwODg6G5U+7/4+XWluPweO2bt2K0aNHGz6auHr1agBtPxZPm9+srbGbM9++fRuOjo6GZe7u7ibvt7KyEj4+Pq32/fMXDDIPFvgvQO/evdHY2Ij6+nq4uLi0eVKus125cgWnTp1CQkICFAoFhg8f3uLP5p9LS0vD7NmzERAQAAcHh1YZe/fujdraWty7dw8qlQozZswwLKupqYFer8eAAQNarH/r1i00NDTAw8MDf/nLX0zO/rSxmn3wwQdwcHDAkCFDMGfOHGzbtg0AsH79eqxcudJQvkqlEpMnTzY65vfff4+HDx9iwYIFsLe3x+9//3sEBwe3ua6/vz/GjRuHbt264e7du2hsbERTUxMAoLq6Gr6+vu1+ce7Xr59h7ClTpiAgIAD79u0DAOTn52PatGmwt7fHSy+9hClTphi2a+uxf9y+ffvg7++P6dOnw87ODlOnTsWQIUPwn//8p135qGNY4L8Aa9euhYODA3Q6HY4fP479+/dbZNw333wTI0aMwPXr17FixQps27atxaGdx+3fvx9r167FwYMHceHCBRw8eLDF8j/+8Y/46KOPcOPGDXz44YdIS0szLGtsbMTKlStx7Ngx1NXVISQkBMuXL0dQUBAaGhqwd+9e7Nq1y+TcTxur2eHDh3HhwgVkZ2fjs88+Q1ZWFgBg3bp1yMjIQGZmJm7cuIHjx48jJCTE6JgPHjzAa6+9htmzZ6O2thZvvPHGEzN3794dq1atgk6nQ1VVFfr164e//vWvAIDt27cDAK5fv44ffvjB5Pt84sQJDBo0CDqdDitXrsSUKVNQW1sL4NGL1cCBA1FXV4fly5cjJSXFsF1bj/3jamtr8eqrr2LRokW4fv06lixZgldffRXXr183ORt1nA0efSCc6Jk1n8Ay9dMgRPRs+A6cOuzll1/GgAEDYGNjg/DwcERFReHbb7+1diyiXw1eiUkd5u7ujl27dsHV1RXl5eV45513DBeXEJH58RAKEZGkeAiFiEhSFj2EotVqn3jxAhERtc3Hxwf9+vVrNd+iBV5WVgaVSmXJIYmIpPekr13gIRQiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJMUCJyKSFAuciEhSLHAiIknx2wipS/m/gu+tMu6iF0ZYZVyiZ8F34EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpkwrcyckJ27dvR3FxMYqKijB8+HC4uLggMzMT586dQ2ZmJpydnc0clYiIHmdSga9btw779+9HQEAAXnzxRRQXFyMuLg7Z2dnw9/dHdnY24uLizJ2ViIgeY7TA+/TpgzFjxiApKQkA8ODBAzQ0NCAqKgoajQYAoNFoEB0dbdagRETUktEC9/PzQ01NDZKTk5Gbm4vExEQ4OjrCzc0NVVVVAICqqiq4ubmZPSwREf3EaIHb29sjKCgIX3/9NYKCgnD79u02D5cIIdrcPjY2Fjk5OcjJyYFSqXz2xEREBMCEAi8vL0d5eTlOnjwJANixYweCgoJQXV0Nd3d3AIC7uzu0Wm2b2ycmJkKlUkGlUkGn03VidCKiXzejBV5dXY2rV6/C398fADB+/HgUFRUhIyMDarUaAKBWq7F7927zJiUiohZM+i/V3nvvPWzZsgXdunXDpUuXMGfOHNja2iItLQ0xMTEoKyvD1KlTzZ2ViIgeY1KBnz59GiqVqtX8sLCwTg9ERESm4ZWYRESSYoETEUmKBU5EJCkWOBGRpFjgRESSYoETEUmKBU5EJCkWOBGRpFjgRESSYoETEUmKBU5EJCkWOBGRpFjgRESSYoETEUmKBU5EJCkWOBGRpFjgRESSMul/5CHr+L+C760y7qIXRlhlXCJz+6X9m+I7cCIiSbHAiYgkxQInIpIUC5yISFIscCIiSZn0KZTS0lLcvHkTer0eDx8+hEqlgouLC7Zt2wZfX19cvnwZU6dORX19vZnjEhFRM5PfgY8bNw6BgYFQqVQAgLi4OGRnZ8Pf3x/Z2dmIi4szW0giImqtw4dQoqKioNFoAAAajQbR0dGdlYmIiExgUoELIZCZmYlTp04hNjYWAODm5oaqqioAQFVVFdzc3MyXkoiIWjHpGPioUaNQWVmJ5557DllZWSgpKWm1jhCizW1jY2Mxb948AIBSqXyGqES/TL+0qwPJckx6B15ZWQkAqKmpQXp6OoKDg1FdXQ13d3cAgLu7O7RabZvbJiYmQqVSQaVSQafTdVJsIiIyWuCOjo7o1auX4edXXnkFhYWFyMjIgFqtBgCo1Wrs3r3bvEmJiKgFo4dQ3NzckJ6e/mhle3ukpKTgwIEDyMnJQVpaGmJiYlBWVoapU6eaPSyRuVjrMAbRszBa4KWlpRg2bFir+bW1tQgLCzNHJiIiMgGvxCQikhS/D5xa4eEEIjnwHTgRkaRY4EREkuIhFKJfKR4qkx/fgRMRSYoFTkQkKRY4EZGkWOBERJJigRMRSYqfQjGCZ+qJqKviO3AiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJCXNxwj5cT4iopb4DpyISFIscCIiSbHAiYgkxQInIpIUC5yISFImF7itrS1yc3OxZ88eAICvry+OHz+O8+fPIzU1FQqFwmwhiYioNZML/P3330dxcbHh9urVq7FmzRoMGjQIdXV1iImJMUtAIiJqm0kF7unpicjISGzYsMEwLzQ0FDt27AAAaDQaREdHmyUgERG1zaQCX7t2LZYsWYKmpiYAgKurK+rr66HX6wEA5eXl8PT0bHPb2NhY5OTkICcnB0qlspNiExGR0QKPjIyEVqtFbm5uhwZITEyESqWCSqWCTqfr0D6IiKg1o5fSjxw5EpMnT8akSZPQo0cP9OnTB+vWrYOzszPs7Oyg1+vh5eWFiooKS+QlIqL/MfoOfNmyZfD29oafnx+mTZuGgwcPYubMmTh06BCmTJkCAFCr1di9e7fZwxIR0U86/DnwpUuX4s9//jPOnz8PV1dXJCUldWYuIiIyol3fRnj48GEcPnwYAFBaWoqQkBCzhCIiIuN4JSYRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJigVORCQpFjgRkaRY4EREkmKBExFJymiBd+/eHSdOnEB+fj4KCwuRkJAAAPD19cXx48dx/vx5pKamQqFQmDsrERE9xmiB37t3D6GhoRg2bBiGDRuGiRMnIiQkBKtXr8aaNWswaNAg1NXVISYmxhJ5iYjof0w6hHL79m0AgEKhgEKhgBACoaGh2LFjBwBAo9EgOjrabCGJiKg1kwrc1tYWeXl50Gq1yMrKwsWLF1FfXw+9Xg8AKC8vh6enZ5vbxsbGIicnBzk5OVAqlZ2XnIjoV86kAm9qakJgYCC8vLwQHByM3/zmNyYPkJiYCJVKBZVKBZ1O1+GgRETUUrs+hdLQ0IBDhw5hxIgRcHZ2hp2dHQDAy8sLFRUVZglIRERtM1rgSqUSTk5OAIAePXpgwoQJKC4uxqFDhzBlyhQAgFqtxu7du82blIiIWrA3tsLzzz8PjUYDOzs72NraIi0tDXv37kVRURFSU1OxYsUK5OXlISkpyRJ5iYjof4wWeEFBAYKCglrNLy0tRUhIiFlCERGRcbwSk4hIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCRltMC9vLxw8OBBnDlzBoWFhViwYAEAwMXFBZmZmTh37hwyMzPh7Oxs7qxERPQYowX+8OFDLFq0CEOHDsXw4cPx7rvvIiAgAHFxccjOzoa/vz+ys7MRFxdnibxERPQ/Rgu8qqoKeXl5AIBbt26huLgYnp6eiIqKgkajAQBoNBpER0ebNSgREbVk356VfXx8EBgYiBMnTsDNzQ1VVVUAHpW8m5tbm9vExsZi3rx5AAClUvmMcYmIqJnJJzF79uyJnTt3YuHChbh582ar5UKINrdLTEyESqWCSqWCTqfreFIiImrBpAK3t7fHzp07sWXLFqSnpwMAqqur4e7uDgBwd3eHVqs1X0oiImrFpAJPSkpCcXEx1qxZY5iXkZEBtVoNAFCr1di9e7d5EhIRUZuMHgMfOXIkZs2ahR9//NFwMnPZsmVYtWoV0tLSEBMTg7KyMkydOtXsYYmI6CdGC/zYsWOwsbFpc1lYWFinByIiItPwSkwiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJMUCJyKSFAuciEhSLHAiIkmxwImIJMUCJyKSlNECT0pKQnV1NQoKCgzzXFxckJmZiXPnziEzMxPOzs7mzEhERG0wWuAbN27ExIkTW8yLi4tDdnY2/P39kZ2djbi4OLMFJCKithkt8KNHj6K2trbFvKioKGg0GgCARqNBdHS0WcIREdGT2XdkIzc3N1RVVQEAqqqq4Obm9sR1Y2NjMW/ePACAUqnsyHBERNSGTjmJKYR44rLExESoVCqoVCrodLrOGI6IiNDBAq+uroa7uzsAwN3dHVqttlNDERGRcR0q8IyMDKjVagCAWq3G7t27OzUUEREZZ7TAU1JS8P3332Pw4MG4evUq5s6di1WrVmHChAk4d+4cwsLCsGrVKktkJSKixxg9iTljxow254eFhXV6GCIiMh2vxCQikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJscCJiCTFAicikhQLnIhIUixwIiJJPVOBh4eHo6SkBOfPn8fSpUs7KxMREZmgwwVua2uLr776ChERERgyZAimT5+OgICAzsxGRERP0eECDw4OxoULF1BaWooHDx4gNTUVUVFRnZmNiIiewr6jG3p6euLq1auG2+Xl5QgJCWm1XmxsLObNmwcAGDx4MHJycjo0nrKXEjqdrmNhO5FSyRxdKQNzMIcMOUpLS58pg4+PzxOXiY5Mr7/+ukhMTDTcnjlzpvjiiy86tC9TppycHLPtmznkzcAczCFDDnNl6PAhlIqKCnh7extue3l5oaKioqO7IyKidupwgefk5GDQoEHw9fWFQqHAtGnTkJGR0ZnZiIjoKTp8DFyv12P+/Pk4cOAA7Ozs8M0336CoqKgzs7Xwr3/9y2z7bg/m6FoZAOb4OeZoqSvkMFcGGzw6lkJERJLhlZhERJJigRMRSarLF3j37t1x4sQJ5Ofno7CwEAkJCVbJsWDBAhQUFKCwsBDvv/++VTIAjz5P+uOPPyIvL6/Dn6nvDAsXLkRhYSEKCgqQkpKC7t27WzyDv78/8vLyDFNDQ4PVnhsnJyds374dxcXFKCoqwvDhwy2eISkpCdXV1SgoKLD42M28vLxw8OBBnDlzBoWFhViwYIHVsnSlr/qwtbVFbm4u9uzZ0+n7tvpnJI1NPXv2FACEvb29OH78uAgJCbHo+EOHDhUFBQXCwcFB2NnZiaysLDFw4ECrPBalpaXC1dXVqs+Hh4eHuHTpkujRo4cAILZt2ybUarVVM9na2opr166J/v37W2X8jRs3ipiYGAFAKBQK4eTkZPEMo0ePFoGBgaKgoMBqz4O7u7sIDAwUAESvXr3E2bNnRUBAgFV+Hy5cuCD8/PyEQqEQ+fn5VsnRPP3pT38SW7ZsEXv27Onc+wkJ3L59GwCgUCigUCgghLDo+AEBAThx4gQaGxuh1+tx+PBhvPbaaxbN0NXY29vDwcEBdnZ2cHR0RGVlpVXzjB8/HhcvXsSVK1csPnafPn0wZswYJCUlAQAePHiAhoYGi+c4evQoamtrLT7u46qqqpCXlwcAuHXrFoqLi+Hp6WnxHF3pqz48PT0RGRmJDRs2dPq+pShwW1tb5OXlQavVIisrCydPnrTo+IWFhRg9ejT69u0LBwcHTJo0qcVFTJYkhEBmZiZOnTqF2NhYq2SorKzEZ599hitXruDatWtoaGhAVlaWVbI0mzZtGrZu3WqVsf38/FBTU4Pk5GTk5uYiMTERjo6OVsnSlfj4+CAwMBAnTpyw+NhtfdWHNV5IAGDt2rVYsmQJmpqaOn3fUhR4U1MTAgMD4eXlheDgYAwdOtSi45eUlGD16tXIzMzE/v37kZ+fD71eb9EMzUaNGoWXXnoJERERePfddzF69GiLZ3B2dkZUVBT8/Pzg4eGBnj174s0337R4jmYKhQKTJ0/G9u3brTK+vb09goKC8PXXXyMoKAi3b99GXFycVbJ0FT179sTOnTuxcOFC3Lx509pxrCYyMhJarRa5ublm2b8UBd6soaEBhw4dwsSJEy0+9jfffIOXX34ZY8eORV1dHc6dO2fxDAAMhypqamqQnp6O4OBgi2cICwszfDnPw4cPsWvXLvz2t7+1eI5mERERyM3NhVartcr45eXlKC8vN/xluGPHDgQFBVklS1dgb2+PnTt3YsuWLUhPT7dKhq7yVR8jR47E5MmTUVpaitTUVISGhmLTpk2dOobVDuybMimVSsMJoR49eogjR46IyMhIi+d47rnnBADh7e0tiouLrXKSytHRUfTq1cvw87Fjx0R4eLjFcwQHB4vCwkLh4OAggEcn8ObPn2+135GtW7eK2bNnW218AOLIkSPC399fABDx8fHi008/tUoOHx8fq57EBCA0Go1Ys2aNVTPY2dmJixcvCl9fX8NJzCFDhlg109ixYzv9JCaseYdMmV544QWRm5srTp8+LQoKCsQHH3xglRxHjhwRZ86cEfn5+SI0NNQqGfz8/ER+fr7Iz88XhYWFYtmyZVZ7XhISEkRxcbEoKCgQ//73v0W3bt2sksPR0VHodDrRp08fqz0WAMSLL74ocnJyxOnTp0V6erpwdna2eIaUlBRRWVkp7t+/L65evSrmzp1r8QwjR44UQghx+vRpkZeXJ/Ly8kRERIRVnpOIiAhx9uxZceHCBav+W2mezFHgvJSeiEhSUh0DJyKin7DAiYgkxQInIpIUC5yISFIscCIiSbHAiYgkxQInIpLU/wOxgcLA18Z4iQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(mini_y_test)\n",
    "plt.title('Training data label distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we need to standardise the data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = mini_X_train.mean(axis = 0)\n",
    "mini_X_train -= mean\n",
    "\n",
    "std = mini_X_train.std(axis = 0)\n",
    "mini_X_train /= std\n",
    "\n",
    "mini_X_test -= mean\n",
    "mini_X_test /= std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that numbers appear in the data set with similar frequency, so this should not be a problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can use the face that cells have indices\n",
    "index_array = mini_X_train.index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1),\n",
       " (0, 2),\n",
       " (0, 3),\n",
       " (0, 4),\n",
       " (0, 5),\n",
       " (0, 6),\n",
       " (0, 7),\n",
       " (0, 8),\n",
       " (0, 9),\n",
       " (0, 10)]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We need to obtain a new dataset with each combination of values\n",
    "image_combinations = list(itertools.combinations(index_array, 2))\n",
    "image_combinations[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_combinations[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pixel1      0.0\n",
       "pixel2      0.0\n",
       "pixel3      0.0\n",
       "pixel4      0.0\n",
       "pixel5      0.0\n",
       "           ... \n",
       "pixel780    0.0\n",
       "pixel781    0.0\n",
       "pixel782    0.0\n",
       "pixel783    0.0\n",
       "pixel784    0.0\n",
       "Name: 0, Length: 784, dtype: float64"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pixel array for image A, index 0\n",
    "mini_X_train.loc[image_combinations[0][0],]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pixel1      0.0\n",
       "pixel2      0.0\n",
       "pixel3      0.0\n",
       "pixel4      0.0\n",
       "pixel5      0.0\n",
       "           ... \n",
       "pixel780    0.0\n",
       "pixel781    0.0\n",
       "pixel782    0.0\n",
       "pixel783    0.0\n",
       "pixel784    0.0\n",
       "Name: 1, Length: 784, dtype: float64"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pixel array for image B, index 1\n",
    "mini_X_train.loc[image_combinations[0][1],]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First just try putting them one on top of the other\n",
    "concatenated_dataset_X = []\n",
    "concatenated_dataset_y = []\n",
    "\n",
    "for combination in image_combinations:\n",
    "#     print(combination)\n",
    "    \n",
    "    index_A = combination[0]\n",
    "    index_B = combination[1]\n",
    "    \n",
    "    image_A = mini_X_train.loc[index_A,]\n",
    "    image_B = mini_X_train.loc[index_B,]\n",
    "    \n",
    "    intermediate_label_A = mini_y_train.loc[index_A,]\n",
    "    intermediate_label_B = mini_y_train.loc[index_B,]\n",
    "    \n",
    "    concatenated_images = np.concatenate((np.array(image_A.values),np.array(image_B.values)), axis = 0)\n",
    "    \n",
    "    final_label = min(intermediate_label_A, intermediate_label_B)\n",
    "    \n",
    "#     print(\"image_A\\n\"+ str(image_A))\n",
    "#     print(\"image_B\\n\"+ str(image_B))\n",
    "#     print(\"Concatenated image\\n\"+ str(len(concatenated_images)))\n",
    "#     print(\"label_A\\n\"+ str(intermediate_label_A))\n",
    "#     print(\"label_B\\n\"+ str(intermediate_label_B))\n",
    "#     print(\"Final_label\\n\"+ str(final_label))\n",
    "    \n",
    "    concatenated_dataset_X.append(concatenated_images)\n",
    "    concatenated_dataset_y.append(final_label)\n",
    "    \n",
    "#     if i == 20:\n",
    "#         break\n",
    "    \n",
    "    \n",
    "#     i = i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(60500, 60501),\n",
       " (60500, 60502),\n",
       " (60500, 60503),\n",
       " (60500, 60504),\n",
       " (60500, 60505),\n",
       " (60500, 60506),\n",
       " (60500, 60507),\n",
       " (60500, 60508),\n",
       " (60500, 60509),\n",
       " (60500, 60510)]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_array = mini_X_test.index.values\n",
    "image_combinations = list(itertools.combinations(index_array, 2))\n",
    "image_combinations[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_array = mini_X_test.index.values\n",
    "image_combinations = list(itertools.combinations(index_array, 2))\n",
    "\n",
    "# First just try putting them one on top of the other\n",
    "concatenated_dataset_X_test = []\n",
    "concatenated_dataset_y_test = []\n",
    "\n",
    "for combination in image_combinations:\n",
    "#     print(combination)\n",
    "    \n",
    "    index_A = combination[0]\n",
    "    index_B = combination[1]\n",
    "    \n",
    "    image_A = mini_X_test.loc[index_A,]\n",
    "    image_B = mini_X_test.loc[index_B,]\n",
    "    \n",
    "    intermediate_label_A = mini_y_test.loc[index_A,]\n",
    "    intermediate_label_B = mini_y_test.loc[index_B,]\n",
    "    \n",
    "    concatenated_images = np.concatenate((np.array(image_A.values),np.array(image_B.values)), axis = 0)\n",
    "    \n",
    "    final_label = min(intermediate_label_A, intermediate_label_B)\n",
    "    \n",
    "    concatenated_dataset_X_test.append(concatenated_images)\n",
    "    concatenated_dataset_y_test.append(final_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(124750, 1568)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(concatenated_dataset_X_test).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0., 0., 0., ..., 0., 0., 0.]),\n",
       " array([0., 0., 0., ..., 0., 0., 0.]),\n",
       " array([0., 0., 0., ..., 0., 0., 0.])]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Take a look at the data\n",
    "concatenated_dataset_X[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0', '4', '1']"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Take a look at the output\n",
    "concatenated_dataset_y[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's now one-hot encode the labels\n",
    "one_hot_train_labels = to_categorical(concatenated_dataset_y)\n",
    "one_hot_test_labels = to_categorical(concatenated_dataset_y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# See if a simple neural network will work\n",
    "model = tf.keras.Sequential()\n",
    "model.add(layers.Dense(64, activation = 'relu', input_shape = (1568,)))\n",
    "model.add(layers.Dense(64, activation = 'relu'))\n",
    "model.add(layers.Dense(10, activation = 'softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Softmax is used as the final layer because we want to output probabilities over the 10 possible digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer = 'rmsprop',\n",
    "             loss = 'categorical_crossentropy',\n",
    "             metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use only some of the test data as validation data. TODO "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use cateforical_crossentropy here as this calculates the distance between two probability distributions, here: the distribution output by the network and the true distribution of the labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'T'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-129-f8398757c655>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Let's not train the network for 20 epochs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m history = model.fit(concatenated_dataset_X.T,\n\u001b[0m\u001b[1;32m      3\u001b[0m                     \u001b[0mone_hot_train_labels\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                     \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m                     \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'T'"
     ]
    }
   ],
   "source": [
    "# Let's not train the network for 20 epochs\n",
    "history = model.fit(concatenated_dataset_X,\n",
    "                    one_hot_train_labels,\n",
    "                    epochs = 20,\n",
    "                    batch_size = 16,\n",
    "                    validation_data = (concatenated_dataset_X_test,one_hot_test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "general_ML",
   "language": "python",
   "name": "general_ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}