max-pool2d-nchw-perf/max-pool2d.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Jul 28 11:49:42 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.36       Driver Version: 450.36       CUDA Version: 11.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  TITAN V             Off  | 00000000:17:00.0 Off |                  N/A |\n",
      "| 42%   54C    P0    37W / 250W |      0MiB / 12066MiB |      0%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  TITAN V             Off  | 00000000:65:00.0 Off |                  N/A |\n",
      "| 26%   38C    P0    34W / 250W |      0MiB / 12066MiB |      8%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "! nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "error = 10 ** -5 \n",
    "\n",
    "def main(device):\n",
    "    import torch \n",
    "    print(torch.__version__)\n",
    "    print()\n",
    "\n",
    "    for N in [32]:\n",
    "        for C in [3, 32, 128, 256]:\n",
    "            for H, W in [(7, 7), (28, 28), (56, 56), (112, 112)]:\n",
    "                for dtype in [torch.float16, torch.float32]:\n",
    "                    torch.cuda.empty_cache() # this solves the problem where pytorch GPU tensor is not freed in Jupyter\n",
    "\n",
    "                    ks = 3\n",
    "                    print(f\"(N,C,H,W,dtype)=({N},{C},{H},{W},{dtype})\")\n",
    "\n",
    "                    x = torch.randn(N, C, H, W, device=device, dtype=dtype)\n",
    "                    x.requires_grad_()\n",
    "\n",
    "                    net = torch.nn.MaxPool2d(kernel_size=ks)\n",
    "                    y = net(x)\n",
    "                    g = torch.randn_like(y)\n",
    "                    y.backward(g, retain_graph=True)\n",
    "\n",
    "                    def forward():\n",
    "                        net(x)\n",
    "                        torch.cuda.synchronize()\n",
    "\n",
    "                    print(\"forward: \")\n",
    "                    %timeit -n 1000 forward()\n",
    "                    \n",
    "                    torch.cuda.synchronize()\n",
    "\n",
    "                    def backward(): \n",
    "                        y.backward(g, retain_graph=True)\n",
    "                        torch.cuda.synchronize()\n",
    "\n",
    "                    print(\"backward: \")\n",
    "                    %timeit -n 1000 backward()\n",
    "\n",
    "                    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7.0a0+f805184\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,7,7,torch.float16)\n",
      "forward: \n",
      "28 µs ± 4.77 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "144 µs ± 8.88 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,7,7,torch.float32)\n",
      "forward: \n",
      "28.2 µs ± 5.99 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "143 µs ± 11.1 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,28,28,torch.float16)\n",
      "forward: \n",
      "28.8 µs ± 6.29 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "143 µs ± 11.9 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,28,28,torch.float32)\n",
      "forward: \n",
      "28.3 µs ± 5.96 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "138 µs ± 11.2 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,56,56,torch.float16)\n",
      "forward: \n",
      "28.5 µs ± 5.67 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "143 µs ± 11.5 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,56,56,torch.float32)\n",
      "forward: \n",
      "28.2 µs ± 6.06 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "144 µs ± 4.88 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,112,112,torch.float16)\n",
      "forward: \n",
      "28.1 µs ± 5.5 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "200 µs ± 929 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,112,112,torch.float32)\n",
      "forward: \n",
      "32.7 µs ± 4.75 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "221 µs ± 3.69 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,7,7,torch.float16)\n",
      "forward: \n",
      "28.5 µs ± 5.17 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "144 µs ± 10.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,7,7,torch.float32)\n",
      "forward: \n",
      "28.2 µs ± 5.79 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "143 µs ± 10.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,28,28,torch.float16)\n",
      "forward: \n",
      "28.4 µs ± 5.27 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "161 µs ± 2.9 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,28,28,torch.float32)\n",
      "forward: \n",
      "28 µs ± 5.51 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "178 µs ± 930 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,56,56,torch.float16)\n",
      "forward: \n",
      "39.3 µs ± 3.78 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "346 µs ± 106 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,56,56,torch.float32)\n",
      "forward: \n",
      "49.9 µs ± 173 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "403 µs ± 191 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,112,112,torch.float16)\n",
      "forward: \n",
      "87.3 µs ± 314 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.31 ms ± 6.64 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,112,112,torch.float32)\n",
      "forward: \n",
      "133 µs ± 408 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.52 ms ± 42.6 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,7,7,torch.float16)\n",
      "forward: \n",
      "25.7 µs ± 96.7 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "143 µs ± 9.74 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,7,7,torch.float32)\n",
      "forward: \n",
      "28.1 µs ± 5.69 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "143 µs ± 11 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,28,28,torch.float16)\n",
      "forward: \n",
      "40.9 µs ± 3.89 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "346 µs ± 2.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,28,28,torch.float32)\n",
      "forward: \n",
      "51.9 µs ± 129 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "404 µs ± 6.1 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,56,56,torch.float16)\n",
      "forward: \n",
      "86.2 µs ± 194 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.28 ms ± 33.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,56,56,torch.float32)\n",
      "forward: \n",
      "134 µs ± 260 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.55 ms ± 4.26 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,112,112,torch.float16)\n",
      "forward: \n",
      "284 µs ± 1.44 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "4.83 ms ± 2.09 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,112,112,torch.float32)\n",
      "forward: \n",
      "466 µs ± 1.34 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "5.69 ms ± 56.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,7,7,torch.float16)\n",
      "forward: \n",
      "26.5 µs ± 1.59 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "90.3 µs ± 22.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,7,7,torch.float32)\n",
      "forward: \n",
      "27.5 µs ± 4.54 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "140 µs ± 7.33 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,28,28,torch.float16)\n",
      "forward: \n",
      "57.8 µs ± 2.79 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "635 µs ± 3.88 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,28,28,torch.float32)\n",
      "forward: \n",
      "82.2 µs ± 302 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "752 µs ± 350 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,56,56,torch.float16)\n",
      "forward: \n",
      "151 µs ± 704 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "2.47 ms ± 3.41 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,56,56,torch.float32)\n",
      "forward: \n",
      "247 µs ± 688 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "2.93 ms ± 1.36 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,112,112,torch.float16)\n",
      "forward: \n",
      "546 µs ± 2.69 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "9.51 ms ± 2.05 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,112,112,torch.float32)\n",
      "forward: \n",
      "911 µs ± 3.24 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "11.3 ms ± 4.11 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "main(\"cuda:0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## After"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7.0a0+d221193\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,7,7,torch.float16)\n",
      "forward: \n",
      "24.8 µs ± 268 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "134 µs ± 11.9 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,7,7,torch.float32)\n",
      "forward: \n",
      "27.5 µs ± 6.09 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "133 µs ± 7.24 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,28,28,torch.float16)\n",
      "forward: \n",
      "27.5 µs ± 5.68 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "134 µs ± 10.5 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,28,28,torch.float32)\n",
      "forward: \n",
      "27.6 µs ± 5.73 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "132 µs ± 10.9 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,56,56,torch.float16)\n",
      "forward: \n",
      "27.4 µs ± 5.18 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "142 µs ± 4.75 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,56,56,torch.float32)\n",
      "forward: \n",
      "27.3 µs ± 5.05 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "136 µs ± 8.88 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,112,112,torch.float16)\n",
      "forward: \n",
      "27.5 µs ± 5.39 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "136 µs ± 8.55 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,3,112,112,torch.float32)\n",
      "forward: \n",
      "32.4 µs ± 4.24 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "144 µs ± 6.58 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,7,7,torch.float16)\n",
      "forward: \n",
      "27.4 µs ± 5.32 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "141 µs ± 12.3 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,7,7,torch.float32)\n",
      "forward: \n",
      "27.6 µs ± 5.74 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "131 µs ± 7.52 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,28,28,torch.float16)\n",
      "forward: \n",
      "27.8 µs ± 6.16 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "137 µs ± 9.24 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,28,28,torch.float32)\n",
      "forward: \n",
      "27.6 µs ± 6.07 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "135 µs ± 12.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,56,56,torch.float16)\n",
      "forward: \n",
      "39.2 µs ± 4.35 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "195 µs ± 3.33 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,56,56,torch.float32)\n",
      "forward: \n",
      "50.9 µs ± 3.06 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "226 µs ± 6.07 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,112,112,torch.float16)\n",
      "forward: \n",
      "87.6 µs ± 2.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "521 µs ± 1.44 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,32,112,112,torch.float32)\n",
      "forward: \n",
      "132 µs ± 129 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "644 µs ± 120 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,7,7,torch.float16)\n",
      "forward: \n",
      "25 µs ± 186 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "61.9 µs ± 254 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,7,7,torch.float32)\n",
      "forward: \n",
      "25.4 µs ± 579 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "64.3 µs ± 4.86 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,28,28,torch.float16)\n",
      "forward: \n",
      "40.5 µs ± 3.48 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "198 µs ± 2 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,28,28,torch.float32)\n",
      "forward: \n",
      "52.7 µs ± 2.88 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "226 µs ± 3.83 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,56,56,torch.float16)\n",
      "forward: \n",
      "86.4 µs ± 1.77 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "533 µs ± 1.67 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,56,56,torch.float32)\n",
      "forward: \n",
      "134 µs ± 134 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "647 µs ± 4.53 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,112,112,torch.float16)\n",
      "forward: \n",
      "283 µs ± 340 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.95 ms ± 196 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,128,112,112,torch.float32)\n",
      "forward: \n",
      "466 µs ± 292 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "2.45 ms ± 270 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,7,7,torch.float16)\n",
      "forward: \n",
      "25.1 µs ± 502 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "62.2 µs ± 341 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,7,7,torch.float32)\n",
      "forward: \n",
      "25.2 µs ± 58.9 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "89.6 µs ± 35.7 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,28,28,torch.float16)\n",
      "forward: \n",
      "57.5 µs ± 2.9 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "298 µs ± 398 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,28,28,torch.float32)\n",
      "forward: \n",
      "81.7 µs ± 140 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "361 µs ± 338 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,56,56,torch.float16)\n",
      "forward: \n",
      "150 µs ± 116 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.01 ms ± 580 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,56,56,torch.float32)\n",
      "forward: \n",
      "247 µs ± 135 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "1.36 ms ± 2.29 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,112,112,torch.float16)\n",
      "forward: \n",
      "545 µs ± 703 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "3.98 ms ± 2.1 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n",
      "(N,C,H,W,dtype)=(32,256,112,112,torch.float32)\n",
      "forward: \n",
      "909 µs ± 733 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "backward: \n",
      "4.97 ms ± 1.41 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "main(\"cuda:0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After manually copying the outputs to before.txt and after.txt \n",
    "\n",
    "import numpy as np \n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "with open('before.txt', 'r') as f: \n",
    "    before = f.readlines() \n",
    "with open('after.txt', 'r') as f: \n",
    "    after = f.readlines() \n",
    "\n",
    "def parse(lines):\n",
    "    d_fwd_time = {}\n",
    "    d_back_time = {}\n",
    "    \n",
    "    for i in range(0, len(lines), 6): \n",
    "        line = lines[i] \n",
    "        \n",
    "        tensor_size = line[line.index('=')+1:].rstrip()\n",
    "        \n",
    "        key = tensor_size\n",
    "        \n",
    "        def parse_time(linex):\n",
    "            space = linex.index(' ')\n",
    "            t = float(linex[: space])\n",
    "            if linex[space + 1] == u'n': \n",
    "                t /= 1000 \n",
    "            elif linex[space + 1] == u'µ':\n",
    "                pass\n",
    "            elif linex[space + 1] == u'm': \n",
    "                t *= 1000\n",
    "            \n",
    "            return t\n",
    "            \n",
    "        fwd_time = parse_time(lines[i+2])\n",
    "        back_time = parse_time(lines[i+4])\n",
    "    \n",
    "        d_fwd_time[key] = fwd_time \n",
    "        d_back_time[key] = back_time\n",
    "        \n",
    "    return (d_fwd_time, d_back_time)   \n",
    "\n",
    "d_fwd_before, d_back_before = parse(before)\n",
    "d_fwd_after, d_back_after = parse(after)\n",
    "\n",
    "def compare(d_before, d_after, s): \n",
    "    print('compare for', s)\n",
    "    print('negative percentage is speed up, positive is slow down')\n",
    "    print()\n",
    "    print('tensor size\\t\\t\\tbefore\\tafter\\ttime cost change')\n",
    "    \n",
    "    diffs = []\n",
    "    for key in d_before: \n",
    "        t_before = d_before[key] \n",
    "        t_after = d_after[key] \n",
    "        \n",
    "        diffs.append((t_after - t_before)/t_before * 100)\n",
    "        \n",
    "        if abs((t_after - t_before) / t_before) > 0.05: \n",
    "            print(f\"{key}\", t_before, t_after, \n",
    "                  f\"{(t_after - t_before) / t_before * 100.0 : .1f}%\", sep='\\t')\n",
    "    \n",
    "    print()\n",
    "    print('(t_after - t_before) / t_before:')\n",
    "    print('mean =', np.mean(diffs), '%')\n",
    "    print('median =', np.median(diffs), '%')\n",
    "    print('max =', np.max(diffs), '%')\n",
    "    print('min =', np.min(diffs), '%')\n",
    "    \n",
    "    ax = sns.distplot(diffs, bins=40)\n",
    "    ax.set(xlabel='time cost change (negative means speed up)', ylabel='PDF')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# compare(d_fwd_before, d_fwd_after, 'forward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compare for backward\n",
      "negative percentage is speed up, positive is slow down\n",
      "\n",
      "tensor size\t\t\tbefore\tafter\ttime cost change\n",
      "(32,3,7,7,torch.float16)\t144.0\t134.0\t-6.9%\n",
      "(32,3,7,7,torch.float32)\t143.0\t133.0\t-7.0%\n",
      "(32,3,28,28,torch.float16)\t143.0\t134.0\t-6.3%\n",
      "(32,3,56,56,torch.float32)\t144.0\t136.0\t-5.6%\n",
      "(32,3,112,112,torch.float16)\t200.0\t136.0\t-32.0%\n",
      "(32,3,112,112,torch.float32)\t221.0\t144.0\t-34.8%\n",
      "(32,32,7,7,torch.float32)\t143.0\t131.0\t-8.4%\n",
      "(32,32,28,28,torch.float16)\t161.0\t137.0\t-14.9%\n",
      "(32,32,28,28,torch.float32)\t178.0\t135.0\t-24.2%\n",
      "(32,32,56,56,torch.float16)\t346.0\t195.0\t-43.6%\n",
      "(32,32,56,56,torch.float32)\t403.0\t226.0\t-43.9%\n",
      "(32,32,112,112,torch.float16)\t1310.0\t521.0\t-60.2%\n",
      "(32,32,112,112,torch.float32)\t1520.0\t644.0\t-57.6%\n",
      "(32,128,7,7,torch.float16)\t143.0\t61.9\t-56.7%\n",
      "(32,128,7,7,torch.float32)\t143.0\t64.3\t-55.0%\n",
      "(32,128,28,28,torch.float16)\t346.0\t198.0\t-42.8%\n",
      "(32,128,28,28,torch.float32)\t404.0\t226.0\t-44.1%\n",
      "(32,128,56,56,torch.float16)\t1280.0\t533.0\t-58.4%\n",
      "(32,128,56,56,torch.float32)\t1550.0\t647.0\t-58.3%\n",
      "(32,128,112,112,torch.float16)\t4830.0\t1950.0\t-59.6%\n",
      "(32,128,112,112,torch.float32)\t5690.0\t2450.0\t-56.9%\n",
      "(32,256,7,7,torch.float16)\t90.3\t62.2\t-31.1%\n",
      "(32,256,7,7,torch.float32)\t140.0\t89.6\t-36.0%\n",
      "(32,256,28,28,torch.float16)\t635.0\t298.0\t-53.1%\n",
      "(32,256,28,28,torch.float32)\t752.0\t361.0\t-52.0%\n",
      "(32,256,56,56,torch.float16)\t2470.0\t1010.0\t-59.1%\n",
      "(32,256,56,56,torch.float32)\t2930.0\t1360.0\t-53.6%\n",
      "(32,256,112,112,torch.float16)\t9510.0\t3980.0\t-58.1%\n",
      "(32,256,112,112,torch.float32)\t11300.0\t4970.0\t-56.0%\n",
      "\n",
      "(t_after - t_before) / t_before:\n",
      "mean = -36.98282279495079 %\n",
      "median = -43.781107015304286 %\n",
      "max = -0.6993006993006993 %\n",
      "min = -60.229007633587784 %\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5wddb3/8dd7Sza9b9omISEFSAIECEWacEGIBYLX0AKKyBWVYrkqF3/3/vgp+vtdsVwsoIiCItIUBKNGKdKkhSyENCBkk0A62fS6SXb38/tjZuFwMmkkJ9vez8djHzvzne+Z+cyZc87nzHznfL+KCMzMzPIVNXYAZmbWNDlBmJlZJicIMzPL5ARhZmaZnCDMzCxTSWMHsK/07NkzBg0a1NhhmJk1Ky+99NKKiCjPWtZiEsSgQYOorKxs7DDMzJoVSW/taJkvMZmZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoIwM7NMThBmZpbJCcLMzDI5QZiZWaYW80tqaxrunrzgPfMTjh3YSJGY2d7yGYSZmWVygjAzs0xOEGZmlskJwszMMjlBmJlZpoImCEljJc2WVCXp2ozlZZLuS5dPljQoLS+VdIekGZJek/SNQsZpZmbbK1iCkFQM3Ax8GBgBXChpRF61y4DVETEUuBG4IS0/FyiLiEOBo4DPNSQPMzPbPwp5BnEMUBUR8yJiK3AvMC6vzjjgjnT6fuA0SQIC6CCpBGgHbAXWFTBWMzPLU8gEUQEszJlflJZl1omIWmAt0IMkWWwElgILgB9ExKoCxmpmZnmaaiP1MUAd0A8YDHxV0oH5lSRdLqlSUmV1dfX+jtHMrEUrZIJYDAzIme+flmXWSS8ndQFWAhOAv0fEtohYDjwLjMnfQETcGhFjImJMeXl5AXbBzKz1KmSCmAIMkzRYUhvgAmBiXp2JwCXp9Hjg8YgIkstK/wIgqQNwHPB6AWM1M7M8BUsQaZvCVcDDwGvA7yNilqTrJZ2dVrsN6CGpCvh3oOFW2JuBjpJmkSSaX0fE9ELFamZm2ytob64RMQmYlFd2Xc50DcktrfmP25BVbmZm+09TbaQ2M7NG5gRhZmaZnCDMzCyTE4SZmWVygjAzs0xOEGZmlskJwszMMjlBmJlZJicIMzPL5ARhZmaZnCDMzCyTE4SZmWVygjAzs0xOEGZmlskJwszMMjlBmJlZpoImCEljJc2WVCXp2ozlZZLuS5dPljQoLb9I0is5f/WSRhcyVjMze6+CJQhJxSRDh34YGAFcKGlEXrXLgNURMRS4EbgBICLuiojRETEa+CQwPyJeKVSsZma2vUKeQRwDVEXEvIjYCtwLjMurMw64I52+HzhNkvLqXJg+1szM9qNCJogKYGHO/KK0LLNORNQCa4EeeXXOB+7J2oCkyyVVSqqsrq7eJ0GbmVmiSTdSSzoW2BQRM7OWR8StETEmIsaUl5fv5+jMzFq2QiaIxcCAnPn+aVlmHUklQBdgZc7yC9jB2YOZmRVWIRPEFGCYpMGS2pB82E/MqzMRuCSdHg88HhEBIKkIOA+3P5iZNYqSQq04ImolXQU8DBQDt0fELEnXA5URMRG4DbhTUhWwiiSJNDgZWBgR8woVo5mZ7VjBEgRAREwCJuWVXZczXQOcu4PHPgkcV8j4zMxsx5p0I7WZmTUeJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpkK2heTtXx3T16w28snHDuw0OGY2T7kMwgzM8vkBGFmZpmcIMzMLJMThJmZZSpogpA0VtJsSVWSrs1YXibpvnT5ZEmDcpYdJul5SbMkzZDUtpCxmpnZexUsQUgqBm4GPgyMAC6UNCKv2mXA6ogYCtwI3JA+tgT4HfD5iBgJnAJsK1SsZma2vUKeQRwDVEXEvIjYCtwLjMurMw64I52+HzhNkoAzgOkRMQ0gIlZGRF0BYzUzszyFTBAVwMKc+UVpWWadiKgF1gI9gOFASHpY0suSrsnagKTLJVVKqqyurt7nO2Bm1po11UbqEuBE4KL0/8clnZZfKSJujYgxETGmvLx8f8doZtaiFTJBLAYG5Mz3T8sy66TtDl2AlSRnG09HxIqI2ARMAo4sYKxmZpankAliCjBM0mBJbYALgIl5dSYCl6TT44HHIyKAh4FDJbVPE8cHgVcLGKuZmeUpWF9MEVEr6SqSD/ti4PaImCXpeqAyIiYCtwF3SqoCVpEkESJitaT/IUkyAUyKiL8WKlYzM9teQTvri4hJJJeHcsuuy5muAc7dwWN/R3Krq5mZNYKm2khtZmaNzAnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZSpogpA0VtJsSVWSrs1YXibpvnT5ZEmD0vJBkjZLeiX9u6WQcZqZ2fYKNmCQpGLgZuBDJGNMT5E0MSJyhw69DFgdEUMlXQDcAJyfLpsbEaMLFZ+Zme1cIc8gjgGqImJeRGwF7gXG5dUZB9yRTt8PnCZJBYzJzMx2UyETRAWwMGd+UVqWWSciaoG1QI902WBJUyU9JemkAsZpZmYZCjom9V5YCgyMiJWSjgIekjQyItblVpJ0OXA5wMCBAxshTDOzlquQZxCLgQE58/3Tssw6kkqALsDKiNgSESsBIuIlYC4wPH8DEXFrRIyJiDHl5eUF2AUzs9arkAliCjBM0mBJbYALgIl5dSYCl6TT44HHIyIklaeN3Eg6EBgGzCtgrGZmlqdgl5giolbSVcDDQDFwe0TMknQ9UBkRE4HbgDslVQGrSJIIwMnA9ZK2AfXA5yNiVaFiNTOz7RW0DSIiJgGT8squy5muAc7NeNwDwAOFjM3MzHbOv6Q2M7NMThBmZpbJCcLMzDLtNEFIeiRn+huFD8fMzJqKXZ1B5P64YLvGZDMza7l2lSBiv0RhZmZNzq5ucz1Q0kRAOdPviIizCxaZmZk1ql0liNzeV39QyEDMzKxp2WmCiIinGqYlladl1YUOyszMGt+u7mKSpP8jaQUwG3hDUrWk63b2ODMza/521Uj9FeBE4OiI6B4R3YBjgRMkfaXg0ZmZWaPZVYL4JHBhRMxvKIiIecDFwKcKGZiZmTWuXSWI0ohYkV+YtkOUFiYkMzNrCnaVILa+z2VmZtbM7eo218MlrSP5HQS8+8M5AW0LFpW1CndPXvCe+QnH7pthY/PXW4ht2Lve7/NdqONv+86ubnMt3l+BmJlZ07Kr21zbSvqypJskXZ6OG73bJI2VNFtSlaRrM5aXSbovXT5Z0qC85QMlbZD0tT3ZrpmZ7b1dtUHcAYwBZgAfAX64uytOx5S+GfgwMAK4UNKIvGqXAasjYihwI3BD3vL/Af62u9s0M7N9Z1dnBCMi4lAASbcBL+7Buo8BqtLbYpF0L0nXHa/m1BkHfDOdvh+4SZIiIiSdA8wHNu7BNs3MbB/Z1RnEtoaJiKjdw3VXAAtz5helZZl10vWvBXpI6gj8B/CtnW0gvexVKamyuto9gJiZ7Uu7ShCHS1qX/q0HDmuYTu9uKpRvAjdGxIadVYqIWyNiTESMKS8v31lVMzPbQ4W8i2kxMCBnvn9allVnUdoA3gVYSdKdx3hJ3wO6AvWSaiLipr2Ix8zM9sAe3ZW0h6YAwyQNJkkEFwAT8upMBC4BngfGA49HRAAnNVSQ9E1gg5ODmdn+VbAEERG1kq4CHgaKgdsjYpak64HKiJgI3AbcKakKWEWSRMzMrAko5BkEETEJmJRXdl3OdA27GOs6Ir5ZkODMzGyndtVIbWZmrZQThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpkKmiAkjZU0W1KVpGszlpdJui9dPlnSoLT8GEmvpH/TJH28kHGamdn2CpYgJBUDNwMfBkYAF0oakVftMmB1RAwFbgRuSMtnAmMiYjQwFviFpIKOfmdmZu9VyDOIY4CqiJgXEVuBe4FxeXXGAXek0/cDp0lSRGyKiNq0vC0QBYzTzMwyFDJBVAALc+YXpWWZddKEsBboASDpWEmzgBnA53MSxjskXS6pUlJldXV1AXbBzKz1arKN1BExOSJGAkcD35DUNqPOrRExJiLGlJeX7/8gzcxasEImiMXAgJz5/mlZZp20jaELsDK3QkS8BmwARhUsUjMz204hE8QUYJikwZLaABcAE/PqTAQuSafHA49HRKSPKQGQdABwMPBmAWM1M7M8BbszKCJqJV0FPAwUA7dHxCxJ1wOVETERuA24U1IVsIokiQCcCFwraRtQD1wRESsKFauZmW2voLeORsQkYFJe2XU50zXAuRmPuxO4s5CxmZnZzjXZRmozM2tcThBmZpbJCcLMzDI5QZiZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoIwM7NMThBmZpbJCcLMzDI5QZiZWSYnCDMzy+QEYWZmmZwgzMwsU0EThKSxkmZLqpJ0bcbyMkn3pcsnSxqUln9I0kuSZqT//6WQcZqZ2fYKNmCQpGLgZuBDwCJgiqSJEfFqTrXLgNURMVTSBcANwPnACuCsiFgiaRTJqHQVhYrV9o+7Jy/Y7eUTjh1Y6HD22K7i35n3uz/529xXz8verHdPnodCxW/7RyHPII4BqiJiXkRsBe4FxuXVGQfckU7fD5wmSRExNSKWpOWzgHaSygoYq5mZ5SlkgqgAFubML2L7s4B36kRELbAW6JFX5xPAyxGxJX8Dki6XVCmpsrq6ep8FbmZmTbyRWtJIkstOn8taHhG3RsSYiBhTXl6+f4MzM2vhCtYGASwGBuTM90/LsuosklQCdAFWAkjqDzwIfCoi5hYwTttPauvqWbauhiVralhXs41NW2vZtLUOAW1KiikrKaJT2xLKO5WxYOUmKrq1o7hIjR22WatVyAQxBRgmaTBJIrgAmJBXZyJwCfA8MB54PCJCUlfgr8C1EfFsAWO0Alu5YQszF69l1tJ1LF1TQ13EO8valRbTrk0xAFtr69lSW8e2umT5b59/i85tSzhmcA+OH9KDk4eXM7RXx0bZB7PWqmAJIiJqJV1FcgdSMXB7RMySdD1QGRETgduAOyVVAatIkgjAVcBQ4DpJ16VlZ0TE8kLFa/tObX090xeu5bl5K1iypgaA/t3aceKwnvTr2o6Kru3o2r6UIm1/drBpSy3VG7ZwYHkHpi5Yw3NzV/LYa28DcHCfTnz00L6cdXg/BvXssF/3yaw1KuQZBBExCZiUV3ZdznQNcG7G474DfKeQsdm+t62unhfmreTZqhWsq6mlV6cyPjKqD6MqutC1fZvdWkf7shIOKCvh/KMHcv7RyS2Ri9ds5tFZy/jL9KX88NE3+OGjb3Di0J5cdOxATh/Rm9LiJt2UZtZsFTRBWOsQEcxcso6/zVzKmk3bOLBnBz5+RH+G9+6IMs4S9lRF13Z8+oTBfPqEwSxZs5kHXlrEPS8u4At3vUzvzmVcesJgJhw7kM5tS/fB3phZAycI2yvV67fw0CuLmb9iI306t+XfTuzPgeWFayvo17UdV582jCtOHcoTry/n18/N57t/e52bHq9iwrED+beTBhds22atjROEvS/19cEdz7/JTx+fQ2lxEeNG9+PoQd0z2xUKobhInD6iN6eP6M3MxWv5xdPz+NU/53HHc28y5oBunDS83GcUZnvJCcL22LK1NXz1D6/wbNVKDurdiY8fWdGoH8ajKrrw0wuP4N8/NJybHq/iwamLmDx/FccdmNz91LHML3Oz98PvHNsjz89dydX3vMymrXX8978eSn197JN2hn1hcM8O/PC8wxnUoz1PzF7Os1UrePHNVZwwpAcnDSunbWlxY4do1qz49g/bLRHBrU/P5eLbJtO5XSl/uvIELjxmYJNJDrl6dCxj/FED+NJpwxjeuxNPzK7m+w/P5p9zqqnZVtfY4Zk1Gz6DsF2q2VbHtQ9M56FXlvDhUX343vjD6NQMru/36tyWCccMZPGazTwyaxl/m7mMVxY+yZdPH8YnjuxPiW+PNdspJwjbqVUbt3L5byupfGs1Xz/zIK44ZUiTPGvYmYqu7bj0hMHMrd7AS2+t5j8emMGtT8/j62cexJkj+zS7/THbX/wVynZoXvUGPv6zZ5m+eC03TTiCK08d2qw/TIeUd+TBK47nlouPQhKf/93LnHPzszwzZ0Vjh2bWJPkMwjJNW7iGS38zBQH3fPY4jjqgW2OHtE9IYuyoPpx+SC/+OHUxP35sDhffNpnjh/Tgq2cc1GL202xf8BmEbefpN6q58Jcv0KGsmPu/cHyL/NAsKS7ivDEDePxrH+T/nDWC2cvW84mfP8elv36RGYvWNnZ4Zk2CE4S9x5+nLeGyO6ZwQI8OPPD54xncwjvFKysp5tITBvP0NadyzdiDeHnBGs666Rk++9tKZi52orDWzZeY7B13T17Afz40g6MHdedXl4xpVb9E7lBWwhWnDOXi4w7g18+8yW3PzONjr77NaQf34urThjF6QNfGDtFsv3OCMAB++fQ8/u+k1zj1oHJ+fvFRrfZHZZ3blvKl04dx6YmDuOPZN/nVM/M55+ZnOX5IDw7u05kh5R2adUO92Z5wgmjlIoIbH5vDT/4xh48e1pcbzxtNmxJfeezctpSrTxvGpScO5p7JC/jlP+fx3NyV9OvalhOHlnNoRRePdmctnj8JWrGI4P/+9TV+8o85nDemPz+54Agnhzwdy0r47MkH8s//OJWPH1HBttrg95UL+cEjs3nqjWo2bqlt7BDNCqagnwaSxkqaLalK0rUZy8sk3ZcunyxpUFreQ9ITkjZIuqmQMbZW9fXBfz40k189M59PHz+I7/7rYf5GvBNlJcUcPag7Xzp9GJd84AB6dGzDw7OWccPfX+cPlQtZsGoTkTOcqllLULBLTJKKgZuBDwGLgCmSJkbEqznVLgNWR8RQSRcANwDnAzXA/wZGpX+2D9XW1XPN/dP549TFXHHKEL5+5kG+rr6biiQO6tOZg/p05u11NbwwbyVTF65h6sI19OpUxpgDujF6YLcW2YNszbY6lqzZzFsrN7JxSx2bt9VSVw/1EdRH0Ka4iLLSYspKiujSrpRu7dv4jLSZK+Sr+BigKiLmAUi6FxgH5CaIccA30+n7gZskKSI2As9IGlrA+FqlLbV1XH33VB559W2+fuZBXHmqn+L3q3fntowbXcGZI/swY9FaKt9axaSZy/j7rGUM69WJwwd04ZC+nSkraV4N/qs3buXVpeuYvWw9c5avp2r5Bhas2sTb67bs8bo6lJXQp3MZ/bu1p6JrO1Zs2ELPjmUFiNoKoZAJogJYmDO/CDh2R3UiolbSWqAHsFt9H0i6HLgcYODAgXsbb4u3aWstn7vzJf45ZwXfOnsklxw/qLFDahHalhZz9ODuHD24O8vW1fDKgjVMW7SG2ZXrKSkSw3p1pKRYnH5Ib7p32L2xufeXmm11LF6zmZ89WcX0hWuZsXgti9dsfmd5t/alDOvdiZOHlTOge/IhP2PxWjq0KaFdm2JKikRRkRDJmOQ1tfXUbK1j7eZtrN60lVUbt7J0bQ3PzFlBXQR3v7iAkf06c/LwcqI+GNC9vc9em7BmfR4cEbcCtwKMGTPGF4B3Yu2mbXzmjilMXbCa748/jHPHDGjskFqkPp3bMnZUH84Y2Zu3Vm5i5pK1vLZkHdfcPx0JRvXrwknDenLi0J6MHtiV9m3231uwZlsdry9bz4xFa3hw6mIWrt7MivVbaHjjDOrRniMGduVTHziAkf26cFCfTvTs2Ga7D/AttfV7vO1tdfUsW1tDx7YlPPVGNb98eh619UHX9qUcVtGV0QP9O5OmqJCvzsVA7qdQ/7Qsq84iSSVAF2BlAWNqlZatreGS219k/oqN3DzhSD58aN/GDqnFK5IY3LMDg3t24GOH9uXwAV35x2vLeaaqmlufnsfPnpxLcZE4pG8njhjQjUP6dubgvp0Y3rvTXrdfbK2tZ8mazcyt3sDst9cz5+0NvLZ0HXOWb6CuPkkHHcpKGNCtHYf378KAbu350unD6Nq+cGc3pcVFDOjengnHDuTKU4eyvmYb1//5VaYtWsMzVdU8PaeaZ6tWcNGxA/nIoX1b7e9wmppCJogpwDBJg0kSwQXAhLw6E4FLgOeB8cDj4VtB9qn5KzZy8a8ms2bTVn5z6dEcP7RnY4fU6khiVEUXRlV04UunD2N9zTYq31zNywtW89Jbq/njy4vYuPXdgYx6dGhD/27t6Ne1HSs3bqV9aTHt2hRTXCTqIiiW2FZXz+ZtdWzaWseaTVtZsWELK9ZvZfGazSxdu5n6nHdR3y5tObhPJz40ojcj+3VhVEVnnppd/Z4zg0Imhyyd2pZyxMBuHDGwGxu21DJ1wWpeX7aef//9NL7z19f41AcO4FMfGNTkLsm1NgVLEGmbwlXAw0AxcHtEzJJ0PVAZEROB24A7JVUBq0iSCACS3gQ6A20knQOckXcHlO3CywtW89k7KgngnsuP47D+Po1vCjq1LeXUg3tx6sG9gOSW40WrN/P6suRb/qLVm1m0ehOz317PsrU11Gyre+cD/y/Tl263vi7tSunZsQ09OpZx9KBuDOxewYDu7TmwvAPDenfK7DKlKV3371hWwknDyvnZRUfy3NyV3P7MfH702BxueWou540ZwOc+OISKru0aO8xWqaAXQCNiEjApr+y6nOka4NwdPHZQIWNr6f4+cxlfuncqvTu35TeXHs2B5R0bOyTbgaIiMbBHewb2aM8ZI9+77O7JC4gIttbWU1cfjDuigrr6oE1JEe3SW0qLWsjvVyRxwtCenDC0J1XL1/PLp+dzz4sLuOfFBYw/agBXnDKEAd3bN3aYrUqzbqS27UUEtz/7Jt/566sc3r8rt10yhh6+rbBZk0RZek2+vFPrOJZDe3XihvGH8cXTh3HLk3O5b8pC/lC5kPOPHsDV/zKMPl3aNnaIrYITRAuypbaO6x6axX2VCzlzZG9+dP4RtGvjxj5rviq6tuPb54ziylOHcvMTVdw7ZQF/eGkRnzzuAK44ZYi//BSYf+bYQlSv38KEX07mvsqFXHXqUH5+0VFODtZi9OnSlm+fM4rHv3oKZx/ej18/O5+Tv/cEP3rsDTa4P6yCcYJoASrfXMVZP32GV5es46YJR/C1Mw9qMdelzXIN6N6eH5x7OI985WROHl7Ojx6bw8nfe4LbnplPzba6Xa/A9ogTRDNWXx/c8tRczr/1BcpKi7j/Cx/gY4f1a+ywzApuaK9O/Pzio3joyhM4pG8nvv2XVznth0/x+8qF1Nbt+Q/5LJsTRDO1fH0Nl90xhe/+7XXGjuzDn68+kZH9ujR2WGb71egBXbnr347jd5cdS4+Obbjm/umc8aOn+fO0JdTX+ydVe8sJohn687QlnHHj0zw7dyXXjxvJTROOaFXDg5rlO3FYT/505QnccvFRlBYVcfU9U/nIT/7J32cudaLYC76LqRlZvr6Gb018lb/OWMrh/bvww/MOZ2ivTo0dllmTIImxo/rwoRG9+cv0Jfz4sTl8/ncvc3CfTnzxtGGMHdnHbXN7yAmiGaitq+e3z7/FjY++wZbaer5+5kF87uQDKSn2CaBZvuIiMW50BR87rB9/nraEnzw+hyvuepkh5R343MlDOOeICo9TsZucIJqwiOCpN6r57t9e5/Vl6zl5eDnfOnskg3t2aOzQzJq84iJxzhEVnHV4P/46Yym3PDmXax6Yzv88+gafOv4ALjx6IN3c19NOOUE0UZVvruJ7D8/mxfmrGNC9HbdcfBRnjuzdpPrQMWsOiovE2Yf346zD+vLPOSv4xdNz+d7fZ/Pjx+bw8SMquPi4AxhV4Rs8sjhBNCH19cETs5dz69PzmDx/FeWdyvj2uJGcf/RAnxKb7SVJnDy8nJOHlzN72Xp+89ybPDh1EfdOWciois6cP2YAZx9eQZf2vuGjgRNEE7Bq41YemrqYuya/xdzqjfTr0pb/+ughTDh24H4dUMastTioTyf++18P5dqxB/OnaYu558WF/O8/zeL6v7zKB4f3Ytzofpx2SK9W//5r3XvfiDZvreOpN6qZOG0xj776NtvqgsP6d+HHF4zmI4f2pdQN0GYF16V9KZ/6wCA+edwBzFqyjoemLubP05fw2Gtv06akiBOH9uRDI3pz6kG9WmUHgU4Q+9HCVZt4bu4Knni9miffWE7Ntnq6d2jDJ48bxLlj+nNI386NHaJZq5Q7qNM3PnIIL85fxSOvLuPRV9/m8deXAzC0V0dOHNqT4w7szpEDu9Grc8tPGE4QBbK1tp7Zy9bzyqI1TFu4hhfnr2LBqk1AMm7xeWMGMHZkH44Z3N23q5o1IcVF4gNDevCBIT247mMjmP32ev75xgqeqVrBfVMW8pvn3gRgQPd2HN6/KyP7dWFkv2TI2PKOZS3qRpKCJghJY4Efk4wo96uI+G7e8jLgt8BRJGNRnx8Rb6bLvgFcBtQBX4yIhwsZ6/uxtbae5etrWLx6M2+t2sTCVZuYW72BN97ewPwVG98Z/7dnxzYcMbAbnzlhECcM7cnQXh1b1IvIrKWSxMF9OnNwn8589uQD2VJbx6wl63j5rWS42KkL1rxnlL/ObUsY0qsjg3t2YEC39vTv1o6Kbu3o07ktvTu3pcNejje+vxUsWknFwM3Ah4BFwBRJE/OGDb0MWB0RQyVdANwAnC9pBMnwoyOBfsBjkoZHxD7vrnHz1jrmr9jIlto6arbVs6W2js1bk7F+N22rY33NNtbX1LJu8zbWbN7G6o1bWbVxK9Xrt7By49b3rKtISW+Tw3t3YuzIPhzctxOjB3Sloms7JwSzFqCspJgjB3bjyIHd+LeTkrK1m7Yxa+laZi9bz7zqjcyt3sDzc1fy4LrFRF4vHx3LSujeoQ3dOrShe/tSurRL/jq3K6VDWQkd2hTTrk0J7UqLademiLYlxZSVFtGmuJg2JUWUFovS4iJKikVJURHFRaK4SJSVFNG2dN9371/IdGpxavMAAA1eSURBVHYMUBUR8wAk3QuMA3ITxDjgm+n0/cBNSj5JxwH3RsQWYH46ZvUxwPP7OsjZb6/nnJuf3Wmd0mLRqW0pXduX0r19G/p3a8+RB3Sjd6e29OlSRt8u7TigR3v6dW3nxmWzVqZL+1KOH9KT44f0fE/5lto6lq6pYfGazby9robl67fw9roaVm/cysqNW6nesIW51RtZu3kb62u2sTddRn30sL7cPOHIvdyT7RUyQVQAC3PmFwHH7qhORNRKWgv0SMtfyHtsRf4GJF0OXJ7ObpA0e9+Evl/1BFY0dhB7aZ/vw0X7cmW7t96CHod9tT+7WM/73odCPd97sp2cZX5P7KGfAT97/wfxgB0taF4XxPJExK3ArY0dx96QVBkRYxo7jr3hfWgaWsI+QMvYj5awD1DY7r4XAwNy5vunZZl1JJUAXUgaq3fnsWZmVkCFTBBTgGGSBktqQ9LoPDGvzkTgknR6PPB4RERafoGkMkmDgWHAiwWM1czM8hTsElPapnAV8DDJba63R8QsSdcDlRExEbgNuDNthF5FkkRI6/2epEG7FriyEHcwNRHN+hJZyvvQNLSEfYCWsR8tYR9Q5N+HZWZmhoccNTOzHXCCMDOzTE4QjUDSaEkvSHpFUqWkY9JySfqJpCpJ0yXt+1++7GOSrpb0uqRZkr6XU/6NdD9mSzqzMWPcHZK+Kikk9Uznm82xkPT99BhMl/SgpK45y5rNcZA0No2zStK1jR3P7pA0QNITkl5N3wNfSsu7S3pU0pz0f7fGjvV9iQj/7ec/4BHgw+n0R4Anc6b/Bgg4Dpjc2LHuYj9OBR4DytL5Xun/EcA0oAwYDMwFihs73p3sxwCSmyneAno2t2MBnAGUpNM3ADc0t+NAciPLXOBAoE0a94jGjms34u4LHJlOdwLeSJ/37wHXpuXXNhyT5vbnM4jGEUBD395dgCXp9Djgt5F4AegqqW9jBLibvgB8N5IuUYiI5Wn5O12lRMR8oKGrlKbqRuAakuPSoNkci4h4JCJq09kXSH43BM3rOLzTNU9EbAUauuZp0iJiaUS8nE6vB14j6fVhHHBHWu0O4JzGiXDvOEE0ji8D35e0EPgB8I20PKt7ku26GGlChgMnSZos6SlJR6flzWY/JI0DFkfEtLxFzWYf8nyG5MwHmtc+NKdYM0kaBBwBTAZ6R0RDN6/LgN6NFNZeadZdbTRlkh4D+mQs+k/gNOArEfGApPNIfg9y+v6Mb3ftYj9KgO4kl2COBn4v6cD9GN5u2cU+/C+SSzRN2s72ISL+lNb5T5LfDd21P2MzkNQReAD4ckSsy+29OSJCUrP8PYETRIFExA4/8CX9FvhSOvsH4FfpdJPrYmQX+/EF4I+RXGh9UVI9SSdlTWo/drQPkg4luTY/LX1D9wdeTm8aaBb70EDSp4GPAaelxwOa2D7sQnOK9T0klZIkh7si4o9p8duS+kbE0vTS5PIdr6Hp8iWmxrEE+GA6/S/AnHR6IvCp9A6a44C1OaepTdFDJA3VSBpO0ri4gmbSVUpEzIiIXhExKCIGkVzWODIiltGMjoWSgbmuAc6OiE05i5rFcUjtTtc8TU46PMFtwGsR8T85i3K7EboE+NP+jm1f8BlE4/gs8OO0g8Ia3u2yfBLJ3TNVwCbg0sYJb7fdDtwuaSawFbgk/fbaErpKaU7H4iaSO5UeTc+EXoiIz0cz6rImdtA1TyOHtTtOAD4JzJD0Slr2v4DvklxyvYzk7rjzGim+veKuNszMLJMvMZmZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoJo4iR1lXRFznw/Sfc3Zkw7IunTkvrtQf1TJP2lkDHtCUk/knTyftzelyW1z5mflNsTq70/kjbso/V8TMkImK2WE0TT1xV4J0FExJKIGN+I8ezMp4HdThBNiaQewHER8fR+3OyXgXcSRER8JCLW7Mft2879FTgrN4m3Nk4QTd93gSFKxo74vqRB6Q/TGr6xP5T2N/+mpKsk/bukqUrGm+ie1hsi6e+SXpL0T0kH529EUkdJv5Y0Ix1X4BNp+YVp2UxJN6RlxZJ+k5bNkPQVSeOBMcBdaazt8tY/VNJjkqZJelnSkHRRR0n3KxnP4K70l6lIuk7SlHQbt+aUPynpBkkvSnpD0klpeXtJv1fSL/+DSjoQHJMuO0PS8+l2/5D2m5PvE8Dfc+J9U9K30sfMaHjOJHWQdHu6/alKOvvb1fZ/rmTcj1mSvpWWfZEkmT4h6YmcbfaU9F1JV+bE8k1JX0unv54+L9Mb1pVxLDekr5VZ6XN+TPq8zZN0ds4x/H7Ouj6X8zr4R85+N+zfIEmvSfplut5HGo6xpC+m+z1d0r0Z8YxMn69X0jrD0vU1HPPX0tdA+7T+UUo6f3xJ0sNKe9HVDl7HSn59/Xwa73d28Jy8875J578m6Zs5r6kfp/HNVDo+S/qjzydJujBpnRq7v3H/7fwPGATMzJon+cZeRdIPfTmwFvh8uuxGko7DAP4BDEunjwUez9jODcCPcua7kXyALUjXXQI8TtJt8VHAozl1u6b/nwTG7GA/JgMfT6fbknxzPiWNuT/Jl5XngRPTOt1zHnsncFbONn6YTn8EeCyd/hrwi3R6FMkvh8eQ9A31NNAhXfYfwHUZ8d3RsI10/k3g6nT6CuBX6fT/Ay5u2G+S/v877Gj7uftC8gvhJ4HDcrbRM2+bPUl6BH0qp/xVkn6KzgBuJRmjogj4C3Byxr4E74438iDJ+COlwOHAK2n55cB/pdNlQCVJv1QlQOe0vCfJ60skr7taYHS67Pc5z8MS3h0TpGtGPD8FLkqn2wDt0vUFcEJafnv6HJYCzwHlafn5JL+qhh28jkm7RUmnrwQ27Mb76GvAN3NeU79Mp0/Oq3cR8NPG/hxorD93tdH8PRFJP/TrJa0F/pyWzwAOS78tHw/8Qe/2MFmWsZ7TSfq/ASAiViu5Hv9kRFQDSLqL5A30beBAST8lOQ1/ZGcBSuoEVETEg+m6a9JygBcjYlE6/wrJG/kZ4FRJ15Akku7ArJx9a+gQ7aW0PsCJwI/T9c+UND0tP45kAJdn0+21IUlE+foC1Xlludv513T6DODshm/0JMlu4E62D3CepMtJPnz7pvHkLn+PiJgqqZeS9pxyYHVELFQyWtkZwNS0akeS/pXyL4tt5d2zoRnAlojYJmkG7z5fZ5C8PhouV3ZJ17UI+H/psa8n6XK7oavq+RHR0J1E7nM/neTM8SGS/rnyPQ/8p6T+JJ07zkmPxcKIeDat8zvgi2nco3i325BiYOkuXscnkJwBQvJl4oaMGHblHoCIeFpSZ0ldI7nct5xmetl0X3CCaP625EzX58zXkxzfImBNRIzeVxtMk8fhwJnA50n6mfnM+1xdbvx1QImktsDPSL6BL0wvBbTNeEwdu34Ni+Rs58Jd1Nuct40dbUfAJyJi9ns2ktO9c175YJJvq0enz9tvMraT5Q/AeJIuvu/L2fZ/R8QvdvHYbZF+/SXnNRER9Ur6/2pY19UR8XBevJ8mSUpHpUnlzZx4849Vw2XEj5J8cTiLJBEcGu8OYERE3C1pclpvUno5ax7vHaCJdF7ArIj4QF5cndn563hXfQbV8t5L6vnHICuWhnqbd7HuFsttEE3fepJLSO9LRKwD5ks6F94Za/nwjKqPkpyek9brRtLz5wfT6+LFwIXAU0rGbS6KiAeA/wIaxmvOjDU9w1kk6Zx03WXaecNfw5t3RfrNcXca5Z8l7RBN0gjg0LT8BeAESUPTZR2U9Dyb7zVg6G5s52HgaumdNpEjdrH9zsBGYK2k3sCHc9a1s2N7H8kZ3XiSZNGw7c+kzwmSKiT12o2Yd7QfX1DSVTWShkvqQHImsTxNDqcCB+xsJZKKgAER8QTJ5bsuJGc2uXUOBOZFxE9IejU9LF00UFJDIphAcuY4GyhvKJdUKmnkLl7Hz/Lu2e9FOwj1baCXpB6Syti+XeH8dL0nkvTcuzYtHw7MpJVygmjiImIlyeWRmZK+/z5XcxFwmaRpJJdqsoZy/A7QLd3ONODUSLq3vhZ4gmSM4JciGZymAngyvST0O94dEe83wC3KaKQm6fHyi+mll+fIHvwGgPTU/pckb8yHSbqC3pWfkXywvJruyyySN3o1SVvNPem2nwe2a6QnuVR2ym5s59sk18mnS5qVzu9s+9NILgm9DtxN8mHW4Fbg70obqXNF0pNpJ5LR7pamZY+k63g+vVx0P+//y8OvSNo2Xk4bb39BcpZ0FzAmXf+n0rh3phj4XVp/KvCT2P5OrPOAmenrZRTw27R8NnClpNdI2rx+Hslwo+OBG9LX4Sskl5Zgx6/jL6XrmcEORqGLiG3A9SRfeh7N2K8aSVOBW4DLcspPJXlttEruzdVahPQMpzQiapTcIfUYcFD6gbO763gG+FjGB9x+2X5romR4zr9ExKhGDgVJTwJfi4jKvPLewN0RcVqjBNYEuA3CWor2JLeMlpJcx77ifXw4f5Wkwfn9/BZhX2zfmpaBJK+JVstnEGZmlsltEGZmlskJwszMMjlBmJlZJicIMzPL5ARhZmaZ/j8JrmkraBe8KwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "compare(d_back_before, d_back_after, 'backward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit",
   "language": "python",
   "name": "python37564bit60e57c818ef44aef9efb750a08331318"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}