makemore_mlp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:44.924711Z",
     "start_time": "2023-10-28T11:38:43.116700Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "['emma',\n 'olivia',\n 'ava',\n 'isabella',\n 'sophia',\n 'charlotte',\n 'mia',\n 'amelia',\n 'harper',\n 'evelyn']"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = open(\"names.txt\", \"r\").read().splitlines()\n",
    "words[:10]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:44.937469Z",
     "start_time": "2023-10-28T11:38:44.926590Z"
    }
   },
   "id": "ecde69c10d549bec"
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "32033"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:44.938675Z",
     "start_time": "2023-10-28T11:38:44.935067Z"
    }
   },
   "id": "a34d0c78ff880cfb"
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "chars = sorted(list(set(\"\".join(words))))\n",
    "stoi = {c: i for i, c in enumerate(chars)}\n",
    "stoi['.'] = 0\n",
    "itos = {i: c for c, i in stoi.items()}\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:44.946229Z",
     "start_time": "2023-10-28T11:38:44.943150Z"
    }
   },
   "id": "855141c579e65102"
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "# build the dataset\n",
    "\n",
    "def build_dataset(words, block_size=3):\n",
    "    X, Y = [], []\n",
    "    for w in words:\n",
    "        context = [0] * block_size\n",
    "        for ch in w + '.':\n",
    "            ix = stoi[ch]\n",
    "            X.append(context)\n",
    "            Y.append(ix)\n",
    "            context = context[1:] + [ix]\n",
    "    return torch.tensor(X), torch.tensor(Y)\n",
    "\n",
    "\n",
    "import random\n",
    "\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(len(words) * 0.8)\n",
    "n2 = int(len(words) * 0.9)\n",
    "\n",
    "Xtrain, Ytrain = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xtest, Ytest = build_dataset(words[n2:])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T12:24:02.720256Z",
     "start_time": "2023-10-28T12:24:02.384454Z"
    }
   },
   "id": "d930492df1c455af"
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "(torch.Size([182625, 3]), torch.Size([182625]))"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtrain.shape, Ytrain.shape  # 80% of the data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:45.391079Z",
     "start_time": "2023-10-28T11:38:45.384372Z"
    }
   },
   "id": "ad82d25c2a3f9908"
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "outputs": [],
   "source": [
    "n_embd = 10\n",
    "n_hidden = 200\n",
    "vocab_size = 27\n",
    "block_size = 3\n",
    "\n",
    "g = torch.Generator().manual_seed(42)\n",
    "C = torch.randn([vocab_size, n_embd], generator=g)\n",
    "W1 = torch.randn([n_embd * block_size, n_hidden], generator=g) * (5/3) / (n_embd * block_size) ** 0.5\n",
    "# b1 = torch.randn([n_hidden], generator=g) * 0.01\n",
    "W2 = torch.randn([n_hidden, vocab_size], generator=g) * 0.01\n",
    "b2 = torch.randn([vocab_size], generator=g) * 0\n",
    "\n",
    "\n",
    "bngain = torch.ones((1, n_hidden))\n",
    "bnbias = torch.zeros((1, n_hidden))\n",
    "bnmean_running = torch.zeros((1, n_hidden))\n",
    "bnstd_running = torch.ones((1, n_hidden))\n",
    "\n",
    "params = [C, W1, W2, b2, bngain, bnbias]\n",
    "for p in params:\n",
    "    p.requires_grad = True"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-29T13:09:00.134017Z",
     "start_time": "2023-10-29T13:09:00.127082Z"
    }
   },
   "id": "efeccbc1c68e0fc7"
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-0.0153) tensor(0.9988)\n",
      "tensor(-0.0086) tensor(3.0655)\n"
     ]
    },
    {
     "data": {
      "text/plain": "(array([1.39114465e-05, 0.00000000e+00, 2.08671697e-05, 1.39114465e-05,\n        2.08671697e-05, 5.56457858e-05, 1.04335848e-04, 9.04244020e-05,\n        2.64317483e-04, 3.13007545e-04, 8.13819618e-04, 1.05726993e-03,\n        2.10062841e-03, 3.58219746e-03, 5.75238311e-03, 9.22328900e-03,\n        1.41827197e-02, 2.13123360e-02, 3.13564003e-02, 4.49061492e-02,\n        6.24206602e-02, 8.27313721e-02, 1.04314981e-01, 1.20848735e-01,\n        1.34412396e-01, 1.39316181e-01, 1.32012671e-01, 1.18762018e-01,\n        9.98494069e-02, 7.80849490e-02, 6.03687219e-02, 4.19221439e-02,\n        2.89636315e-02, 1.91560618e-02, 1.27915750e-02, 7.93648020e-03,\n        5.06376651e-03, 2.79620074e-03, 1.74588653e-03, 8.97288296e-04,\n        6.60793707e-04, 3.19963268e-04, 2.64317483e-04, 1.53025911e-04,\n        8.34686787e-05, 2.08671697e-05, 3.47786161e-05, 1.39114465e-05,\n        6.95572323e-06, 6.95572323e-06]),\n array([-18.26063919, -17.54180668, -16.82297417, -16.10414165,\n        -15.38530914, -14.66647663, -13.94764412, -13.22881161,\n        -12.5099791 , -11.79114658, -11.07231407, -10.35348156,\n         -9.63464905,  -8.91581654,  -8.19698402,  -7.47815151,\n         -6.759319  ,  -6.04048649,  -5.32165398,  -4.60282146,\n         -3.88398895,  -3.16515644,  -2.44632393,  -1.72749142,\n         -1.00865891,  -0.28982639,   0.42900612,   1.14783863,\n          1.86667114,   2.58550365,   3.30433617,   4.02316868,\n          4.74200119,   5.4608337 ,   6.17966621,   6.89849873,\n          7.61733124,   8.33616375,   9.05499626,   9.77382877,\n         10.49266129,  11.2114938 ,  11.93032631,  12.64915882,\n         13.36799133,  14.08682384,  14.80565636,  15.52448887,\n         16.24332138,  16.96215389,  17.6809864 ]),\n <BarContainer object of 50 artists>)"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 2000x500 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABkoAAAGsCAYAAACSD/sZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMAUlEQVR4nO3df3iX9X0v/idgSfxF1FISoNSgUi1TiQVJsbbaY2awrC1b50FPT6G5PPaqFatLq4ITqNMuqOhSlcl0xx/t6mQ9W92mHjablXa9TLWFcjq1ctQjxV8J4o5E8TRY8vn+0a/pUoLyiQkfyefxuK77ks/9ed/vvO77usX77TPv+z2iUCgUAgAAAAAAUIZGlroAAAAAAACAUhGUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULYEJQAAAAAAQNkSlAAAAAAAAGVLUAIAAAAAAJSt/UpdwGDo6enJ888/n4MPPjgjRowodTkAADDkCoVCXnnllUyYMCEjR/r9J96acRMAAOWkmDHTsAhKnn/++UyaNKnUZQAAwF73zDPP5L3vfW+py2AfYNwEAEA52pMx07AISg4++OAkvz7hMWPGlLgaAAAYel1dXZk0aVLvszC8FeMmAADKSTFjpmERlLwxbXzMmDEe+AEAKCteocSeMm4CAKAc7cmYycuMAQAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgBAAAAAADKlqAEAABgkK1cuTK1tbWprKxMfX19Hn744d22ffTRR/PpT386tbW1GTFiRFpbW9+07+XLl2fEiBG56KKLBrdoAAAoU4ISAACAQbR69eo0Nzdn2bJlWb9+faZNm5bGxsZs2bKl3/avvfZajjjiiCxfvjw1NTVv2vePf/zj/MVf/EWOP/74oSgdAADKkqAEAABgEF1//fU599xz09TUlKlTp2bVqlU54IADctttt/Xb/sQTT8y1116bs846KxUVFbvt99VXX81nPvOZ3HrrrTn00EOHqnwAACg7ghIAAIBBsmPHjqxbty4NDQ29+0aOHJmGhoa0t7e/rb7PP//8zJkzp0/fb6a7uztdXV19NgAAYFeCEgAAgEGydevW7Ny5M9XV1X32V1dXp6OjY8D93n333Vm/fn1aWlr2+JiWlpZUVVX1bpMmTRrwzwcAgOFMUAIAAPAO9swzz+TCCy/Mt771rVRWVu7xcYsXL862bdt6t2eeeWYIqwQAgH3XfqUuAIChU7vovqKP2bR8zhBUAgDlYezYsRk1alQ6Ozv77O/s7HzLhdp3Z926ddmyZUs++MEP9u7buXNnfvCDH+Smm25Kd3d3Ro0atctxFRUVb7rmCQB7j7EZwDubGSUAAACDZPTo0Zk+fXra2tp69/X09KStrS2zZs0aUJ+nnXZa/u3f/i0bNmzo3WbMmJHPfOYz2bBhQ78hCQAAsOfMKAEAABhEzc3NWbBgQWbMmJGZM2emtbU127dvT1NTU5Jk/vz5mThxYu96Izt27Mhjjz3W++fnnnsuGzZsyEEHHZSjjjoqBx98cI499tg+P+PAAw/Mu9/97l32AwAAxROUAAAADKJ58+blxRdfzNKlS9PR0ZG6urqsWbOmd4H3zZs3Z+TI30zuf/7553PCCSf0fl6xYkVWrFiRU045JWvXrt3b5QMAQNkRlAAAAAyyhQsXZuHChf1+99vhR21tbQqFQlH9C1AAAGDwWKMEAAAAAAAoW4ISAAAAAACgbAlKAAAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLI1oKBk5cqVqa2tTWVlZerr6/Pwww/v0XF33313RowYkblz5/bZXygUsnTp0owfPz77779/Ghoa8sQTTwykNAAAAAAAgD22X7EHrF69Os3NzVm1alXq6+vT2tqaxsbGbNy4MePGjdvtcZs2bcpXvvKVfOQjH9nlu2uuuSY33HBD7rzzzkyePDlLlixJY2NjHnvssVRWVhZbIgAAAAAMidpF95W6BAAGWdEzSq6//vqce+65aWpqytSpU7Nq1aoccMABue2223Z7zM6dO/OZz3wmV1xxRY444og+3xUKhbS2tubyyy/Ppz71qRx//PH5xje+keeffz733HNPv/11d3enq6urzwYAAAAAAFCsooKSHTt2ZN26dWloaPhNByNHpqGhIe3t7bs97k/+5E8ybty4nHPOObt89/TTT6ejo6NPn1VVVamvr99tny0tLamqqurdJk2aVMxpAAAAAAAAJCkyKNm6dWt27tyZ6urqPvurq6vT0dHR7zE//OEP89//+3/Prbfe2u/3bxxXTJ+LFy/Otm3berdnnnmmmNMAAAAAAABIMoA1Sorxyiuv5LOf/WxuvfXWjB07dtD6raioSEVFxaD1BwAAAAAAlKeigpKxY8dm1KhR6ezs7LO/s7MzNTU1u7R/6qmnsmnTpnziE5/o3dfT0/PrH7zfftm4cWPvcZ2dnRk/fnyfPuvq6oopDwAAAAAAoChFBSWjR4/O9OnT09bWlrlz5yb5dfDR1taWhQsX7tL+mGOOyb/927/12Xf55ZfnlVdeyde//vVMmjQp73rXu1JTU5O2trbeYKSrqysPPfRQzjvvvIGdFQAAAADsw2oX3Vf0MZuWzxmCSgCGv6JfvdXc3JwFCxZkxowZmTlzZlpbW7N9+/Y0NTUlSebPn5+JEyempaUllZWVOfbYY/scf8ghhyRJn/0XXXRRrrrqqkyZMiWTJ0/OkiVLMmHChN4wBgAAAAAAYCgUHZTMmzcvL774YpYuXZqOjo7U1dVlzZo1vYuxb968OSNHFrVGfC655JJs3749n//85/Pyyy/n5JNPzpo1a1JZWVlseQAAAAAAAHtsRKFQKJS6iLerq6srVVVV2bZtW8aMGVPqcgDeMQYyVXsgTO8G2Ps8A1Ms9wzA4Nhb46yBMDYD+I1inn+Lm/oBAAAAAAAwjAhKAAAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICytV+pCwAoV7WL7iuq/ablc4aoEgAAAAAoX2aUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULasUQKwjyh2TRMAAAAA4K0JSgB42wYS4licHgAAAIB3Aq/eAgAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgBAAAYZCtXrkxtbW0qKytTX1+fhx9+eLdtH3300Xz6059ObW1tRowYkdbW1l3atLS05MQTT8zBBx+ccePGZe7cudm4ceMQngEAAJSP/UpdAAAAwHCyevXqNDc3Z9WqVamvr09ra2saGxuzcePGjBs3bpf2r732Wo444oiceeaZ+aM/+qN++/z+97+f888/PyeeeGJ+9atf5bLLLsvpp5+exx57LAceeOBQnxLAsFa76L5SlwBAiQlKAAAABtH111+fc889N01NTUmSVatW5b777sttt92WRYsW7dL+xBNPzIknnpgk/X6fJGvWrOnz+Y477si4ceOybt26fPSjH+33mO7u7nR3d/d+7urqGtD5AADAcOfVWwAAAINkx44dWbduXRoaGnr3jRw5Mg0NDWlvbx+0n7Nt27YkyWGHHbbbNi0tLamqqurdJk2aNGg/HwAAhhNBCQAAwCDZunVrdu7cmerq6j77q6ur09HRMSg/o6enJxdddFE+/OEP59hjj91tu8WLF2fbtm292zPPPDMoPx8AAIYbr94CAADYh5x//vl55JFH8sMf/vBN21VUVKSiomIvVQUAAPsuQQkAAMAgGTt2bEaNGpXOzs4++zs7O1NTU/O2+1+4cGHuvffe/OAHP8h73/vet90fAAAgKAEAABg0o0ePzvTp09PW1pa5c+cm+fWrstra2rJw4cIB91soFHLBBRfkO9/5TtauXZvJkycPUsUADCe1i+4r+phNy+cMQSUA+xZBCQAAwCBqbm7OggULMmPGjMycOTOtra3Zvn17mpqakiTz58/PxIkT09LSkuTXC8A/9thjvX9+7rnnsmHDhhx00EE56qijkvz6dVt33XVX/v7v/z4HH3xw73onVVVV2X///UtwlgAAMHwMaDH3lStXpra2NpWVlamvr8/DDz+827Z/93d/lxkzZuSQQw7JgQcemLq6unzzm9/s0+Zzn/tcRowY0WebPXv2QEoDAAAoqXnz5mXFihVZunRp6urqsmHDhqxZs6Z3gffNmzfnhRde6G3//PPP54QTTsgJJ5yQF154IStWrMgJJ5yQ//bf/ltvm5tvvjnbtm3LqaeemvHjx/duq1ev3uvnBwAAw03RM0pWr16d5ubmrFq1KvX19WltbU1jY2M2btyYcePG7dL+sMMOyx//8R/nmGOOyejRo3Pvvfemqakp48aNS2NjY2+72bNn5/bbb+/9bNFBAABgX7Vw4cLdvmpr7dq1fT7X1tamUCi8aX9v9T0AADBwRc8ouf7663PuueemqakpU6dOzapVq3LAAQfktttu67f9qaeemt///d/PBz7wgRx55JG58MILc/zxx+eHP/xhn3YVFRWpqanp3Q499NDd1tDd3Z2urq4+GwAAAAAAQLGKCkp27NiRdevWpaGh4TcdjByZhoaGtLe3v+XxhUIhbW1t2bhxYz760Y/2+W7t2rUZN25cjj766Jx33nl56aWXdttPS0tLqqqqerdJkyYVcxoAAAAAAABJigxKtm7dmp07d/a+W/cN1dXVvYsJ9mfbtm056KCDMnr06MyZMyc33nhjfvd3f7f3+9mzZ+cb3/hG2tracvXVV+f73/9+zjjjjOzcubPf/hYvXpxt27b1bs8880wxpwEAAAAAAJBkAGuUDMTBBx+cDRs25NVXX01bW1uam5tzxBFH5NRTT02SnHXWWb1tjzvuuBx//PE58sgjs3bt2px22mm79FdRUWENEwAAAAAA4G0rKigZO3ZsRo0alc7Ozj77Ozs7U1NTs9vjRo4cmaOOOipJUldXl5///OdpaWnpDUp+2xFHHJGxY8fmySef7DcoAXinqV10X6lLAAAAAAAGoKhXb40ePTrTp09PW1tb776enp60tbVl1qxZe9xPT09Puru7d/v9s88+m5deeinjx48vpjwAAAAAAICiFP3qrebm5ixYsCAzZszIzJkz09ramu3bt6epqSlJMn/+/EycODEtLS1Jfr3w+owZM3LkkUemu7s7999/f775zW/m5ptvTpK8+uqrueKKK/LpT386NTU1eeqpp3LJJZfkqKOOSmNj4yCeKgAAAAAAQF9FByXz5s3Liy++mKVLl6ajoyN1dXVZs2ZN7wLvmzdvzsiRv5mosn379nzxi1/Ms88+m/333z/HHHNM/uqv/irz5s1LkowaNSo/+9nPcuedd+bll1/OhAkTcvrpp+fKK6+0DgkAAAAAADCkRhQKhUKpi3i7urq6UlVVlW3btmXMmDGlLgcoQ9YoKd6m5XNKXQLAPs0zMMVyzwD0r9zHc8ZmwHBVzPNvUWuUAAAAAAAADCeCEgAAAAAAoGwVvUYJAAyGYqe3mw4OAAAAwFAwowQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyZY0SgN9S7NoZAAAAAMC+y4wSAAAAAACgbAlKAAAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLK1X6kLAIA9UbvovqKP2bR8zhBUAgAAAMBwYkYJAAAAAABQtgQlAAAAAABA2RKUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULYEJQAAAAAAQNkSlAAAAAAAAGVLUAIAAAAAAJQtQQkAAAAAAFC2BCUAAAAAAEDZEpQAAAAAAABla79SFwAAAAAAg6F20X2lLgGAfZAZJQAAAAAAQNkyowSAYWsgv022afmcIagEAAAAgHcqM0oAAAAAAICyJSgBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbA0oKFm5cmVqa2tTWVmZ+vr6PPzww7tt+3d/93eZMWNGDjnkkBx44IGpq6vLN7/5zT5tCoVCli5dmvHjx2f//fdPQ0NDnnjiiYGUBgAAUHLFjJkeffTRfPrTn05tbW1GjBiR1tbWt90nAACw54oOSlavXp3m5uYsW7Ys69evz7Rp09LY2JgtW7b02/6www7LH//xH6e9vT0/+9nP0tTUlKampvzTP/1Tb5trrrkmN9xwQ1atWpWHHnooBx54YBobG/PLX/5y4GcGAABQAsWOmV577bUcccQRWb58eWpqagalTwAAYM+NKBQKhWIOqK+vz4knnpibbropSdLT05NJkyblggsuyKJFi/aojw9+8IOZM2dOrrzyyhQKhUyYMCFf/vKX85WvfCVJsm3btlRXV+eOO+7IWWed9Zb9dXV1paqqKtu2bcuYMWOKOR2AXdQuuq/UJVBCm5bPKXUJAHvEM/A719sZM9XW1uaiiy7KRRddNGh9vsE9A5QD47niGQMBw1Uxz79FzSjZsWNH1q1bl4aGht90MHJkGhoa0t7e/pbHFwqFtLW1ZePGjfnoRz+aJHn66afT0dHRp8+qqqrU19fvts/u7u50dXX12QAAAErt7Y6ZBrNP4yYAANgz+xXTeOvWrdm5c2eqq6v77K+urs7jjz++2+O2bduWiRMnpru7O6NGjcqf//mf53d/93eTJB0dHb19/Hafb3z321paWnLFFVcUUzoAAMCQG+iYaSj6NG4CYE8UOwvHDBRgOBrQYu7FOvjgg7Nhw4b8+Mc/zte+9rU0Nzdn7dq1A+5v8eLF2bZtW+/2zDPPDF6xAAAAw4BxEwAA7JmiZpSMHTs2o0aNSmdnZ5/9nZ2du110MPn1tPCjjjoqSVJXV5ef//znaWlpyamnntp7XGdnZ8aPH9+nz7q6un77q6ioSEVFRTGlAwAADLmBjpmGok/jJgAA2DNFzSgZPXp0pk+fnra2tt59PT09aWtry6xZs/a4n56ennR3dydJJk+enJqamj59dnV15aGHHiqqTwAAgFIbrDHTUPcJAAD8RlEzSpKkubk5CxYsyIwZMzJz5sy0trZm+/btaWpqSpLMnz8/EydOTEtLS5Jfvxd3xowZOfLII9Pd3Z37778/3/zmN3PzzTcnSUaMGJGLLrooV111VaZMmZLJkydnyZIlmTBhQubOnTt4ZwoAALAXFDtm2rFjRx577LHePz/33HPZsGFDDjrooN6Z+W/VJwAAMHBFByXz5s3Liy++mKVLl6ajoyN1dXVZs2ZN78KCmzdvzsiRv5mosn379nzxi1/Ms88+m/333z/HHHNM/uqv/irz5s3rbXPJJZdk+/bt+fznP5+XX345J598ctasWZPKyspBOEUAAIC9p9gx0/PPP58TTjih9/OKFSuyYsWKnHLKKb1rO75VnwAAwMCNKBQKhVIX8XZ1dXWlqqoq27Zty5gxY0pdDrCPq110X6lLoIQ2LZ9T6hIA9ohnYIrlngHKgfHc0DNmAvYVxTz/FrVGCQAAAAAAwHAiKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbAlKAAAAAACAsrVfqQsAGEq1i+4rdQkAAAAAwDuYGSUAAAAAAEDZEpQAAAAAAABlS1ACAAAAAACULUEJAAAAAABQtgQlAAAAAABA2RKUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULYEJQAAAAAAQNkSlAAAAAAAAGVLUAIAAAAAAJQtQQkAAAAAAFC2BCUAAAAAAEDZEpQAAAAAAABlS1ACAAAAAACULUEJAAAAAABQtgQlAAAAAABA2RKUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULYEJQAAAAAAQNkSlAAAAAAAAGVLUAIAAAAAAJStAQUlK1euTG1tbSorK1NfX5+HH354t21vvfXWfOQjH8mhhx6aQw89NA0NDbu0/9znPpcRI0b02WbPnj2Q0gAAAAAAAPZY0UHJ6tWr09zcnGXLlmX9+vWZNm1aGhsbs2XLln7br127NmeffXa+973vpb29PZMmTcrpp5+e5557rk+72bNn54UXXujd/vqv/3pgZwQAAAAAALCHig5Krr/++px77rlpamrK1KlTs2rVqhxwwAG57bbb+m3/rW99K1/84hdTV1eXY445Jn/5l3+Znp6etLW19WlXUVGRmpqa3u3QQw8d2BkBAAAAAADsoaKCkh07dmTdunVpaGj4TQcjR6ahoSHt7e171Mdrr72W119/PYcddlif/WvXrs24ceNy9NFH57zzzstLL7202z66u7vT1dXVZwMAAAAAAChWUUHJ1q1bs3PnzlRXV/fZX11dnY6Ojj3q49JLL82ECRP6hC2zZ8/ON77xjbS1teXqq6/O97///ZxxxhnZuXNnv320tLSkqqqqd5s0aVIxpwEAAAAAAJAk2W9v/rDly5fn7rvvztq1a1NZWdm7/6yzzur983HHHZfjjz8+Rx55ZNauXZvTTjttl34WL16c5ubm3s9dXV3CEgAAAAAAoGhFBSVjx47NqFGj0tnZ2Wd/Z2dnampq3vTYFStWZPny5fnud7+b448//k3bHnHEERk7dmyefPLJfoOSioqKVFRUFFM6AAAAAPuQ2kX3lboEAMpEUa/eGj16dKZPn95nIfY3FmafNWvWbo+75pprcuWVV2bNmjWZMWPGW/6cZ599Ni+99FLGjx9fTHkAAAAAAABFKSooSZLm5ubceuutufPOO/Pzn/885513XrZv356mpqYkyfz587N48eLe9ldffXWWLFmS2267LbW1teno6EhHR0deffXVJMmrr76aiy++OD/60Y+yadOmtLW15VOf+lSOOuqoNDY2DtJpAgAAAAAA7KrooGTevHlZsWJFli5dmrq6umzYsCFr1qzpXeB98+bNeeGFF3rb33zzzdmxY0f+8A//MOPHj+/dVqxYkSQZNWpUfvazn+WTn/xk3v/+9+ecc87J9OnT86//+q9erwUAAOyTVq5cmdra2lRWVqa+vj4PP/zwm7b/9re/nWOOOSaVlZU57rjjcv/99/f5/tVXX83ChQvz3ve+N/vvv3+mTp2aVatWDeUpAABA2RjQYu4LFy7MwoUL+/1u7dq1fT5v2rTpTfvaf//980//9E8DKQMAAOAdZ/Xq1Wlubs6qVatSX1+f1tbWNDY2ZuPGjRk3btwu7R988MGcffbZaWlpye/93u/lrrvuyty5c7N+/foce+yxSX49s/9f/uVf8ld/9Vepra3NP//zP+eLX/xiJkyYkE9+8pN7+xQBAGBYKXpGCQAAALt3/fXX59xzz01TU1PvzI8DDjggt912W7/tv/71r2f27Nm5+OKL84EPfCBXXnllPvjBD+amm27qbfPggw9mwYIFOfXUU1NbW5vPf/7zmTZt2lvOVAEAAN6aoAQAAGCQ7NixI+vWrUtDQ0PvvpEjR6ahoSHt7e39HtPe3t6nfZI0Njb2aX/SSSflH/7hH/Lcc8+lUCjke9/7Xv73//7fOf3003dbS3d3d7q6uvpsAADArgQlAAAAg2Tr1q3ZuXNn7xqOb6iurk5HR0e/x3R0dLxl+xtvvDFTp07Ne9/73owePTqzZ8/OypUr89GPfnS3tbS0tKSqqqp3mzRp0ts4MwAAGL4EJQAAAO9wN954Y370ox/lH/7hH7Ju3bpcd911Of/88/Pd7353t8csXrw427Zt692eeeaZvVgxAADsOwa0mDsADFe1i+4r+phNy+cMQSUA7IvGjh2bUaNGpbOzs8/+zs7O1NTU9HtMTU3Nm7b/f//v/+Wyyy7Ld77zncyZ8+v/5hx//PHZsGFDVqxYsctru95QUVGRioqKt3tKAAAw7JlRAgAAMEhGjx6d6dOnp62trXdfT09P2traMmvWrH6PmTVrVp/2SfLAAw/0tn/99dfz+uuvZ+TIvsO3UaNGpaenZ5DPAAAAyo8ZJQAAAIOoubk5CxYsyIwZMzJz5sy0trZm+/btaWpqSpLMnz8/EydOTEtLS5LkwgsvzCmnnJLrrrsuc+bMyd13352f/OQnueWWW5IkY8aMySmnnJKLL744+++/fw4//PB8//vfzze+8Y1cf/31JTtPAAAYLgQlAAAAg2jevHl58cUXs3Tp0nR0dKSuri5r1qzpXbB98+bNfWaHnHTSSbnrrrty+eWX57LLLsuUKVNyzz335Nhjj+1tc/fdd2fx4sX5zGc+k3//93/P4Ycfnq997Wv5whe+sNfPD4Dy5nXFwHA0olAoFEpdxNvV1dWVqqqqbNu2LWPGjCl1OcA7yEAe4KBYHvqBUvAMTLHcM8C+xnhu+DBmAkqhmOdfa5QAAAAAAABlS1ACAAAAAACULUEJAAAAAABQtgQlAAAAAABA2RKUAAAAAAAAZUtQAgAAAAAAlK39Sl0AwJ6qXXRfqUsAAAAAAIYZM0oAAAAAAICyJSgBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbAlKAAAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbAlKAAAAAACAsjWgoGTlypWpra1NZWVl6uvr8/DDD++27a233pqPfOQjOfTQQ3PooYemoaFhl/aFQiFLly7N+PHjs//++6ehoSFPPPHEQEoDAAAAAADYY0UHJatXr05zc3OWLVuW9evXZ9q0aWlsbMyWLVv6bb927dqcffbZ+d73vpf29vZMmjQpp59+ep577rneNtdcc01uuOGGrFq1Kg899FAOPPDANDY25pe//OXAzwwAAAAAAOAtFB2UXH/99Tn33HPT1NSUqVOnZtWqVTnggANy22239dv+W9/6Vr74xS+mrq4uxxxzTP7yL/8yPT09aWtrS/Lr2SStra25/PLL86lPfSrHH398vvGNb+T555/PPffc87ZODgAAAAAA4M0UFZTs2LEj69atS0NDw286GDkyDQ0NaW9v36M+Xnvttbz++us57LDDkiRPP/10Ojo6+vRZVVWV+vr63fbZ3d2drq6uPhsAAAAAAECxigpKtm7dmp07d6a6urrP/urq6nR0dOxRH5deemkmTJjQG4y8cVwxfba0tKSqqqp3mzRpUjGnAQAAAAAAkCTZb2/+sOXLl+fuu+/O2rVrU1lZOeB+Fi9enObm5t7PXV1dwhIASqZ20X1FH7Np+ZwhqAQAAACAYhUVlIwdOzajRo1KZ2dnn/2dnZ2pqal502NXrFiR5cuX57vf/W6OP/743v1vHNfZ2Znx48f36bOurq7fvioqKlJRUVFM6QAAAAAAALso6tVbo0ePzvTp03sXYk/SuzD7rFmzdnvcNddckyuvvDJr1qzJjBkz+nw3efLk1NTU9Omzq6srDz300Jv2CQAAAAAA8HYV/eqt5ubmLFiwIDNmzMjMmTPT2tqa7du3p6mpKUkyf/78TJw4MS0tLUmSq6++OkuXLs1dd92V2tra3nVHDjrooBx00EEZMWJELrroolx11VWZMmVKJk+enCVLlmTChAmZO3fu4J0pAAAAAADAbyk6KJk3b15efPHFLF26NB0dHamrq8uaNWt6F2PfvHlzRo78zUSVm2++OTt27Mgf/uEf9uln2bJl+epXv5okueSSS7J9+/Z8/vOfz8svv5yTTz45a9aseVvrmAAAAAAAALyVEYVCoVDqIt6urq6uVFVVZdu2bRkzZkypywGGyEAWzIZ3Kou5A2+XZ2CK5Z4B9jXGgMOH8Q9QCsU8/xa1RgkAAAAAAMBwIigBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbO1X6gIAAAAAGN4szA7AO5kZJQAAAAAAQNkSlAAAAAAAAGXLq7eAkjDtGgAYzlauXJlrr702HR0dmTZtWm688cbMnDlzt+2//e1vZ8mSJdm0aVOmTJmSq6++Oh//+Mf7tPn5z3+eSy+9NN///vfzq1/9KlOnTs3f/u3f5n3ve99Qnw4AAAxrZpQAAAAMotWrV6e5uTnLli3L+vXrM23atDQ2NmbLli39tn/wwQdz9tln55xzzslPf/rTzJ07N3Pnzs0jjzzS2+app57KySefnGOOOSZr167Nz372syxZsiSVlZV767QAAGDYGlEoFAqlLuLt6urqSlVVVbZt25YxY8aUuhxgD5hRQrnbtHxOqUsA9nGegd+56uvrc+KJJ+amm25KkvT09GTSpEm54IILsmjRol3az5s3L9u3b8+9997bu+9DH/pQ6urqsmrVqiTJWWedlXe961355je/ucd1dHd3p7u7u/dzV1dXJk2a5J4BSsIYkGIZMwFvVzFjJjNKAAAABsmOHTuybt26NDQ09O4bOXJkGhoa0t7e3u8x7e3tfdonSWNjY2/7np6e3HfffXn/+9+fxsbGjBs3LvX19bnnnnvetJaWlpZUVVX1bpMmTXp7JwcAAMOUNUqAQeG3gwAAkq1bt2bnzp2prq7us7+6ujqPP/54v8d0dHT0276joyNJsmXLlrz66qtZvnx5rrrqqlx99dVZs2ZN/uAP/iDf+973csopp/Tb7+LFi9Pc3Nz7+Y0ZJQAAQF+CEgAAgHewnp6eJMmnPvWp/NEf/VGSpK6uLg8++GBWrVq126CkoqIiFRUVe61OAADYV3n1FgAAwCAZO3ZsRo0alc7Ozj77Ozs7U1NT0+8xNTU1b9p+7Nix2W+//TJ16tQ+bT7wgQ9k8+bNg1g9AACUJ0EJAADAIBk9enSmT5+etra23n09PT1pa2vLrFmz+j1m1qxZfdonyQMPPNDbfvTo0TnxxBOzcePGPm3+9//+3zn88MMH+QwAAKD8ePUWAADAIGpubs6CBQsyY8aMzJw5M62trdm+fXuampqSJPPnz8/EiRPT0tKSJLnwwgtzyimn5LrrrsucOXNy99135yc/+UluueWW3j4vvvjizJs3Lx/96EfzsY99LGvWrMk//uM/Zu3ataU4RQAAGFYEJQAAAINo3rx5efHFF7N06dJ0dHSkrq4ua9as6V2wffPmzRk58jeT+0866aTcddddufzyy3PZZZdlypQpueeee3Lsscf2tvn93//9rFq1Ki0tLfnSl76Uo48+On/7t3+bk08+ea+fHwAADDcjCoVCodRFvF1dXV2pqqrKtm3bMmbMmFKXA2WpdtF9pS4B9imbls8pdQnAPs4zMMVyzwClZMxIsYyZgLermOdfa5QAAAAAAABlS1ACAAAAAACULUEJAAAAAABQtgQlAAAAAABA2RKUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULYEJQAAAAAAQNkSlAAAAAAAAGVLUAIAAAAAAJQtQQkAAAAAAFC2BCUAAAAAAEDZEpQAAAAAAABlS1ACAAAAAACULUEJAAAAAABQtgYUlKxcuTK1tbWprKxMfX19Hn744d22ffTRR/PpT386tbW1GTFiRFpbW3dp89WvfjUjRozosx1zzDEDKQ0AAAAAAGCPFR2UrF69Os3NzVm2bFnWr1+fadOmpbGxMVu2bOm3/WuvvZYjjjgiy5cvT01NzW77/Z3f+Z288MILvdsPf/jDYksDAAAAAAAoStFByfXXX59zzz03TU1NmTp1alatWpUDDjggt912W7/tTzzxxFx77bU566yzUlFRsdt+99tvv9TU1PRuY8eOLbY0AAAAAACAohQVlOzYsSPr1q1LQ0PDbzoYOTINDQ1pb29/W4U88cQTmTBhQo444oh85jOfyebNm3fbtru7O11dXX02AAAAAACAYhUVlGzdujU7d+5MdXV1n/3V1dXp6OgYcBH19fW54447smbNmtx88815+umn85GPfCSvvPJKv+1bWlpSVVXVu02aNGnAPxsAAAAAAChf+5W6gCQ544wzev98/PHHp76+Pocffnj+5m/+Juecc84u7RcvXpzm5ubez11dXcISAPYptYvuK6r9puVzhqgSAAAAgPJWVFAyduzYjBo1Kp2dnX32d3Z2vulC7cU65JBD8v73vz9PPvlkv99XVFS86XonAAAAAAAAe6KoV2+NHj0606dPT1tbW+++np6etLW1ZdasWYNW1Kuvvpqnnnoq48ePH7Q+AQAAAAAAflvRr95qbm7OggULMmPGjMycOTOtra3Zvn17mpqakiTz58/PxIkT09LSkuTXC8A/9thjvX9+7rnnsmHDhhx00EE56qijkiRf+cpX8olPfCKHH354nn/++SxbtiyjRo3K2WefPVjnCQAAAAAAsIuig5J58+blxRdfzNKlS9PR0ZG6urqsWbOmd4H3zZs3Z+TI30xUef7553PCCSf0fl6xYkVWrFiRU045JWvXrk2SPPvsszn77LPz0ksv5T3veU9OPvnk/OhHP8p73vOet3l6AAAAAAAAuzegxdwXLlyYhQsX9vvdG+HHG2pra1MoFN60v7vvvnsgZQAAAAAAALwtRa1RAgAAAAAAMJwISgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICytV+pCwAAAABg31G76L5SlwAAg8qMEgAAAAAAoGyZUQIA+4CB/NbepuVzhqASAAAAgOHFjBIAAAAAAKBsCUoAAAAAAICyJSgBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbO1X6gIAAAAAAP6j2kX3FX3MpuVzhqASoByYUQIAADDIVq5cmdra2lRWVqa+vj4PP/zwm7b/9re/nWOOOSaVlZU57rjjcv/99++27Re+8IWMGDEira2tg1w1AACUJ0EJAADAIFq9enWam5uzbNmyrF+/PtOmTUtjY2O2bNnSb/sHH3wwZ599ds4555z89Kc/zdy5czN37tw88sgju7T9zne+kx/96EeZMGHCUJ8GAACUDUEJAADAILr++utz7rnnpqmpKVOnTs2qVatywAEH5Lbbbuu3/de//vXMnj07F198cT7wgQ/kyiuvzAc/+MHcdNNNfdo999xzueCCC/Ktb30r73rXu/bGqQAAQFkQlAAAAAySHTt2ZN26dWloaOjdN3LkyDQ0NKS9vb3fY9rb2/u0T5LGxsY+7Xt6evLZz342F198cX7nd35nj2rp7u5OV1dXnw0AANiVoAQAAGCQbN26NTt37kx1dXWf/dXV1eno6Oj3mI6Ojrdsf/XVV2e//fbLl770pT2upaWlJVVVVb3bpEmTijgTAAAoH/uVugDgnad20X2lLgEAgP/funXr8vWvfz3r16/PiBEj9vi4xYsXp7m5ufdzV1eXsAQAAPphRgkAAMAgGTt2bEaNGpXOzs4++zs7O1NTU9PvMTU1NW/a/l//9V+zZcuWvO9978t+++2X/fbbL7/4xS/y5S9/ObW1tbutpaKiImPGjOmzAQAAuxKUAAAADJLRo0dn+vTpaWtr693X09OTtra2zJo1q99jZs2a1ad9kjzwwAO97T/72c/mZz/7WTZs2NC7TZgwIRdffHH+6Z/+aehOBgAAyoRXbwEAAAyi5ubmLFiwIDNmzMjMmTPT2tqa7du3p6mpKUkyf/78TJw4MS0tLUmSCy+8MKecckquu+66zJkzJ3fffXd+8pOf5JZbbkmSvPvd78673/3uPj/jXe96V2pqanL00Ufv3ZMDAIBhSFACAAAwiObNm5cXX3wxS5cuTUdHR+rq6rJmzZreBds3b96ckSN/M7n/pJNOyl133ZXLL788l112WaZMmZJ77rknxx57bKlOAQAAyoqgBAAAYJAtXLgwCxcu7Pe7tWvX7rLvzDPPzJlnnrnH/W/atGmAlQEAAL/NGiUAAAAAAEDZEpQAAAAAAABlS1ACAAAAAACULUEJAAAAAABQtizmDgDDVO2i+4o+ZtPyOUNQCQAAAMA7lxklAAAAAABA2RKUAAAAAAAAZWtAQcnKlStTW1ubysrK1NfX5+GHH95t20cffTSf/vSnU1tbmxEjRqS1tfVt9wkAAAAAADAYig5KVq9enebm5ixbtizr16/PtGnT0tjYmC1btvTb/rXXXssRRxyR5cuXp6amZlD6BAAAAAAAGAxFByXXX399zj333DQ1NWXq1KlZtWpVDjjggNx22239tj/xxBNz7bXX5qyzzkpFRcWg9AkAAAAAADAYigpKduzYkXXr1qWhoeE3HYwcmYaGhrS3tw+ogIH02d3dna6urj4bAAAAAABAsYoKSrZu3ZqdO3emurq6z/7q6up0dHQMqICB9NnS0pKqqqrebdKkSQP62QAAAAAAQHkb0GLupbZ48eJs27atd3vmmWdKXRIAAAAAALAP2q+YxmPHjs2oUaPS2dnZZ39nZ+duF2ofij4rKip2u94JAAAAAADAnipqRsno0aMzffr0tLW19e7r6elJW1tbZs2aNaAChqJPAAAAAACAPVHUjJIkaW5uzoIFCzJjxozMnDkzra2t2b59e5qampIk8+fPz8SJE9PS0pLk14u1P/bYY71/fu6557Jhw4YcdNBBOeqoo/aoTwAAAAAAgKFQdFAyb968vPjii1m6dGk6OjpSV1eXNWvW9C7Gvnnz5owc+ZuJKs8//3xOOOGE3s8rVqzIihUrcsopp2Tt2rV71CcAAAAAQ6N20X2lLgEASqrooCRJFi5cmIULF/b73Rvhxxtqa2tTKBTeVp8AAAAAAABDYUBBCbDv8JtBAAAAAAC7V9Ri7gAAAAAAAMOJoAQAAAAAAChbghIAAAAAAKBsWaMEAAAAANjnDWSd1k3L5wxBJcC+xowSAAAAAACgbAlKAAAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICytV+pCwAA3jlqF91X9DGbls8ZgkoAAAAA9g4zSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChb1iiBfchA1g4AAAAAAGD3zCgBAAAAAADKlqAEAAAAAAAoW4ISAAAAAACgbAlKAAAAAACAsmUxdwAAgEG2cuXKXHvtteno6Mi0adNy4403ZubMmbtt/+1vfztLlizJpk2bMmXKlFx99dX5+Mc/niR5/fXXc/nll+f+++/P//k//ydVVVVpaGjI8uXLM2HChL11SsA+onbRfaUuAQD2OWaUAAAADKLVq1enubk5y5Yty/r16zNt2rQ0NjZmy5Yt/bZ/8MEHc/bZZ+ecc87JT3/608ydOzdz587NI488kiR57bXXsn79+ixZsiTr16/P3/3d32Xjxo355Cc/uTdPCwAAhq0RhUKhUOoi3q6urq5UVVVl27ZtGTNmTKnLgSHjN4OAd6JNy+eUugQoS56B37nq6+tz4okn5qabbkqS9PT0ZNKkSbnggguyaNGiXdrPmzcv27dvz7333tu770Mf+lDq6uqyatWqfn/Gj3/848ycOTO/+MUv8r73vW+P6nLPQHkwboTiGM/A8FXM868ZJQAAAINkx44dWbduXRoaGnr3jRw5Mg0NDWlvb+/3mPb29j7tk6SxsXG37ZNk27ZtGTFiRA455JDdtunu7k5XV1efDQAA2JWgBAAAYJBs3bo1O3fuTHV1dZ/91dXV6ejo6PeYjo6Ootr/8pe/zKWXXpqzzz77TX8zrqWlJVVVVb3bpEmTijwbAAAoD4ISAACAfcTrr7+e//yf/3MKhUJuvvnmN227ePHibNu2rXd75pln9lKVAACwb9mv1AVAufLeWACA4Wfs2LEZNWpUOjs7++zv7OxMTU1Nv8fU1NTsUfs3QpJf/OIX+Zd/+Ze3fM9yRUVFKioqBnAWAFA+iv3/M9Y0geHJjBIAAIBBMnr06EyfPj1tbW29+3p6etLW1pZZs2b1e8ysWbP6tE+SBx54oE/7N0KSJ554It/97nfz7ne/e2hOAAAAypAZJQAAAIOoubk5CxYsyIwZMzJz5sy0trZm+/btaWpqSpLMnz8/EydOTEtLS5LkwgsvzCmnnJLrrrsuc+bMyd13352f/OQnueWWW5L8OiT5wz/8w6xfvz733ntvdu7c2bt+yWGHHZbRo0eX5kQBAGCYGNCMkpUrV6a2tjaVlZWpr6/Pww8//Kbtv/3tb+eYY45JZWVljjvuuNx///19vv/c5z6XESNG9Nlmz549kNIAAABKat68eVmxYkWWLl2aurq6bNiwIWvWrOldsH3z5s154YUXetufdNJJueuuu3LLLbdk2rRp+R//43/knnvuybHHHpskee655/IP//APefbZZ1NXV5fx48f3bg8++GBJzhEAAIaTomeUrF69Os3NzVm1alXq6+vT2tqaxsbGbNy4MePGjdul/YMPPpizzz47LS0t+b3f+73cddddmTt3btavX9/74J8ks2fPzu2339772bt0AQCAfdXChQuzcOHCfr9bu3btLvvOPPPMnHnmmf22r62tTaFQGMzyAACA/6DoGSXXX399zj333DQ1NWXq1KlZtWpVDjjggNx22239tv/617+e2bNn5+KLL84HPvCBXHnllfngBz+Ym266qU+7ioqK1NTU9G6HHnrobmvo7u5OV1dXnw0AAAAAAKBYRQUlO3bsyLp169LQ0PCbDkaOTENDQ9rb2/s9pr29vU/7JGlsbNyl/dq1azNu3LgcffTROe+88/LSSy/tto6WlpZUVVX1bpMmTSrmNAAAAAAAAJIU+eqtrVu3ZufOnb3v1n1DdXV1Hn/88X6P6ejo6Lf9G4sPJr9+7dYf/MEfZPLkyXnqqady2WWX5Ywzzkh7e3tGjRq1S5+LFy9Oc3Nz7+euri5hCQCUSO2i+4o+ZtPyOUNQCQAAAEDxil6jZCicddZZvX8+7rjjcvzxx+fII4/M2rVrc9ppp+3SvqKiwhomAAAAAADA21bUq7fGjh2bUaNGpbOzs8/+zs7O1NTU9HtMTU1NUe2T5IgjjsjYsWPz5JNPFlMeAAAAAABAUYoKSkaPHp3p06enra2td19PT0/a2toya9asfo+ZNWtWn/ZJ8sADD+y2fZI8++yzeemllzJ+/PhiygMAAAAAAChKUUFJkjQ3N+fWW2/NnXfemZ///Oc577zzsn379jQ1NSVJ5s+fn8WLF/e2v/DCC7NmzZpcd911efzxx/PVr341P/nJT7Jw4cIkyauvvpqLL744P/rRj7Jp06a0tbXlU5/6VI466qg0NjYO0mkCAAAAAADsqug1SubNm5cXX3wxS5cuTUdHR+rq6rJmzZreBds3b96ckSN/k7+cdNJJueuuu3L55Zfnsssuy5QpU3LPPffk2GOPTZKMGjUqP/vZz3LnnXfm5ZdfzoQJE3L66afnyiuvtA4JAAAAAAAwpEYUCoVCqYt4u7q6ulJVVZVt27ZlzJgxpS4H9kjtovtKXQJAyWxaPqfUJcA+zzMwxXLPwL7HuBHeeYxlYN9RzPNv0a/eAgAAAAAAGC4EJQAAAAAAQNkSlAAAAAAAAGWr6MXcAQAAAADK0UDWDrKuCbzzmVECAAAAAACULTNKYJAM5DcKAAAAAAAoLUEJALDXFRsum6oOAAAADBWv3gIAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgBAAAAAADK1n6lLgAA4K3ULrqv6GM2LZ8zBJUAAAzMQJ5nAIC9w4wSAAAAAACgbAlKAAAAAACAsuXVW9APU6IBAAAAGAxeJQzvfGaUAAAAAAAAZUtQAgAAAAAAlC1BCQAAAAAAULYEJQAAAAAAQNmymDvDnoXZAQAAAADYHUEJAAAAQBH8Qh4ADC+CEgBgWBrI/8DYtHzOEFQCAAAAvJMJSgAAAAAA3kH84hfsXYISAID/n8EIAAAAlJ+RpS4AAAAAAACgVMwoYZ9iwTwAAAAAAAaToAQAAAAoW34hDwDw6i0AAAAAAKBsmVECAAAAALCPG8gMuU3L5wxBJbDvMaMEAAAAAAAoW2aUAAAAAMOGNUcAgGIJSigZD68ADAemtwMAAPsq4xn4tQEFJStXrsy1116bjo6OTJs2LTfeeGNmzpy52/bf/va3s2TJkmzatClTpkzJ1VdfnY9//OO93xcKhSxbtiy33nprXn755Xz4wx/OzTffnClTpgykPACAd7S98csCBi9QWsZMMDj8gh0AsDcUHZSsXr06zc3NWbVqVerr69Pa2prGxsZs3Lgx48aN26X9gw8+mLPPPjstLS35vd/7vdx1112ZO3du1q9fn2OPPTZJcs011+SGG27InXfemcmTJ2fJkiVpbGzMY489lsrKyrd/luwVHmAB4J3Db4ZB6RgzQf+MGQGGh2L/PjfOYF8wolAoFIo5oL6+PieeeGJuuummJElPT08mTZqUCy64IIsWLdql/bx587J9+/bce++9vfs+9KEPpa6uLqtWrUqhUMiECRPy5S9/OV/5yleSJNu2bUt1dXXuuOOOnHXWWbv02d3dne7u7t7P27Zty/ve974888wzGTNmTDGnUxaOXfZPpS4BABimHrmisdQllK2urq5MmjQpL7/8cqqqqkpdDv/BO2HMlBg3MbSMMwF4pzE24bcVM2YqakbJjh07sm7duixevLh338iRI9PQ0JD29vZ+j2lvb09zc3OffY2NjbnnnnuSJE8//XQ6OjrS0NDQ+31VVVXq6+vT3t7e70N/S0tLrrjiil32T5o0qZjTAQDgbapqLXUFvPLKK4KSd5B3ypgpMW4CAMqLsQm7sydjpqKCkq1bt2bnzp2prq7us7+6ujqPP/54v8d0dHT0276jo6P3+zf27a7Nb1u8eHGfgURPT0/+/d//Pe9+97szYsSIYk7pbXkjkfIbWQwV9xhDzT3GUHJ/MdTK/R4rFAp55ZVXMmHChFKXwn/wThkzJf2Pm37xi1+krq6ubP+92RvK/e+mvcE1Hnqu8dBzjYeea7x3uM5DzzUeuGLGTANazL3UKioqUlFR0WffIYccUppikowZM8ZNypByjzHU3GMMJfcXQ62c7zEzSXgz/Y2bRo4cmaS8/73ZW1zjoecaDz3XeOi5xkPPNd47XOeh5xoPzJ6OmUYW0+nYsWMzatSodHZ29tnf2dmZmpqafo+pqal50/Zv/LOYPgEAAN6JjJkAAGDfU1RQMnr06EyfPj1tbW29+3p6etLW1pZZs2b1e8ysWbP6tE+SBx54oLf95MmTU1NT06dNV1dXHnrood32CQAA8E5kzAQAAPueol+91dzcnAULFmTGjBmZOXNmWltbs3379jQ1NSVJ5s+fn4kTJ6alpSVJcuGFF+aUU07Jddddlzlz5uTuu+/OT37yk9xyyy1JkhEjRuSiiy7KVVddlSlTpmTy5MlZsmRJJkyYkLlz5w7emQ6BioqKLFu2bJfp7DBY3GMMNfcYQ8n9xVBzj/FO9U4eM/n3Zui5xkPPNR56rvHQc42Hnmu8d7jOQ8813jtGFAqFQrEH3XTTTbn22mvT0dGRurq63HDDDamvr0+SnHrqqamtrc0dd9zR2/7b3/52Lr/88mzatClTpkzJNddck49//OO93xcKhSxbtiy33HJLXn755Zx88sn58z//87z//e9/+2cIAACwlxkzAQDAvmNAQQkAAAAAAMBwUNQaJQAAAAAAAMOJoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgZZN3d3amrq8uIESOyYcOGUpfDMLFp06acc845mTx5cvbff/8ceeSRWbZsWXbs2FHq0tiHrVy5MrW1tamsrEx9fX0efvjhUpfEMNHS0pITTzwxBx98cMaNG5e5c+dm48aNpS6LYWr58uUZMWJELrroolKXAu9oe/o8+bOf/Swf+chHUllZmUmTJuWaa64pUcX7pq997Ws56aSTcsABB+SQQw7pt82IESN22e6+++69W+g+bE+u8ebNmzNnzpwccMABGTduXC6++OL86le/2ruFDjO1tbW73LfLly8vdVn7NOOxofPVr351l/v1mGOOKXVZ+7Qf/OAH+cQnPpEJEyZkxIgRueeee/p8XygUsnTp0owfPz77779/Ghoa8sQTT5Sm2H3UW13jz33uc7vc17Nnzy5NscOUoGSQXXLJJZkwYUKpy2CYefzxx9PT05O/+Iu/yKOPPpo/+7M/y6pVq3LZZZeVujT2UatXr05zc3OWLVuW9evXZ9q0aWlsbMyWLVtKXRrDwPe///2cf/75+dGPfpQHHnggr7/+ek4//fRs37691KUxzPz4xz/OX/zFX+T4448vdSnwjrcnz5NdXV05/fTTc/jhh2fdunW59tpr89WvfjW33HJLCSvft+zYsSNnnnlmzjvvvDdtd/vtt+eFF17o3ebOnbt3ChwG3uoa79y5M3PmzMmOHTvy4IMP5s4778wdd9yRpUuX7uVKh58/+ZM/6XPfXnDBBaUuaZ9lPDb0fud3fqfP/frDH/6w1CXt07Zv355p06Zl5cqV/X5/zTXX5IYbbsiqVavy0EMP5cADD0xjY2N++ctf7uVK911vdY2TZPbs2X3u67/+67/eixWWgQKD5v777y8cc8wxhUcffbSQpPDTn/601CUxjF1zzTWFyZMnl7oM9lEzZ84snH/++b2fd+7cWZgwYUKhpaWlhFUxXG3ZsqWQpPD973+/1KUwjLzyyiuFKVOmFB544IHCKaecUrjwwgtLXRLsc377efLP//zPC4ceemihu7u7d9+ll15aOProo0tR3j7t9ttvL1RVVfX7XZLCd77znb1az3C0u2t8//33F0aOHFno6Ojo3XfzzTcXxowZ0+fepjiHH3544c/+7M9KXcawYTw2tJYtW1aYNm1aqcsYtn77v2M9PT2FmpqawrXXXtu77+WXXy5UVFQU/vqv/7oEFe77+ntWWLBgQeFTn/pUSeopF2aUDJLOzs6ce+65+eY3v5kDDjig1OVQBrZt25bDDjus1GWwD9qxY0fWrVuXhoaG3n0jR45MQ0ND2tvbS1gZw9W2bduSxN9ZDKrzzz8/c+bM6fN3GVCc336ebG9vz0c/+tGMHj26d19jY2M2btyY//t//28pShy2zj///IwdOzYzZ87MbbfdlkKhUOqSho329vYcd9xxqa6u7t3X2NiYrq6uPProoyWsbN+3fPnyvPvd784JJ5yQa6+91uvMBsh4bO944oknMmHChBxxxBH5zGc+k82bN5e6pGHr6aefTkdHR597uqqqKvX19e7pQbZ27dqMGzcuRx99dM4777y89NJLpS5pWNmv1AUMB4VCIZ/73OfyhS98ITNmzMimTZtKXRLD3JNPPpkbb7wxK1asKHUp7IO2bt2anTt39hk8Jkl1dXUef/zxElXFcNXT05OLLrooH/7wh3PssceWuhyGibvvvjvr16/Pj3/841KXAvus/p4nOzo6Mnny5D7t3nhe6OjoyKGHHrpXaxyu/uRP/iT/6T/9pxxwwAH553/+53zxi1/Mq6++mi996UulLm1Y6Ojo6Pc5943vGJgvfelL+eAHP5jDDjssDz74YBYvXpwXXngh119/falL2+cYjw29+vr63HHHHTn66KPzwgsv5IorrshHPvKRPPLIIzn44INLXd6w88bfrf3d0/7eHTyzZ8/OH/zBH2Ty5Ml56qmnctlll+WMM85Ie3t7Ro0aVeryhgUzSt7EokWL+l1o7z9ujz/+eG688ca88sorWbx4calLZh+zp/fYf/Tcc89l9uzZOfPMM3PuueeWqHKAPXP++efnkUcesUgtg+aZZ57JhRdemG9961uprKwsdTlQcp4nh95ArvGbWbJkST784Q/nhBNOyKWXXppLLrkk11577RCewTvfYF9j9kwx1725uTmnnnpqjj/++HzhC1/IddddlxtvvDHd3d0lPgvY1RlnnJEzzzwzxx9/fBobG3P//ffn5Zdfzt/8zd+UujQYsLPOOiuf/OQnc9xxx2Xu3Lm599578+Mf/zhr164tdWnDhhklb+LLX/5yPve5z71pmyOOOCL/8i//kvb29lRUVPT5bsaMGfnMZz6TO++8cwirZF+2p/fYG55//vl87GMfy0knnWRRTQZs7NixGTVqVDo7O/vs7+zsTE1NTYmqYjhauHBh7r333vzgBz/Ie9/73lKXwzCxbt26bNmyJR/84Ad79+3cuTM/+MEPctNNN6W7u9tvVFFWBvN5sqampt/ngze+K1fFXuNi1dfX58orr0x3d/cuY8pyMZjXuKamJg8//HCffe7j/r2d615fX59f/epX2bRpU44++ughqG74Mh7b+w455JC8//3vz5NPPlnqUoalN+7bzs7OjB8/vnd/Z2dn6urqSlTV8HfEEUdk7NixefLJJ3PaaaeVupxhQVDyJt7znvfkPe95z1u2u+GGG3LVVVf1fn7++efT2NiY1atXp76+fihLZB+3p/dY8uvf/PvYxz6W6dOn5/bbb8/IkSaEMTCjR4/O9OnT09bWlrlz5yb59euR2trasnDhwtIWx7BQKBRywQUX5Dvf+U7Wrl27y2tc4O047bTT8m//9m999jU1NeWYY47JpZdeKiSh7Azm8+SsWbPyx3/8x3n99dfzrne9K0nywAMP5Oijjy7r124Vc40HYsOGDTn00EPLNiRJBvcaz5o1K1/72teyZcuWjBs3Lsmv7+MxY8Zk6tSpg/Izhou3c903bNiQkSNH9l5j9pzx2N736quv5qmnnspnP/vZUpcyLE2ePDk1NTVpa2vrDUa6urry0EMP5bzzzittccPYs88+m5deeqlPOMXbIygZBO973/v6fD7ooIOSJEceeaTfoGVQPPfcczn11FNz+OGHZ8WKFXnxxRd7v/MbJwxEc3NzFixYkBkzZmTmzJlpbW3N9u3b09TUVOrSGAbOP//83HXXXfn7v//7HHzwwb3vpa2qqsr+++9f4urY1x188MG7rHdz4IEH5t3vfrd1cOBN7Mnz5H/5L/8lV1xxRc4555xceumleeSRR/L1r389f/Znf1aqsvc5mzdvzr//+79n8+bN2blzZzZs2JAkOeqoo3LQQQflH//xH9PZ2ZkPfehDqayszAMPPJA//dM/zVe+8pXSFr4PeatrfPrpp2fq1Kn57Gc/m2uuuSYdHR25/PLLc/7555d1GPV2tLe356GHHsrHPvaxHHzwwWlvb88f/dEf5b/+1/9a1iHq22E8NrS+8pWv5BOf+EQOP/zwPP/881m2bFlGjRqVs88+u9Sl7bNeffXVPjNynn766WzYsCGHHXZY3ve+9+Wiiy7KVVddlSlTpmTy5MlZsmRJJkyY0BsG8tbe7BofdthhueKKK/LpT386NTU1eeqpp3LJJZfkqKOOSmNjYwmrHmYKDLqnn366kKTw05/+tNSlMEzcfvvthST9bjBQN954Y+F973tfYfTo0YWZM2cWfvSjH5W6JIaJ3f19dfvtt5e6NIapU045pXDhhReWugx4R9vT58n/9b/+V+Hkk08uVFRUFCZOnFhYvnx5iSreNy1YsKDfa/y9732vUCgUCv/zf/7PQl1dXeGggw4qHHjggYVp06YVVq1aVdi5c2dpC9+HvNU1LhQKhU2bNhXOOOOMwv77718YO3Zs4ctf/nLh9ddfL13R+7h169YV6uvrC1VVVYXKysrCBz7wgcKf/umfFn75y1+WurR9mvHY0Jk3b15h/PjxhdGjRxcmTpxYmDdvXuHJJ58sdVn7tO9973v9/t27YMGCQqFQKPT09BSWLFlSqK6uLlRUVBROO+20wsaNG0tb9D7mza7xa6+9Vjj99NML73nPewrvete7Cocffnjh3HPPLXR0dJS67GFlRKFQKAxhDgMAAAAAAPCOZZEDAAAAAACgbAlKAAAAAACAsiUoAQAAAAAAypagBAAAAAAAKFuCEgAAAAAAoGwJSgAAAAAAgLIlKAEAAAAAAMqWoAQAAAAAAChbghIAAAAAAKBsCUoAAAAAAICyJSgBAAAAAADK1v8HQggAxzzOh4cAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.randn(1000, 10)\n",
    "w = torch.randn(10, 200) / 10 ** 0.5\n",
    "y = x @ w\n",
    "print(x.mean(), x.std())\n",
    "print(y.mean(), y.std())\n",
    "plt.figure(figsize=(20, 5))\n",
    "plt.subplot(121)\n",
    "plt.hist(x.view(-1).tolist(),50, density=True)\n",
    "plt.subplot(122)\n",
    "plt.hist(y.view(-1).tolist(),50, density=True)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:46.088867Z",
     "start_time": "2023-10-28T11:38:45.396579Z"
    }
   },
   "id": "a9ba00ca1e0fc602"
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0011,\n        0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011,\n        0.0011, 0.0011, 0.0011, 0.0012, 0.0012, 0.0012, 0.0012, 0.0012, 0.0012,\n        0.0012, 0.0012, 0.0012, 0.0012, 0.0012, 0.0012, 0.0013, 0.0013, 0.0013,\n        0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0013, 0.0014,\n        0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014, 0.0014,\n        0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015, 0.0015,\n        0.0015, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016, 0.0016,\n        0.0016, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017, 0.0017,\n        0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0018, 0.0019,\n        0.0019, 0.0019, 0.0019, 0.0019, 0.0019, 0.0019, 0.0019, 0.0020, 0.0020,\n        0.0020, 0.0020, 0.0020, 0.0020, 0.0020, 0.0021, 0.0021, 0.0021, 0.0021,\n        0.0021, 0.0021, 0.0021, 0.0022, 0.0022, 0.0022, 0.0022, 0.0022, 0.0022,\n        0.0022, 0.0023, 0.0023, 0.0023, 0.0023, 0.0023, 0.0023, 0.0024, 0.0024,\n        0.0024, 0.0024, 0.0024, 0.0024, 0.0025, 0.0025, 0.0025, 0.0025, 0.0025,\n        0.0025, 0.0026, 0.0026, 0.0026, 0.0026, 0.0026, 0.0027, 0.0027, 0.0027,\n        0.0027, 0.0027, 0.0027, 0.0028, 0.0028, 0.0028, 0.0028, 0.0028, 0.0029,\n        0.0029, 0.0029, 0.0029, 0.0029, 0.0030, 0.0030, 0.0030, 0.0030, 0.0030,\n        0.0031, 0.0031, 0.0031, 0.0031, 0.0032, 0.0032, 0.0032, 0.0032, 0.0032,\n        0.0033, 0.0033, 0.0033, 0.0033, 0.0034, 0.0034, 0.0034, 0.0034, 0.0034,\n        0.0035, 0.0035, 0.0035, 0.0035, 0.0036, 0.0036, 0.0036, 0.0036, 0.0037,\n        0.0037, 0.0037, 0.0037, 0.0038, 0.0038, 0.0038, 0.0039, 0.0039, 0.0039,\n        0.0039, 0.0040, 0.0040, 0.0040, 0.0040, 0.0041, 0.0041, 0.0041, 0.0042,\n        0.0042, 0.0042, 0.0042, 0.0043, 0.0043, 0.0043, 0.0044, 0.0044, 0.0044,\n        0.0045, 0.0045, 0.0045, 0.0045, 0.0046, 0.0046, 0.0046, 0.0047, 0.0047,\n        0.0047, 0.0048, 0.0048, 0.0048, 0.0049, 0.0049, 0.0049, 0.0050, 0.0050,\n        0.0050, 0.0051, 0.0051, 0.0051, 0.0052, 0.0052, 0.0053, 0.0053, 0.0053,\n        0.0054, 0.0054, 0.0054, 0.0055, 0.0055, 0.0056, 0.0056, 0.0056, 0.0057,\n        0.0057, 0.0058, 0.0058, 0.0058, 0.0059, 0.0059, 0.0060, 0.0060, 0.0060,\n        0.0061, 0.0061, 0.0062, 0.0062, 0.0062, 0.0063, 0.0063, 0.0064, 0.0064,\n        0.0065, 0.0065, 0.0066, 0.0066, 0.0067, 0.0067, 0.0067, 0.0068, 0.0068,\n        0.0069, 0.0069, 0.0070, 0.0070, 0.0071, 0.0071, 0.0072, 0.0072, 0.0073,\n        0.0073, 0.0074, 0.0074, 0.0075, 0.0075, 0.0076, 0.0076, 0.0077, 0.0077,\n        0.0078, 0.0079, 0.0079, 0.0080, 0.0080, 0.0081, 0.0081, 0.0082, 0.0082,\n        0.0083, 0.0084, 0.0084, 0.0085, 0.0085, 0.0086, 0.0086, 0.0087, 0.0088,\n        0.0088, 0.0089, 0.0090, 0.0090, 0.0091, 0.0091, 0.0092, 0.0093, 0.0093,\n        0.0094, 0.0095, 0.0095, 0.0096, 0.0097, 0.0097, 0.0098, 0.0099, 0.0099,\n        0.0100, 0.0101, 0.0101, 0.0102, 0.0103, 0.0104, 0.0104, 0.0105, 0.0106,\n        0.0106, 0.0107, 0.0108, 0.0109, 0.0109, 0.0110, 0.0111, 0.0112, 0.0112,\n        0.0113, 0.0114, 0.0115, 0.0116, 0.0116, 0.0117, 0.0118, 0.0119, 0.0120,\n        0.0121, 0.0121, 0.0122, 0.0123, 0.0124, 0.0125, 0.0126, 0.0127, 0.0127,\n        0.0128, 0.0129, 0.0130, 0.0131, 0.0132, 0.0133, 0.0134, 0.0135, 0.0136,\n        0.0137, 0.0137, 0.0138, 0.0139, 0.0140, 0.0141, 0.0142, 0.0143, 0.0144,\n        0.0145, 0.0146, 0.0147, 0.0148, 0.0149, 0.0150, 0.0151, 0.0152, 0.0154,\n        0.0155, 0.0156, 0.0157, 0.0158, 0.0159, 0.0160, 0.0161, 0.0162, 0.0163,\n        0.0165, 0.0166, 0.0167, 0.0168, 0.0169, 0.0170, 0.0171, 0.0173, 0.0174,\n        0.0175, 0.0176, 0.0178, 0.0179, 0.0180, 0.0181, 0.0182, 0.0184, 0.0185,\n        0.0186, 0.0188, 0.0189, 0.0190, 0.0192, 0.0193, 0.0194, 0.0196, 0.0197,\n        0.0198, 0.0200, 0.0201, 0.0202, 0.0204, 0.0205, 0.0207, 0.0208, 0.0210,\n        0.0211, 0.0212, 0.0214, 0.0215, 0.0217, 0.0218, 0.0220, 0.0221, 0.0223,\n        0.0225, 0.0226, 0.0228, 0.0229, 0.0231, 0.0232, 0.0234, 0.0236, 0.0237,\n        0.0239, 0.0241, 0.0242, 0.0244, 0.0246, 0.0247, 0.0249, 0.0251, 0.0253,\n        0.0254, 0.0256, 0.0258, 0.0260, 0.0261, 0.0263, 0.0265, 0.0267, 0.0269,\n        0.0271, 0.0273, 0.0274, 0.0276, 0.0278, 0.0280, 0.0282, 0.0284, 0.0286,\n        0.0288, 0.0290, 0.0292, 0.0294, 0.0296, 0.0298, 0.0300, 0.0302, 0.0304,\n        0.0307, 0.0309, 0.0311, 0.0313, 0.0315, 0.0317, 0.0320, 0.0322, 0.0324,\n        0.0326, 0.0328, 0.0331, 0.0333, 0.0335, 0.0338, 0.0340, 0.0342, 0.0345,\n        0.0347, 0.0350, 0.0352, 0.0354, 0.0357, 0.0359, 0.0362, 0.0364, 0.0367,\n        0.0369, 0.0372, 0.0375, 0.0377, 0.0380, 0.0382, 0.0385, 0.0388, 0.0390,\n        0.0393, 0.0396, 0.0399, 0.0401, 0.0404, 0.0407, 0.0410, 0.0413, 0.0416,\n        0.0418, 0.0421, 0.0424, 0.0427, 0.0430, 0.0433, 0.0436, 0.0439, 0.0442,\n        0.0445, 0.0448, 0.0451, 0.0455, 0.0458, 0.0461, 0.0464, 0.0467, 0.0471,\n        0.0474, 0.0477, 0.0480, 0.0484, 0.0487, 0.0491, 0.0494, 0.0497, 0.0501,\n        0.0504, 0.0508, 0.0511, 0.0515, 0.0518, 0.0522, 0.0526, 0.0529, 0.0533,\n        0.0537, 0.0540, 0.0544, 0.0548, 0.0552, 0.0556, 0.0559, 0.0563, 0.0567,\n        0.0571, 0.0575, 0.0579, 0.0583, 0.0587, 0.0591, 0.0595, 0.0599, 0.0604,\n        0.0608, 0.0612, 0.0616, 0.0621, 0.0625, 0.0629, 0.0634, 0.0638, 0.0642,\n        0.0647, 0.0651, 0.0656, 0.0660, 0.0665, 0.0670, 0.0674, 0.0679, 0.0684,\n        0.0688, 0.0693, 0.0698, 0.0703, 0.0708, 0.0713, 0.0718, 0.0723, 0.0728,\n        0.0733, 0.0738, 0.0743, 0.0748, 0.0753, 0.0758, 0.0764, 0.0769, 0.0774,\n        0.0780, 0.0785, 0.0790, 0.0796, 0.0802, 0.0807, 0.0813, 0.0818, 0.0824,\n        0.0830, 0.0835, 0.0841, 0.0847, 0.0853, 0.0859, 0.0865, 0.0871, 0.0877,\n        0.0883, 0.0889, 0.0895, 0.0901, 0.0908, 0.0914, 0.0920, 0.0927, 0.0933,\n        0.0940, 0.0946, 0.0953, 0.0959, 0.0966, 0.0973, 0.0979, 0.0986, 0.0993,\n        0.1000, 0.1007, 0.1014, 0.1021, 0.1028, 0.1035, 0.1042, 0.1050, 0.1057,\n        0.1064, 0.1072, 0.1079, 0.1087, 0.1094, 0.1102, 0.1109, 0.1117, 0.1125,\n        0.1133, 0.1140, 0.1148, 0.1156, 0.1164, 0.1172, 0.1181, 0.1189, 0.1197,\n        0.1205, 0.1214, 0.1222, 0.1231, 0.1239, 0.1248, 0.1256, 0.1265, 0.1274,\n        0.1283, 0.1292, 0.1301, 0.1310, 0.1319, 0.1328, 0.1337, 0.1346, 0.1356,\n        0.1365, 0.1374, 0.1384, 0.1394, 0.1403, 0.1413, 0.1423, 0.1433, 0.1443,\n        0.1453, 0.1463, 0.1473, 0.1483, 0.1493, 0.1504, 0.1514, 0.1525, 0.1535,\n        0.1546, 0.1557, 0.1567, 0.1578, 0.1589, 0.1600, 0.1611, 0.1623, 0.1634,\n        0.1645, 0.1657, 0.1668, 0.1680, 0.1691, 0.1703, 0.1715, 0.1727, 0.1739,\n        0.1751, 0.1763, 0.1775, 0.1788, 0.1800, 0.1812, 0.1825, 0.1838, 0.1850,\n        0.1863, 0.1876, 0.1889, 0.1902, 0.1916, 0.1929, 0.1942, 0.1956, 0.1969,\n        0.1983, 0.1997, 0.2010, 0.2024, 0.2038, 0.2053, 0.2067, 0.2081, 0.2096,\n        0.2110, 0.2125, 0.2140, 0.2154, 0.2169, 0.2184, 0.2200, 0.2215, 0.2230,\n        0.2246, 0.2261, 0.2277, 0.2293, 0.2309, 0.2325, 0.2341, 0.2357, 0.2373,\n        0.2390, 0.2406, 0.2423, 0.2440, 0.2457, 0.2474, 0.2491, 0.2508, 0.2526,\n        0.2543, 0.2561, 0.2579, 0.2597, 0.2615, 0.2633, 0.2651, 0.2669, 0.2688,\n        0.2707, 0.2725, 0.2744, 0.2763, 0.2783, 0.2802, 0.2821, 0.2841, 0.2861,\n        0.2880, 0.2900, 0.2921, 0.2941, 0.2961, 0.2982, 0.3002, 0.3023, 0.3044,\n        0.3065, 0.3087, 0.3108, 0.3130, 0.3151, 0.3173, 0.3195, 0.3217, 0.3240,\n        0.3262, 0.3285, 0.3308, 0.3331, 0.3354, 0.3377, 0.3400, 0.3424, 0.3448,\n        0.3472, 0.3496, 0.3520, 0.3544, 0.3569, 0.3594, 0.3619, 0.3644, 0.3669,\n        0.3695, 0.3720, 0.3746, 0.3772, 0.3798, 0.3825, 0.3851, 0.3878, 0.3905,\n        0.3932, 0.3959, 0.3987, 0.4014, 0.4042, 0.4070, 0.4098, 0.4127, 0.4155,\n        0.4184, 0.4213, 0.4243, 0.4272, 0.4302, 0.4331, 0.4362, 0.4392, 0.4422,\n        0.4453, 0.4484, 0.4515, 0.4546, 0.4578, 0.4610, 0.4642, 0.4674, 0.4706,\n        0.4739, 0.4772, 0.4805, 0.4838, 0.4872, 0.4906, 0.4940, 0.4974, 0.5008,\n        0.5043, 0.5078, 0.5113, 0.5149, 0.5185, 0.5221, 0.5257, 0.5293, 0.5330,\n        0.5367, 0.5404, 0.5442, 0.5479, 0.5517, 0.5556, 0.5594, 0.5633, 0.5672,\n        0.5712, 0.5751, 0.5791, 0.5831, 0.5872, 0.5913, 0.5954, 0.5995, 0.6036,\n        0.6078, 0.6120, 0.6163, 0.6206, 0.6249, 0.6292, 0.6336, 0.6380, 0.6424,\n        0.6469, 0.6513, 0.6559, 0.6604, 0.6650, 0.6696, 0.6743, 0.6789, 0.6837,\n        0.6884, 0.6932, 0.6980, 0.7028, 0.7077, 0.7126, 0.7176, 0.7225, 0.7275,\n        0.7326, 0.7377, 0.7428, 0.7480, 0.7531, 0.7584, 0.7636, 0.7689, 0.7743,\n        0.7796, 0.7850, 0.7905, 0.7960, 0.8015, 0.8071, 0.8127, 0.8183, 0.8240,\n        0.8297, 0.8355, 0.8412, 0.8471, 0.8530, 0.8589, 0.8648, 0.8708, 0.8769,\n        0.8830, 0.8891, 0.8953, 0.9015, 0.9077, 0.9140, 0.9204, 0.9268, 0.9332,\n        0.9397, 0.9462, 0.9528, 0.9594, 0.9660, 0.9727, 0.9795, 0.9863, 0.9931,\n        1.0000])"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lre = torch.linspace(-3, 0, 1000)\n",
    "lrs = 10 ** lre\n",
    "lrs"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:46.100365Z",
     "start_time": "2023-10-28T11:38:46.089956Z"
    }
   },
   "id": "ef2d17798f384a65"
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 3.300393581390381\n",
      "1000 2.818566083908081\n",
      "2000 2.7130813598632812\n",
      "3000 2.7395453453063965\n",
      "4000 2.2398953437805176\n",
      "5000 2.1572048664093018\n",
      "6000 2.6469104290008545\n",
      "7000 2.195574998855591\n",
      "8000 1.9594740867614746\n",
      "9000 2.1209499835968018\n",
      "10000 2.3477437496185303\n",
      "11000 2.041003942489624\n",
      "12000 2.448354959487915\n",
      "13000 1.7924888134002686\n",
      "14000 2.2293219566345215\n",
      "15000 2.044398069381714\n",
      "16000 2.467888832092285\n",
      "17000 2.3104867935180664\n",
      "18000 1.9645344018936157\n",
      "19000 1.880371332168579\n"
     ]
    }
   ],
   "source": [
    "lri = []\n",
    "lossi = []\n",
    "stepi = []\n",
    "max_steps = 20000\n",
    "\n",
    "for i in range(max_steps):\n",
    "\n",
    "    # minibatch construct\n",
    "    ix = torch.randint(0, Xtrain.shape[0], (32,), generator=g)\n",
    "    Xb, Yb = Xtrain[ix], Ytrain[ix]\n",
    "\n",
    "    # forward pass\n",
    "    emb = C[Xb]\n",
    "    hpreact = emb.view(emb.shape[0], -1) @ W1 \n",
    "    bnmeani = hpreact.mean(0, keepdim=True)\n",
    "    bnstdi = hpreact.mean(0, keepdim=True)  \n",
    "    hpreact = bngain * (hpreact - bnmeani) / bnstdi + bnbias\n",
    "    h = torch.tanh(hpreact)\n",
    "    logits = h @ W2 + b2\n",
    "    loss = torch.nn.functional.cross_entropy(logits, Yb)\n",
    "    # print(loss.item())\n",
    "    with torch.no_grad():\n",
    "        bnmean_running = 0.999 * bnmean_running + 0.001 * bnmeani\n",
    "        bnstd_running = 0.999 * bnstd_running + 0.001 * bnstdi\n",
    "\n",
    "    # backward pass\n",
    "    for p in params:\n",
    "        p.grad = None\n",
    "    loss.retain_grad()\n",
    "    loss.backward()\n",
    "    # update\n",
    "    lr = 0.1 if i < 10000 else 0.01 \n",
    "    for p in params:\n",
    "        p.data += -lr * p.grad\n",
    "    # lri.append(lr)\n",
    "    if i % 1000 == 0:\n",
    "        print(i, loss.item())\n",
    "    lossi.append(loss.item())\n",
    "    stepi.append(i)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-29T13:10:48.854272Z",
     "start_time": "2023-10-29T13:09:16.130168Z"
    }
   },
   "id": "74f8ba991580ae2b"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "plt.hist(h.view(-1).tolist(), 50)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.262569Z"
    }
   },
   "id": "ab91cf497d8898f"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "plt.hist(hpreact.view(-1).tolist(), 50)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-28T11:38:46.269547Z",
     "start_time": "2023-10-28T11:38:46.265067Z"
    }
   },
   "id": "862a86d583216b7f"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20, 10))\n",
    "plt.imshow(h.abs() > 0.99, cmap='gray', interpolation='nearest')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.267726Z"
    }
   },
   "id": "8cb7d4ae7449c42c"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print(loss.item())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.270912Z"
    }
   },
   "id": "118f88854d7a108d"
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "outputs": [
    {
     "data": {
      "text/plain": "[<matplotlib.lines.Line2D at 0x7fe0504be970>]"
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGiCAYAAAA1LsZRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXUElEQVR4nO3dd3gU1foH8O+GNAIpQEgBQodQg9QQkCIdEcGKiIKKKF644k9F5FpQ8ArXjg0LIgoiggUsEKSFXgMBQokQSigptBQC6ef3R8iym2yb3dk9W76f58mj7E45s7M7884p79EIIQSIiIiIJPGSXQAiIiLybAxGiIiISCoGI0RERCQVgxEiIiKSisEIERERScVghIiIiKRiMEJERERSMRghIiIiqRiMEBERkVQMRoiIiEgqRcHIvHnzEBMTg6CgIAQFBSEuLg6rV682uvzChQuh0Wj0/vz9/W0uNBEREbkPbyULN2jQAHPmzEGLFi0ghMB3332HESNGYP/+/Wjbtq3BdYKCgpCSkqL9t0ajsa3ERERE5FYUBSPDhw/X+/d///tfzJs3Dzt37jQajGg0GkRERFhfQiIiInJrioIRXaWlpVi+fDny8/MRFxdndLlr166hUaNGKCsrQ6dOnfD2228bDVwqFBYWorCwUPvvsrIyXLlyBXXq1GHNChERkYsQQiAvLw/16tWDl5eJniFCoYMHD4oaNWqIatWqieDgYPHXX38ZXXb79u3iu+++E/v37xcJCQnirrvuEkFBQeLs2bMm9zFjxgwBgH/84x//+Mc//rnBn7n7vkYIIaBAUVER0tLSkJOTg59//hnz58/Hpk2b0KZNG7PrFhcXo3Xr1hg9ejRmzZpldLnKNSM5OTlo2LAhzp49i6CgICXFJSIiIklyc3MRFRWF7OxsBAcHG11OcTONr68vmjdvDgDo3Lkz9uzZg7lz5+LLL780u66Pjw86duyIEydOmFzOz88Pfn5+VV6vGMVDRERErsNcFwub84yUlZXp1WKYUlpaikOHDiEyMtLW3RIREZGbUFQzMn36dAwdOhQNGzZEXl4elixZgoSEBKxZswYAMHbsWNSvXx+zZ88GAMycORPdu3dH8+bNkZ2djXfffRdnzpzBk08+qf6REBERkUtSFIxkZWVh7NixSE9PR3BwMGJiYrBmzRoMHDgQAJCWlqbXW/bq1auYMGECMjIyUKtWLXTu3Bnbt2+3qH8JEREReQbFHVhlyM3NRXBwMHJycthnhIiIyEVYev/m3DREREQkFYMRIiIikorBCBEREUnFYISIiIikYjBCREREUjEYISIiIqkYjBAREZFUDEaIiIhIKo8ORi7mFeKLTam4dM2yuXWIiIhIfYpn7XUnT363BwfO5WDtkUz88kwP2cUhIiLySB5dM3LgXA4AIPHMVcklISIi8lweHYwQERGRfAxGiIiISCoGI0RERCQVgxEiIiKSisEIERERScVghIiIiKRiMEJERERSMRghIiIiqRiMuIij6bn4z2+HkJlbILsoREREqvLodPCuZOjcLQCAUxfz8eNT3SWXhoiISD2sGXExRzNyZReBiIhIVQxGiIiISCoGI0RERCQVgxEiIiKSisEIERERScVgRIHrRSV44Ivt+HJTquyiOMyZy/lIvXhNdjFIJUII2UUgIqqCwYgCS3alYc/pq5i9+pjsojhEaZlAn3cT0P/9TcgvLJFdHLJRVl4BeszZgA/+TpFdFCIiPQxGFLhRVCq7CA5VXFqm/f8r+UUSS0Jq+HxjKtJzCvDxhhOyi0JEpIfBCBEREUnFYISIiIikYjCiALv+ERERqY/BCBEREUnFYEQBjookIiJSH4MRIiIikorByE1CCFy6Vmh6GfYaISIiUh2DkZum/nwQXd5ah3VHMmUXhYiIyKMwGLnp58RzAICPNxyXXBLTZPVbYX8ZIiKyFwYjNthwLBMzViajqKTM/MJERERkkLfsAriSyrUDTyzcCwBoVKcGnri9iYQSERERuT7WjChgrKUiI7fAoeUgIiJyJwxGKmHfCCIiIsdiMEJERERSMRip5ETWNWxPvWT4TVabEBERqY7BSCU3ikvx8Ne7cPLiNdlFISIi8ggMRow4kcVgxBVk5BRg0Y7TyC8skV0UIiKyEof2KmCskUbj0FLIoXHSg7zn821IzynA4Qu5mHNfjOziEBGRFVgzogC7jDif9JzyYdUbU7Ikl4SIiKzFYEQFjoxRZNVQMBAjIiJ7YTBCdlVcylT5ROTcbhSVIiuPyStlYjBCdvPKb4fQ5vV4nLt6XXZR9GRfL8LuU1cgWN1DRAC6z16Pbv9dj/PZN2QXxWMxGFFAOLRBBriSX4SNx7JQWuaaN80fdqWhuFRg/pZTsouip//7m/DglzuwOjlDdlGIyAnk3CgGAOxIvSy5JJ6LwYgCjn6QHv7JVjy+cA8W7Tjt2B27ucv5RQCAtUcyJZeEiIgABiOqsFef0ooqw/jDfIIn2znr8GwiIgYjLobdHAzTeES2FyIi98RgxAhD93zGAUREROpjMOIhLl8rxJSl+9lBi4jICNavysNgxIiC4lKkZORZtKwr1Ji8+ccRrEy6gNFf75RdFCIiIj0MRoyYsjQJgz/ajDVu0nn0rI25Phw9rJmIiDwHgxEzfkk8p/1/dh4lIiJSH4MRBYzVDrCdkYiIyHoMRsxYezTTZTOgehLm0CAiW/E6Ig+DETOEAH7YdcbkMoUlnAzOFM4BQ0REpjAYscD6o1nl/2Pknrpw+2nEJ6fbbf8lpfJv5kwqRkTujs9N8jAYUcmzPybZbdt7z1y127ZdhRAC8cnpOHM5X3ZRiIhIZd6yC+BKGDSbl19YgpIygeDqPqpud83hTExcvA8AcHrOMFW37SlYu0VkGvuMyMNgxEM44jcmhEDbGWsAAEdnDlF123tPXzGzb1V3R0REDsRmGgXYEdM03Y/nnI1J1jzJ9hOXmKafiDwagxGVFJWW2bUTq2y2ZGBVO4Q7lpFb5TVXrV7NLyzBw/N3YfTXO1FQXCq7OERuqbi0DCuTziMzt0B2UcgIBiNWMpR7pKJPA9nXlfwi2UVQTX5hifb/C4s5RJzIHr7afFI7xQc5J0XByLx58xATE4OgoCAEBQUhLi4Oq1evNrnO8uXL0apVK/j7+6N9+/ZYtWqVTQV2Ft9tP23zNg5fyMFzS/fj7BXLmzR0m4ryCopR5IE5Th7+ehebNazgqrVHRLbacKw8PUP29WKTy/E3Io+iYKRBgwaYM2cOEhMTsXfvXvTr1w8jRozA4cOHDS6/fft2jB49GuPHj8f+/fsxcuRIjBw5EsnJyaoU3lGSz+cA0O8TMfPPIwaXVdKvZNjHW7Ei6QImLk5UXKbcgmK0f+Nv9PzfBsXr7kuzbKiwM/8wOfuwaztyIRf9309w66ZNIrKcomBk+PDhuPPOO9GiRQu0bNkS//3vf1GzZk3s3Gn4xjB37lwMGTIEU6dORevWrTFr1ix06tQJn376qSqFd5TLN5sFLAkzTmRdU7x9a9Y5cDYbAHAxr1Dxuvd+vh2Xrylfj0gtz/yQiNSL+WzaJCIANvQZKS0txdKlS5Gfn4+4uDiDy+zYsQMDBgzQe23w4MHYsWOHyW0XFhYiNzdX789VlLrIiJvMXPWDEWNH7iIfiVtIu3wdJaXO33R3vYiddYnoFsXByKFDh1CzZk34+flh4sSJ+O2339CmTRuDy2ZkZCA8PFzvtfDwcGRkZJjcx+zZsxEcHKz9i4qKUlpMp5aZW4CVSedV3aazDTtWu4nHHk1GzvCZqVmC1YfS0fvdjZjw/V4Vt0rkOZgYUB7FwUh0dDSSkpKwa9cuPPPMMxg3bhyOHDHcf8Ja06dPR05Ojvbv7Nmzqm7fWmrdu4bO3YIpS5OsWldj4K687cQldHt7PdYdyVS0HjkZG0/Rgm2nAAAbUy6qUBgiz2NLCgOyjeJgxNfXF82bN0fnzp0xe/ZsdOjQAXPnzjW4bEREBDIz9W+QmZmZiIiIMLkPPz8/7Yidij93UnloqgCQc70YxVZWr4+ZvwsX8wrxpIInYkt+dE5QcaBlrizW3MetCdAKiktRZmBYNxERWc/mPCNlZWUoLDTc/yAuLg7r16/Xe23t2rVG+5h4qqKSMnSY+TeGzt3i0P2evpSPR7/ZhTWHM5yiycLZXb5WiFavxeMhFx3Jw7oxsrfcgmJ8t/00svKYXIyUURSMTJ8+HZs3b8bp06dx6NAhTJ8+HQkJCRgzZgwAYOzYsZg+fbp2+SlTpiA+Ph7vv/8+jh07hjfeeAN79+7F5MmT1T0KB7GkNsGWNkfdUTXmsnGqETv8+8f92HL8Ep5elIhnbo5q2J56CQfPZdtlf67u75vNYLtPmZ4nBwC+3nwSwz7eguzr7pOgjcicaT8fxIzfD2PM17tkF4VcjKJgJCsrC2PHjkV0dDT69++PPXv2YM2aNRg4cCAAIC0tDenpt/IG9OjRA0uWLMFXX32FDh064Oeff8aKFSvQrl07dY/CQRx5QzaU4dUSSmo4MnRSI8cfzkBWXgEe/noX7v50m8HlC4pLsfpQOvIKTCcOqlImJ2iH3XbiEmb+ccRhKdf/u+ooDl/IxRebTppcrvLpuqaTkZXI1ay9GbAftyJdgTNgB1Z5FM3a+80335h8PyEhocprDzzwAB544AFFhSLrjV2wG4vGx5pdzlDMkpljerjvayuSsTzxHG5vHorFT1bdhzM39YyZX/6kFhro69D9KsmQ+/fhDEz9+SCm9G+B/xvYUrUy5BUU4/CFXOjGtwXFpfD3qabaPoiIbOHRc9N0aBCs+jZl1wJsOX7JLtvVaIDliecAAFtPWLIP53zCeCc+xeH7PHQuByM+24ZdJ02nsH9lRXlm4rnrj6uyXyEEysoEHvhiBx76aid+2HVG+16/9xJU2QcRkRo8OhiJjgiUXQSXlXO92GDfEnux98hkIQQOncuxSzPOw1/vxIGz2Rj1lX07vlaumHp84R4M/mgzjmXkAQAKdWppLuQ4TwfDrzebbsoiIvfn0cGIPVoVNNDgWEYuvtqc6taT2PV9byPu/nQbthx3jpwWtuZR+WnPWQz/dCvGfrNbpRKV02iAPEn9QBJSLjpt273ub++/q47KKwgROQWPDkbslS5iyEdb8PaqY5i/1Xme+Crfqjcey7IpX8bVm7Nfmkq05kqW7E4DAOw+bXykjL0bnpyzYYvIcZLP5+DLTalW51yqcKOoFDP/OGLRyDddzA0pj0cHI0o7XCpdvmK2X3uwNY56f+0/2gkAb23Tvv1dhBD4evNJbLw5nTfJ72NE5Ezu+mQrZq8+hkU7zphf2ITPE05gwbZTePBL0/Og2cPfhzOweKdt5fdEikbTuJsyJx79oXaE7gxHuuvUFW2V/Ok5wwCUjzYZ/fVO3BYVgtfuMjzHEQAUlzrDEdgXn8qIyh1Jt21y1JOX8lUqiXJPLUoEAMQ2qY0W4eyXaCnPrhlRsOznCSec4oZuz+Gzto6xN1eyDAOdJjccy0Timav4ZuupKu+dvXIdQgjEJ6dj4fbTpvftxIGlpZwpx8GF7BuYv+UkchXmlLEUAy/3xPN6y6VrTHiohIfXjFi+7DvxKajmZf6X5gw/xgvZN1AvpLqq2zR2rzf2EVr6ORir8Vi04zReW3kYD8c2xJJdaZZtzANdLypBVm4hGofWUHW7983bjvScAhw8l4OPR3dUddtEjuYODyvuzqNrRpQ201ibFdXReszZgGV7ymc63ngsC2evXHfIfpV2FgNgtKPau2vK84F4ciBSWGJ+mHGfdxPQ970EHDqnbv+k9Ju1WJudZLQU2Z+9asGILOHRwYi9212sCcYNNVdY472/U7D5n4t4fOEe9HpnoyrbNKciy2llxj6HV1ccwvPLDqiyb1uH9hpzND0XExcl4nhmnlW1XuZW0f1sdLe/ZFcaol+Nx8qk8ybXv5hXnjV37dFMp2hGtNbJi845BNlT/Lb/HGLe+Btz191KuOfIh6+r+UVuV3th7rdL+jw6GLFHB1ZbNznrzyMm388tsDxnxd4zV60uR0pGHv7vpySr19+pk2308IVcbP5H/wl7X9pVLN5pvNZDaXBxPvuGXSalu2/edsQfzsDDRgIte/nPb4cAAFOWJjl0v7JUdPojOV7+pfz79uG6fwAAc9cdR8wba/Qm77SXTf9cRMdZazHtl4N235cjLd1zFheyb8guhstgMOKEPtt4wuT7N4pKFT9FKH2ov+fzbVidnKFwrVs+35iq/f+ks9kYu2A3zly+1Vx07+fbrd62MbfNXIulu9Vt1rleVN5UcjGvkDMXK6D0+3n+Ki/azuTDdf8gv6gUc1Yfs/u+ProZAC3be87u+3K07Oumm75SMvIw/JOtTHcADw9GbMyrY5Bu3ogLOQX4eP1xXL5megK6yt5dk2JyZEVWnv1TeVfchM1Rcs9JU9B3xdpWl5d/PWTVek7Q79ht/LrvHDrNWotEG2rmiDzB04v24tD5HDy+cI/q2y4rE9ieegk5N1yjL5BHByP2aKMc8tEW7f8fOJuND9b+g2eX7te+VmKPCMgYiY/yzjCqqIIlHUHtpfLnkHw+B1fzDTcnOdFHpmfzPxdxz+fbcDwzz6Lln192AFevF+NpNr2Qk7BXnzJbZdsxUPhxTxoe/noX7vlsm932oSaPDkZKHXSz3nbiVv+Jt1fZv9rTkMQ0xz6l2vrbt/XScSIrD/d+vg0zViYj+tV4o8ttTMnCsz/ut6gvjhrXs7s+2Yrus9cbfK9A4lxGhSWlSM8x3FQydsFu7E/LxtOLy4MLy6cRML6cqzV5rdh/Hq/8dsglRtQdTc9Fj9nr8XOi+zV7uDO15zL748AFAHITwCnh2cGIhAvLgm2WjZZRO1BytYu/rSYu3od9adn4zkRaaSEEHv92D34/cAGn7PSD/XpL1fNdaOSiY9P30cYTfNfHWxE3e4PJKQyyrxcjI6cAHWetxVtmOlrbmxACU5cfwCfrj5tfWAXP/ZSEH3al4c+DFxyyP1s8tzQJF3IK8OJydUaqKeHIxH1/HLiAt/48oniOraPpuThz2fLfe1mZwK/7ztntGlFhW+olu27f2Xl0MOLMN+iFFgYtxjhpraTFbK1WvWKkKcQWjmxhcwTdz6hidt8/zNxs5yWcQM6NYsxXaQh6Bd2+Vlm5BUg8YzpnTdLZbCxPPIf31/6jajnMMdbE5kxsnWTO0az9pf/7x/2Yv/UU/lY4WefQuVvQ590Ew2UxcN35Zd85PL/sAO54z/A6pA6PDkacucr1z4PpDt+nvSdtc1SA9IuN1dOXjHQ4rhhuCwB5NiaIKi4twxu/H8aaw9aPWLLVayuSAThfsqtub6/HffN2mOwAe6NYXj8gci6X85UNEFDKYR2xnfd25BCeHYw4c9WIGY4u+Xt/pxgph+GSWPvR/n04Aw9/vdOmmo0Xlh+wqQf5pxtMD60GgJ5zNli9fQBo8cpqLNx+Gm/+Ia+541hG+WRkf6kY+Kp54dbNVUOexZkrdgtUDISd+TgdzaODEUdm/FNaC2OqmcLSdlmlR2dqu2rU1FjycT+1KBHbU22/CdlS62VJNbeS5HMyyMr+eN889fPHkEyu+8BmDUP3BN2X1h3JRKvX4vF5gvkHFlLGo4ORTo1qOWxfFYl9yHk463A/q1Q6FntkblUzeHenj56sd+hcDvalZcsuhsVeupkl9p14wzXFtrB3M7mz8+hgJNDPcZMWL9x22mH7qqD0eq/mj8EdbjZKktVdzCtEv/cSMC8h1fzC9mBFoJB6MR8DP9iE/ELjtTwu3JLpcMnnc9BzzgbOSaLA6K93yi6CHrd6QHExHh2MOPJCm1dYggNnsx22v+JSgeQLuYrWGbdgt+L9rFXYk92U9430S5FBCIHOb62zePlPNhzHyUv5+F+8nDwy1jqedQ1Lb87wDJhuqrtqJrV1ZcWljo1kDpzNRo/Z66UNv520ZB/OZ99wyfmEZMWc10wEwpay93Bie9ZYMPi5xbODEQfvb4QDM+FdyS/CBoXzHSi92QBAZm4hRny2TZWRSZ9Y0HFU11kF6eWVsOb6oHvjlZEG/TMbamR08zSoeeF1dBrqCd/vxYWcAkxest/8wlYy9emokbRKCIEpS/fjpZ9tyBGi4Pu79/QVzPzjiOoJt0g5T6+FdFw7BSlyNN14rUZxmXNdOA6cza4yK+/ZK/af+MzWm76awczPibdqF15YlqTadi1VWias/jxsydB47up1BPr5IDjAx+ptWKqguBT+PtWMvl/kYvk1DMnILcDKpPKanTfubosAX/teou//Yoddty+Nh9/YXZFn14y46Bf2i4RUJJ8znilTBmNZRZ3ZXpVqMJLP5+jVjJy+bJ8aG2czack+3P6/jegw82+77yv5fA5avRavzY2ipGr+4LlsbD/hGtktnTn3kSzmhtJ6esdPd+HRwYirkpF50tO8/7fln++FbPvXAjkj3fwklk6iV6Hyg4C5B4OP1pWnfV+003h6f2Pu/nQbHp6/C1m59p/tmixnSX+RE1l5aPVavEXNVkfTc/Hh2n9wvaikSlOVpSPBKlbT7byuu6o9e3i46sOxWhiMEBnw6UbmEQAsz3S6McV4/6SNx7Kw0Uz/pcKSMkxess/ge1fzi7DuqO0dpTNz7Zup0x3Z6wb5+4ELaDdjjdnlvth0EgCwbO85zDSSILCilmzo3C2Yu/44PrTxQW3JrjR0fmsdPrCgQ31WXgFGfLoVP+1Js2pfhoKb3IJiRXPnuAuPDkZYvUeGKO2d707fosrHbmnHRmM3rfzCEjy+cA8eX7gH14tKTE5qZiyx3rhvlY/yclXO/HSceOYq5qw+pkoG0mk/H1S8zoJtp5BmQRPoYYWjCCt7dUX5tA8fW9Ch/p34FBw4l4Npvxwyu6w5AuXNdJ1nrUWfdxP0JuYTQuDlXw7ii03WdVT/8+AFnLg5/5Sz8uhghBzr6nV1Jxnbfdr0ZGrkOMbuoflFt6ritxy/hHZvrMGPu5U9RR5UqX+UjIeP4tIyjFuwG59usG12YaUz06ol7cr18mYPlGfX/WJTqrxcOgCKSqsGQteL7JsN2dQnr+a+fz9wAW1ej9f2P9OdDmFf2lUs3XMWc1ZbnjpA98Fi8pL9GPDBJtXKag8eHYw481OI6zH/YSodamzOHwecfzp3e7L3RdgQY2fZkt/SpB/24XpRKab/avwpsmK+HGel9Jqx6lA6Nv1zEe9Z0AcpK7cA//6x6rDkq/lF6Pb2erzym+1P30odz7pWZYbbExft/4StZHj9W38dRUaO6f5Alqa8kZn2448DF4wOBLhepLw2ytVq/j06GCH1/HXI8bPP5tlpfphUCy+2mbkFePzb3Xj+pyS7lKPCDQMXor8PZ6DN6+bb3JUydwHbctzwqBS1LnxDPtqi/X81bgzn7dC5+NK1QhSWWHZzUNKk8fKvh5BkIDHikt1puHStED/ssq5fgq0u5lXtayOEMNyEp/LN/JCFtWLxyabnzrI04aS5YNORwYqnPSx7dDDiYefarg5fcK6hxrawdKK+11cmY2PKReRb8dSixKs3h7Pq+o+Ep2QAeu3YuoxdOE9kym2nHqOTblyNi/v57Bvo8tY69HtPeZW3ud+IGnlvNqZk4eRF+3d+nPzjfrR5PR5ZedaNULK0Y3SKwlFa9qB2UFBaJrAxJQvZ14vcYtoMtXh0MOJxoacdOeIC6GwcNTrjl33nAAA514vxc+I55BU4NrOpLVJ1gpcSFfs96F7EK2ofDF3Xrcn5sumfi0Y7+1WMCrKmxmXYx1uRnqN8PSUTFD7+7R7F27fGXwfTUVImsHzvObtsX+ml2ZpvVkqG8kBHN6vwO/HHcCVfeT+4b7edwuPf7sG9n5ue4drTAhXPDkaIXMgzPyTixeUHMHW58pEIlrJ2ng+lM/qqeaGd+afhIZ/WSD6fg3ELdqvW2a/yx5KapSxoF0JY1N8k6Ww2Xv7Fft8LZ6Hm8+OUpcqnDdBNbvh5Qip2nlTeif6Pm6PGzGU+tvVY7T1nj9o8OhhhvQi5kormo/jDju+fY47RC6cdah8r1wwt2ZWG/Wn62XQzcwuQU2muJUtKcsTENAwyGGoCvFFUirVHMvX6Eo38bJvehIeexpqvmaH+PIaCZFfrCFrB1crt0cEIkS3M9eD3JI687N03r2r19j2fb9eb6DH27fUOSVNvb4ZqnKb9chATvt+LF5fbMJkeuaz45HTsqxR8V5ZXUGxVrY1MHh2MjOoaJbsI5MIyHJhePLPKvpyvClZJU42hERqW+kdyp1hbqPG0+vvNIe1/HUpHSWmZ9GRWQghsT72ES9cc04equLQMTyy0vG/MjJVVO4ADzlIzbtnvWLfJZeLifWb7m4xb4HqJAj161t4GtQJkF4HIIr8nOSanirW5S4QA8iyYa0TpNtXdnuNvP7bu0dz6zy7dj1VWDqtXa9bqNYczMHGx4VT+9hCfnFElZ1Hlz2mLzizi3+1QPp+RyY07mbIygf/FH0PHhiEY0i4SALAvLVtuoazg0TUjRKTveysv3Mez8hDzhrrNIu+uScGj3+xSdZvuxtpABAC+235alTKonczQHHPDgs9cvo5cK3MQuVqnT6C8D9mXm086NCC0B4+uGSFyVY6qEreUsXllbH2oNJZkzRprDmfi4LkcjOvRWLVtKqV4yGql5W1N4lZQXIpRX+5A18a1rd6GzNu18X7St96x9DM6c/k6ysoEvLyc8YgsH3FWtQnXNbFmhMgFfJ7gmrMI26tlxJrbxxebUjHj98N2SznviFagq1bktdAVn5yBA+dyMH/rKZVK5Hhqhg6/7T+v4tYsZOEXpWIxGc2LMjAYIXIBV6+7TqIzXTN+P2yX7dpyeb587dYNvaS0DJedrJZJV3Gpfsr19yyY1t4UNRPPybAmOQM7TlqWIdkShpqYKn9Ctnxilc9fVaZDqx93p6HjrLU4eD7bhlK4BgYjROSxmr+yGp3fWoetVjQHWTLvTFKljoRKbmxfbDqJLm+t03stIeWikaXdS6mRoOn9tf/g133q1Wb8dcj0nDa2GvHZNqypnBdIQca/6b8eQvb1YrwTb1sQ6goYjBCRy1EyAZ0lHrnZUVZJE0DPORvMLvPTXv1EZOMW7Mb2E5YFPh+vP66gNO7lVwc2n5gaVXQ1vwjHs2ybH+fpRYlVEvCpSbcVp6xMmGzWsfS7JwODESJyOcaenC1Rca1edyRT+co60cplE/03pv180Oi8JQ/PNzxCyBXnIhFCOMUUX7aUIdfEXE8dZ61F8nnb+xh9suE4hBCY/utBi2cQtsbADzfhcRM5WHS/e6Vlwqk6wnM0DRF5pCe/36t8JQtvej/tPYuCEvvO5mwN3admJcHPqC93KN7X+qOZ6NakNgL9fRSv626uFZZg58kr+HG3ein7S0rL4FXpJKZezEeqhZOWjvpyB/aeuYo//3072tUPVq1c1mLNCBHp2XvatdJIO6vTZiZCq8xeNQw3DMxvo9SuU8q/E+O/26soU2plJWY7f7qWfANJAW2pDbv9fxsx8vNtVpdl75nylPLL9zrHnEYMRohIT3yy803EJ9Mu3dEbLtaU8tXmVLR+PR6r7NRRM6+wBKkXjaej33Pa9BwqpkS/Fm/xsrak2a+c6MxVTnFGbgEOnsuxat22M9aoXBrbMRghIpfjyP4Vo77aaZftOiJ/xNurjgEAXlim3qR6ugnuvtx00mzq8UIrm6uU9Auy5aPU/S7Z83tlr7PtBF12VMFghIj0uHJCLEvMXf8PikqsawK4pGSCPxN3tmk/H8TgjzarPipICUd1PL2ab/8cOYfOW1dDAOifJnsN9bXnZ21uBl9zCoqdozmMwQgReZQ9p69ixu+GZ3I1x9ScJ0qeqn/aexb/ZF7TJt3KyivAccmz79rTvrSryMixX9ryzf9Yn39FNwneXwfT4Wp54f4yMhWDpSoPP5eFo2mIyOWUlNp2x7BkVMP1ohK8u8Z8sqmj6bkG5wexJDapeGL+dINj0v3LuM8evpCD8d+Vj1w6PWeYXfZh7cR4ADDGyFBrtRlqlnOV/imOwGCEiFzO+qP2mSlWo1O90eZ1yzr5DZ27xci2LN+vLXlTrOWofje2zCzsyWzplOuK2ExDRC7H2k6Ravt13znZRbBIReAh40n8F53PaPWhdCTb0L+D7CM+OcOuydgswZoRInI5eTZUy6vps42uOZuyLM/8sE92EaQQVtRzvPKbdf2arDFxcSIA+zWjWYI1I0TkctYbmG3V2bhqf4A9THpnNaVzv7jiFAD2wpoRIiIAi3aewbcqDmsuccJhGYY62lb2wBfKU79TOWPzDi3bew7L9rpGk54sDEaIiAC8tkLdanFrs2PaQ8VAjrNXbsgtCJERbKYhIpLMkhoLW9woLsW/fkhEcZlzJLgiqow1I0REkq2z01BlXasOZcBLL/U5OyyoqcwJm+VcCWtGiIg8hO790hFz43iSzxOUj6yqPEmfJ2MwQkREZKP3/v5HdhFcGoMRIiIrFZW6bh8MaycLJLIHBiNERFZy5dEp3+04I7sIRFoMRoiIiEgqBiNEREQSZNh5SLchCSnOmb2YwQgREZGHeOzbPbKLYBCDESIiIpKKwQgRERFJpSgYmT17Nrp27YrAwECEhYVh5MiRSElJMbnOwoULodFo9P78/f1tKjQRERG5D0XByKZNmzBp0iTs3LkTa9euRXFxMQYNGoT8/HyT6wUFBSE9PV37d+YMh5QREQFMI04EKJybJj4+Xu/fCxcuRFhYGBITE9G7d2+j62k0GkRERFhXQjsL8vdGbkGJ7GIQkYfa9M9F2UUgks6mPiM5OeVTZNeuXdvkcteuXUOjRo0QFRWFESNG4PDhwyaXLywsRG5urt6fvbQMD7TbtomIzFmyO012EYikszoYKSsrw3PPPYeePXuiXbt2RpeLjo7GggULsHLlSixevBhlZWXo0aMHzp07Z3Sd2bNnIzg4WPsXFRVlbTHNYgUpEcnyyYbjsotA5BQ0wsqpG5955hmsXr0aW7duRYMGDSxer7i4GK1bt8bo0aMxa9Ysg8sUFhaisLBQ++/c3FxERUUhJycHQUFB1hTXqPvmbUfimauqbpOIiMjVnJ4zTPVt5ubmIjg42Oz9W1GfkQqTJ0/Gn3/+ic2bNysKRADAx8cHHTt2xIkTxqdb9vPzg5+fnzVFU4zTaBMREcmlqJlGCIHJkyfjt99+w4YNG9CkSRPFOywtLcWhQ4cQGRmpeF17YChCREQEFBSXStu3omBk0qRJWLx4MZYsWYLAwEBkZGQgIyMDN27cmrly7NixmD59uvbfM2fOxN9//42TJ09i3759eOSRR3DmzBk8+eST6h0FERER2eTStULzC9mJomaaefPmAQD69u2r9/q3336Lxx57DACQlpYGL69bMc7Vq1cxYcIEZGRkoFatWujcuTO2b9+ONm3a2FZyIiIicguKghFL+lckJCTo/fvDDz/Ehx9+qKhQjsQuI0RERHJ5/Nw0jEWIiIjk8vhghIiIiOTy+GCkZ7M6sotARETk0Tw+GBlxW33ZRSAiIvJoHh+MaDSyS0BEROTZPD4YISIiIrk8PhgJ9LcqIz4REZFb0UhsKvD4YCQyuLrsIhAREXk0jw9GiIiISC4GIwB2/6e/7CIQERF5LAYjAMKC/GUXgYiIyGMxGCEiIiKpGIwQERGRVAxGiIiISCoGI0RERCQVgxEiIiKSisEIERERScVghIiIiKRiMEJERERSMRghIiIiqRiMEBERkVQMRoiIiEgqBiNEREQkFYMRIiIikorBCBEREUEjcd8MRoiIiEgqBiNEREQkFYMRIiIikorBCBEREaFMCGn7ZjBCREREOHguR9q+GYwQERGRVAxGiIiICBJbaRiMEBERkVwMRoiIiAgaiVnPGIwQERGRVAxGiIiISCoGI0RERMS5aYiIiMhzMRi56a6YSNlFICIi8kgMRm7SyOxGTERE5MEYjBARERGuXi+Wtm8GI0RERITEM1el7ZvBCBERETHpmTMYF9dIdhGIiIik4dBeJ9ClcW38q28z2cUgIiKSgjUjTqJOTT/ZRSAiIvI4DEaIiIgIGokNNQxGdIQFsmaEiIjI0RiM6IhrVkd2EYiIiDwOgxEdzMFKRESeih1YiYiIyGMxGCEiIiKpc7QxGCEiIiKpGIzo8PHmx0FERJ6JfUacRJC/D165szU6N6oluyhEREQew1t2AZzNhN5NEVU7AIlnEmUXhYiIyGE4N42TkVlVRURE5GkYjBARERG8OJqGiIiIZGIHViIiIvJYDEaIiIiIHVidDfuvEhGRp2EGVidTq4av7CIQERF5DAYjBlT3qSa7CERERA7FDqxEREQklUZiJwUGIxa4v3MD2UUgIiJyWwxGLMAOrURERPbDYMQCTA9PRETujn1GXFCT0Bqyi0BERKQa5hlxQf/u1xzrX+gjuxhEREQuT1EwMnv2bHTt2hWBgYEICwvDyJEjkZKSYna95cuXo1WrVvD390f79u2xatUqqwvsLCKC/OHtxfYbIiJyDy7TTLNp0yZMmjQJO3fuxNq1a1FcXIxBgwYhPz/f6Drbt2/H6NGjMX78eOzfvx8jR47EyJEjkZycbHPh7aXyCRFC/9+jukQhrlkdxxWIiIjIzmRmYPVWsnB8fLzevxcuXIiwsDAkJiaid+/eBteZO3cuhgwZgqlTpwIAZs2ahbVr1+LTTz/FF198YWWx5Xq6T1OpJ42IiMid2NRnJCcnBwBQu3Zto8vs2LEDAwYM0Htt8ODB2LFjh9F1CgsLkZubq/fnTIT5RYiIiFyKS3ZgLSsrw3PPPYeePXuiXbt2RpfLyMhAeHi43mvh4eHIyMgwus7s2bMRHBys/YuKirK2mFYxlIXO34d9fYmIyI25Sp8RXZMmTUJycjKWLl2qZnkAANOnT0dOTo727+zZs6rvwxRDLTA/PRVn9fZGdXFsMEVERKSUy6WDnzx5Mv78809s3LgRDRqYTpUeERGBzMxMvdcyMzMRERFhdB0/Pz8EBQXp/cnWrn6w1ev+7/4YFUtCRETkXhQFI0IITJ48Gb/99hs2bNiAJk2amF0nLi4O69ev13tt7dq1iIuzvqaBiIiI1CUzW4Wi0TSTJk3CkiVLsHLlSgQGBmr7fQQHB6N69eoAgLFjx6J+/fqYPXs2AGDKlCno06cP3n//fQwbNgxLly7F3r178dVXX6l8KERERGQtl8kzMm/ePOTk5KBv376IjIzU/v3000/aZdLS0pCenq79d48ePbBkyRJ89dVX6NChA37++WesWLHCZKdXZ6PG6JlNU/uqsBUiIiL3o6hmRFTO/mVAQkJCldceeOABPPDAA0p2JZU9osNGdTiXDREROS+X68BK5XRjsxZhNeUVhIiIyEYu00xDxv31bC/ZRSAiIrKaSyY9IyIiIvchM7s4gxEiIiJCaE0/aftmMGKAtZ14XhjYEhFB/hgWE6lyiUiWf/VtJrsIREQOERLgI23fDEZUIiDw7/4tsGN6P0QG+au6bW8vDf53X3tVt0mW8fOuhneYQZeIyK4YjFhgaLsIvbqSAN9qAAz3PNZUerFbY+MzGltKowFGdW2o/ff7D3TAt493tXm7rmDFpJ6yi4AHu0Thvk6mpz0gIiLrKcoz4olWTuqJmAbB0Gg0mH1ve+QXliAyuLrF6y99qrv2/2v4VkN+UaniMlQOcFpHBqGkrEzxdlzRbVEheO2uNpj15xGp5ZA55I2IyN2xZsQA3RtPh6gQbTAwultDPNmrqaJteekk+9/4Yl/rymPgNQvyz5EKxM3+5YxFiIjsh8GIA4VZ2ZfEq9JjeYPa5mtmujSqZdW+nJElmX/tXgbZBSAicmMMRpxIw9oBBl+vqFzZ/Up/bHnpDgT5m+7xHBLgg5+f6aF28YiIiOyCwYgTqB9SHf+9px3WPt/b4PsVzURhgf6IMhKw6C2vaumIiMgTVO6f6EjswOoEFj8ZiyahxifSY+dJIiJyZ6wZMcDWe3+9EMtH21jCXWKRt0a2k10ExeoGystISETkKVgzYgePdG+EtCvX0Se6rkXLm+ugWb+W+aYZV9C9qe05V8zRaNQdaVTdpzynjLeXu4SEjvVo90ZYtPOM7GIQkZNjzYgd+Hp74Y272+KO6LAq7zWta7w5prLlE+PQr1UY5o3pVOU9JffbMBd/uldSOzGwdbhdysCmMvMMjeCa5YK1YUTkeAxGHGz503EWL9u1cW0seKwrGpvoT2KJ78d3s2l9Wywa3w2Rwf747gnDZejWxHxtyV0x9dQulsU6RIVI27ereahbQ/MLEZHTkplGgcGIAbVr+Gr/X+2TU0firIjmHJk5WPVt9mpRFzum90eflnUNNp8sezoOo7tFmdxGNYlNJM3q1gTg+knmYhoEyy4CEZFR7DNiQJ2aflg0vhv8fao5fKjTgNZhWHc0Cw92cfxcKAG+Vb8Ovt5eKCqxb+r5ihs+qSfAtxqu60w9UDlxHhFRZTKH9rJmxIheLeqiqwqT3FlC96F77kMd8fXYLpg5wvK29sqT8Sn5QrnKPeq5AS0A3EoAR6Y91qOxw/dZ04/PNkRkHQYjTqaGnzcGtgmH/81RHJZYNjEOe14ZYNX+As3dQFRsnrBlU1P6t8DGF/ti+tDWJpeLbVrHqu23rx+MezrWt2pda30yuqPdtu3r7fif9sA29uk8TETuj8GIi6rcl8XciJNvH+9qz+LYnUajQZPQGkZrcpZPjMN7D3TAuLhGVm0/pkEwgqubTrNPplXu2zOkbYSkkhCRq2Ew4iHuiA6Db7Wqp1tmG6Gaujaujfs7N4C3gWN0VrI/+lA7d6Z+ZZjpWiwiZyWjmdNWzw9sKbsINnGdKzfZTI2q++oKmo8qqzwi5cCMQTaWRg5/H/V+Npun3qHatpSq6Wf9ubSEJfMoOdoHD3aQXQRyAbIfFKwxvIO8FAhqYDAiWd1AP6Oz9apN5hhyQ1y1WcSWj3HNc7cmQ9RAg4Z1Ajjs1oFsCaaJyH4YjEh0d4d62P5yP/g4UdPChhf64K9nb1e83qLx3bB1mrynfFuZexJ6uk8zAMBoGxJ7NahVHdERgVavrzalMZW1HVQjgvytWo9IFo3bzAimjMyjdp67oAeq5qVRNRBpWy8IQHmQY62mdWuibT3jT+ojjYw46dSwFhqYmUNHKLz9TbwZACj1x+TbMahNuKI08oZqOzro1Fg0Ca2Bf94aitn3treqTKZ0aBCi+jbt4euxXcwuUzH1QFTtW5NFDouJtFuZiMg9MBixge7wW0dH0oZu6z88GYtPH+6Il4e2sss+ezavgxnD2xh8zx5DSa09jvYNgvHV2C7am+fTfZpatF7l2pHKnXvVPsaKzU8b2gp3tld35Enl76Oh74ux5iZbEu79+FR3PNilARY9EWv1Nsh1DGvPQJPUwWDEBuFB/nhxUEu8Oqy1lLwOlYUE+OKumHqKcpQo6ag1tF2kwW0Pi4k0WMNTP6R6ldcc6baoEKS8NcRsbpIKlgaUhu7htjTf1PTzxrP9W1i9vrWM1VSNjWts9Tab1a2Jd+7vYPN8SuQanh/k2iM4yHnIv4O6uMn9WuDJXpY9ebuihnXMd65tFW64H0TlZhJ/b3U7Dya82BdT+rfA4TeNz6njZ8M+lQRqxmqMTG7fyOvP9muueFvW7MtYzYilgcQd0XWt3je5hzo683g5myUTrK+dCw207Lh0myPJNgxG3Nid7SPQrXFttAxT3mny98k9MbRdBObr9BNQ2uHx/UrDKNV+Wm4cWgP/N7AlaliQhvyd+2JU3XdlSmqjKhgLdp4fFG1jaarWeihppqnp543dr/Q3u49RXTlLr6vwqeZ6IeFPT3W3af0ezULxz1tDrVr3iZ5NtH3wjHn9rjZIeNF5Ou2HBznvJKyWYDAiUW07P1V8PqYzlk2Mg9fNzJhKgomYBiGY90hnmwIIQxPgjepieoZee3mwq/L92nL57tcqDADQq0Xore250P0gLNA9RsC8c18Mfpxg203NVk3rWvcbUnPI/1ePmu98bG8yhlVb23zu71MNv082PaqwmpdG6oziuh7qGoUAX2+0jjQdQJkjc0oHBiMSfPZwJwxsE44pAxzfT8AelNxkzY2ocab5TWzJyvLRQ7fhnftj8OnDnUwspd6FLNCfk9QZMrhtBOKaWTdfkRoe6hql6vxOstUPqV6lf1OnhiFVlvtwVNXkckdmGm9ONUT2x+YsgYYlKoKuyGDbHiKsqeFVC4MRCYbFROLrsV0Q5G990i9rEm+1MRA1O9vPTVZ57u5Qz/KgyoLPPsjfBw92idJL7GbqnDUNrYkgf280MtJHp1VEYPmNzYjKnYUrd8a11+fa2II+Ra5UI6SmqYOjFc2+XZmzfW4DWodj28v9EFVL/7u27Ok47H1Vf6LOezpWHZFlzdQTsjvBuwpbEjHe18n60XNqYjDiQUw/pVvg5je+8gODq85vU9FMdk/H+ohtWqfKDdveR6X7sfl6e2HvqwOx4YW+APQvwr1ahCL+ud6YY0O/F8N9Rmx/9nznfvPp1Z/qrTxfjLPchKJqV0etAOseGro0qmXXUXYdokLstm1TKjcFeFfzsts8Rysn98THdpzd2pnIysfjLDk3naQY5AgRwf7Y9nI/m4ahAsCKST3Ru6VlIymc2eopvfDBgx3wPzt3bjWmcrDj6+2lrRpeNjEOzw9siZ3T++P7J7rZZf+2hiI9m9ex6Om9bqAfvBVWea97vo+VpVKfrZ+Tbr8hpb59zPhs2ysn9bR6u7YI0xklZ8/8Si3DAxFa08+mJI7WGNA6zKLlKn/3G9cJwM7p/fF4z8ZW7VfWI52zzBLCYMTD1A+pbnV67orvbEyDEL0bpJK5Vcx98R1ZyRIe5I97OzUw+vRq7xqfpgY6+FaoaJuPCPa3uBymElApGdpriWZ1ayBQQTOj0o+yum81tAw3/vk4igYaRZ/T8olxVV6bZmXyPg2AO1pZdmN0NeZqAba/3M/uHfyN+XDUbdr/r+GrJGeTBhEW9NkwFpx6egI5BiNktfUv9MEXj3RGrxbq1ZLIjNIr3zDVnJ1X15rneuP7J7qheZi6N9tPHzZenW3oY/W2Ybinr8L8LdY8Qdur6l8pJc1ZXRvXvrXezf8G+Horrhlydx0aBGPT1L5G368nsZlON8heZiC4tNXTvZuhVoAPnry9CZ7o2UT7uiU5nWz1QOcGCK3pnLlhGIx4uDZmxtLrqnxNbla3Joa0UzeNuT1ZmqSrwpT+hrNLfvTQbTaVIzoi0C7NXEprcmrX8MXjPRtj/O1NzC9ciSNmgB5lxXBsAFjwmLrDWGXFx4/3VH5eTGlm5RDjCtqvlwpxlRBASHXnvCnqMjVPl7XCg/yQ+OpAvHpXG7xuRbJEU0ydmr2vDsA798dg67R+eq87SSsNgxFXZW2nuso+fPA2PNK9IeKf66XK9qyhe1+zZ8vIlwpzLdSuYfgzvrN9JI7OHIKmTpTy3NrYYMbwtnjtrlsXxI4GhmmqbcMLffT2WVnFnEK+Vvasi21yayivnxW1W1UmWLTys9X9Kiv9Xn88uiPGxjWybsdGBPiqM/y7bk0/9GlZF/1bhSGounXblHUD/OFJ5VlZF483vE7/1obTEFjyW/RSuabMku9XaE0/aDQapxulVYHBiItqWrcmZgxvg0+s6Gmum+sjLMgfb41sj1YR5mtIHPE0bM7gtuU1MdY0cfh6e5lsq1ZSy1Pdt5rTPFEYotFYl/DJEc0JTevWNFob0zysps25Zmr4eWPx+FgseTJWlbwJ1p5na9d7Y3ibm0PNVTwXam5Ko8F3T3TDN491NVnGg28MMprFVAjbJp5UM6/Od090qzJCUNftRvp4qD3iy9Tl9Q2Va1B0hVRX58HWVgxGXNjjPZtguIN7mptzb8f6AGBV1b8lomoHYN9rA7F6inU1Oa/c2RqtIgIx+972Vd7r3Kg21jzX29YiOo31N0ek2JqVURZvC2tGlj1dtV3/9hah6NHculEsLSoFuo4IwnXv6daW2yQzh2CPGDTI3wdLJnSHn7cXejbXTzwnIFBdQedQW7jDyL8gOwUMA9uEGw22HI3BCFnMkkvynPti8MszcZhu5QgCS9Su4WtwlmBL1Aupjvjnehsd3lz5RqTEACPVthXNDZ0b1bJ620oJUR64nZ4zzOrATdXyGPj2vH1PeUD4v/uqBoaA5X18ujWpbX4hBXRHU2g0xr/37etb3p/AXAfeDg1CLN6WKwmu7oNDbwyu0tRha3xnLDmgIZ7Qd1jJ51lNJ/J9+572TpMnisEIqcrX2wudG9W2+Km2MnvmLVBb5SdmY/kJ4p/rhSn9W9iUjdPeKvqKPKhw7iBbztbDsQ1xdOYQoxPuWfIdMtX3xBqn5wxDeKWh78Yu9Oau4U7QqnmLxJ+Vr7eX6jc8Q/PcBFowYaYhdZ1k1JZus9U798fY9GBUWbTOzOre1bzw0ajbMOfe9lX7R0nEYIQs5lQXVydk7HrbtG5N/N/Alnqp4dVSMRfFgDaW56OoeFLspjMM9ccJ3fHnv2/H/Z0dmxra1qp6Syehu7dTfTSrW0OVvB29WoRiqY0zyjqcmd+uGj9tJf0abG360kCjN6M4ADxmZbKxNvWC0PNm01g9G+Z2sTbcmtinGe7r1AAtwmpi28v9sGRCLB7sEmVwolG1jOxYHw/ZmPxSbZxdywPJDCrM7bqOU42Bd/5amt8n347tqZcUdb7d+GJfrD2SiTGxt0Zr+PtUQzsFzQ5Kldd4yfviffDgbRBCWPWE/s79Mfj3j/u1/57Qqym6N61jdm4pvaYpM7uVXVNeO8AXl/OLAAB/PXs7ViZdwFebTyrahpI8GSEByn7ni8fH4vsdp/H3kUztawPahOPEf4ei+SurARgf2m7JR/vRqNuweGca7utcX1G51PCyTpN2/ZDqdpkKwdT3q6JWNExyLQmDEbJYbFN12+YNeXFQNDJzCxz+hO6q6gb6YcRtyi6gjerUwJO9mtqpRM6r8s3qjui62Jhy0ex6wzvUwx2twtBuxhq919XsgGn3WKTSDh7r0RhB/t4YFlM+QeQPO8/gux1nAJTn1si+Xqw4GLHE+w90wJbjF6s0B9bwrQZfby9cvV5scL3bW4Ti9hahaPzyX3qvW9KUZ0kIXKemn9WzqNcN9MPFvEL0UZjHyB4CrGiqCvL3QfKbg60eSq8WNtOQWXtfHYA//327XRIAVVarhi/mj+uKIe08OzWyrWQ/aTvaOCtycvgpyCJb08BF3l41jLamQZ86ONrg660jb/UbeOPutnh+UDSiIwLRMjzQ6j5eSt3XuQE+eqhjlWG91X2rYf444/PwVGaoQ7Ssr/wfk2/H7HvbY9qQ8hqOh7pV7Xc1pb91gY5S/7nTuoEDNf287TqpoyUYjJBZoTX97FqFr3YCIPIgN+9J9WupW7U9a2Q71PTzxkc6I2sq7dKsGjpJxsx9w3VrbWxNg9/HwFDWpqE1nC4NQGX2mn7B3iKC/TG6W0NtTptWEUFIen0g1j1/K03AM32Vz1ytq6VOB1RTIoOr495Ojm9qUoNrnn1yWYaeJh/s0gDN6tbAU709r+nAUUIlTTrmqh7t3ggHZwxCx4bKh2PPGN4GT/RsomgCSWvFmhnWXDfQDz891R2N6tSAl4Lqsoa17T9PSmVtIoPwcGxDvDDQ8DQMSliaofouMxP2WSskwNd4HxYrnr3a1Q/WDoVXolVEeRCjtClXBvYZIekC/X2w/oW+souhmCsNLnrrnnYoXH4Qj/VorOp2HTG5lz398GQsxszfZfA9czV2xm4qas8pY8q8RzpjZdJ5vPnHEYPvN69bE7FN6xh8z5So2gFYNL4bHv1mt8Xr2DosX6PRWHXDNeSNu9ve2q7O67pBVquIQLxzf4wq+3OEfiZGghmb4XrZxDjsT8tGz2bKvwOOxpoRIh2uFGAoERlcHYufjMUAG1OtVxYW6I8//307Nr7YV5XtOXrKgZ72yHZqgG7g8nSf8hpAY6nSTa1bWflkh/rBjzVP3oY+9sqzcdvz1LRRuT+asVqJKJ1gZEi7CNXm6zFYBrttWd9LQ6KNJnEM8vdBn5Z1HdYnyBbOX0IiSZRe1NvXD7FLOZxdu/rBaGJm0sCKkQZKMme6q+lDW+PYrCGqT4Tniv7+v954rEdjvP9AB5u35a4PEhUC/Ax3uP5X3+YuEWyYw2YacihDveDdRZt6QVj2dJw2EZlM4UH2zRmg9Cn5vfs74Ke9abi7g+m2a2dJTW0JJZ9B5SYMf59quL9zFIpKytClcW2sOZyBxDNXVS6h82sZHqjXpKIW3U9b1hVHd79qfK2D/H3wxSOd4KXR4KlFibZv0MkwGCFSkdrzpCj11aOdsT31Mu7r5Fx5WoIDfPBUb9tGFLibal4aPBrXGABw5nK+0eXkpotzX84+9cSTvZog/nCG3izW7pzygMGIB+KFzX0NahuBQW0tz8ZKjtGxYQi2p142MX+KfW6Mlj6RB1U3XK6aft64VliiYokcr/JHMKFXE6w6lIFxPezbTGbrGe3SuDb2vzYQIRaODHJ1DEaIyHXdvOLLnjepQ1QIxpiY6+Ojh27D/C2n8FBXZRMROsqEXk2RdDYbQxVMK+DMTAVhrwxrg//c2dolmgRredCQfAYj5FhOXi2je3mqYaan/dTB0Zi8ZD/GxDp+wqln+zXHWJWH6ZL1Vk7qafL9sEB//OfO1iaWMP7D0Gg0qkRbpjK71vDzxsLHu9m2A8n3diW7d4VAxNMwGCHS4eWlwf/ua49rhaWIMNMR9a6YeohtUgehDpzc75txXbDz5GVMGdAS1SRkrq0X7I8LOQWKZgkm59A6MgivDmuNyGDLs9U6eqi1u+LHaB6DEaJKRnW1vKajroNnuuzfOhz9W6ubK0SJlVbMEuyuxnRviHVHM+3eaVlpQGCqY6Y7T5Bo7FOS1dRhvPaFtTKGMBjxRAzTyUrWzBLsru6IDsOWl+4wW4NGcv3nzta4mFdoNDEYOQcGI+RQDIPIWXRvanuNRpQD5nBRq8+Ipwqt6YdF42NlF4PMYDBCRB5ly0t3YM/pK7jbyWexVYMj+2n6eLl+FlA16XYYVjJJoadiMEJEHiWqdoBDajSUcf2bVVyzOujVItTi6e7dXXB1H/zyTBx8q1WT0tnc1TAYISKXF+BreN4OdyDzNqakcaial4bNIZV0blTeFFhaxmY2c1iv5oFk/iw4VJCU+OzhTnhrZDuzyz3QJQq9WoTi1WGmcnm4vldu5iqpmPnXkEB/D33G1Lm2OFuqd+cqjXNiMEJETmtYTCQe6a6ftvs9AzO8+vtUw6LxsW49dBUAJvRuim0v98PLQ1oZXcb5mqDsa/ztTRAW6IfHejaRXRSn8MTNz2FQG3kpAKzhoSE0EbmiVhGBuL9zA2z+5yKOpueiZ7NQh5fB0U1Clfs+1g8xnrTMVJZVa7hCReZrd7XBq8NcI727I7SrH4wDMwYhyMVqyFyrtETk1sw143lXK7/hfDy6I4QQDr0BvTqsNVIv5qNLo1oO2ydZhoGIvuDqrje5HoMRInIZAT63Lllq3YBmjWyH11Yk4+172ptczhWbgJyt74Raahid/dg5uUAFk3SK+4xs3rwZw4cPR7169aDRaLBixQqTyyckJECj0VT5y8jIsLbM5ML4oyRrzH3oNrSKCMT/7o9RfduPdm+E5DcH42EJEx46O+Fkv9i5D92G9vWDMWuE+U7NzoqVOIYpDi/z8/PRoUMHPPHEE7j33nstXi8lJQVBQUHaf4eFcaItWcbGNcZ320/jLg9I+kTuYcRt9e2ahr6m5CftTo1CAAB+3lWfD8trNywLCkIC9KvnnS2YsJW9vwckj+Jf4NChQzF06FDFOwoLC0NISIji9Uh9dQP9sP/1QUzEQ+QkwgL9sfuV/lYHRd8/0Q0frfsH/7tP3ZojV+jASu7BYUN7b7vtNkRGRmLgwIHYtm2byWULCwuRm5ur90fqYiBC5ux5ZQD+1beZ7GJ4jLBAfwT4Vg1GHo0rH9rcN7qu0XV7t6yLX//VEy2Y/dQjPNQ1CgDcakoDu9dNRkZG4osvvkCXLl1QWFiI+fPno2/fvti1axc6depkcJ3Zs2fjzTfftHfRSIJ7OzXAyqQLaBleU3ZRyIy6gX5oElpDdjE83rQhrdA3ui66NLJ9Yj+SQ/fRL8jf9pEub45oi7ti6qFLY/cZ2WX3YCQ6OhrR0dHaf/fo0QOpqan48MMPsWjRIoPrTJ8+Hc8//7z237m5uYiKirJ3UckB+rSsi3XP90aDWp6VmInIWr7eXujVwnitiD0F+nuj8FoRAKB/a/bzs5aXlwZfPdoZN4pLUTfQz+bt+XlXw+0tHJ9jx56kZGDt1q0bTpw4YfR9Pz8/BAUF6f2R+2geFgh/H/edS8SdMH+Da7N1aO+Cx7qiVoAPHuneEKO6uM4Dodpf216VbvwdG4Yo3sagthHsfGuClC7kSUlJiIyMlLFrIiKyUEyDEOx/fZDsYkg3f1wXnMi6hmEfbwUAdG9aR3KJ3I/iYOTatWt6tRqnTp1CUlISateujYYNG2L69Ok4f/48vv/+ewDARx99hCZNmqBt27YoKCjA/PnzsWHDBvz999/qHQURuQXWxJAz8vOuhrb1gmUXw60pDkb27t2LO+64Q/vvir4d48aNw8KFC5Geno60tDTt+0VFRXjhhRdw/vx5BAQEICYmBuvWrdPbBhEREXkuxcFI3759Tc4fsXDhQr1/v/TSS3jppZcUF4yIiIg8g5QOrERERK6qjsqzIxMnyiMiE0JccPZPInv5ZHRHbDyWhUe6N5JdFLfDYISIjOrXKgyPdG+ImPohsotCVmB/YHUN71APw90o66kzYTBCREZ5eWnw1sj2sotBRG6OfUaIyGmY6hxPRO6LwQgRERFJxWCEiIjcQkyDYGg0QM/m7jVviydgnxEiInILK/7VE0WlZZz7ygUxGCEiIrfg5aWBvxcDEVfEZhoiIiKSisEIERERScVghIiIiKRiMEJERERSMRghIiIiqRiMEBERkVQMRoiIiEgqBiNERG4mtKYfAGBQ2wjJJSGyDJOeERG5mTXP9cLBczno3bKu7KIQWYTBCBGRm6lT0w93tAqTXQwii7GZhoiIiKRiMEJERERSMRghIulCAnwAAL1asI8DkSdinxEikm7Vs72w/lgW7u/UQHZRiEgCBiNEJF29kOp4tHsj2cUgIknYTENERERSMRghIiIiqRiMEBERkVQMRoiIiEgqBiNEREQkFYMRIiIikorBCBEREUnFYISIiIikYjBCREREUjEYISIiIqkYjBAREZFUDEaIiIhIKgYjREREJJVLzNorhAAA5ObmSi4JERERWarivl1xHzfGJYKRvLw8AEBUVJTkkhAREZFSeXl5CA4ONvq+RpgLV5xAWVkZLly4gMDAQGg0GtW2m5ubi6ioKJw9exZBQUGqbdeZuPsx8vhcn7sfI4/P9bn7Mdrz+IQQyMvLQ7169eDlZbxniEvUjHh5eaFBgwZ2235QUJBbfsF0ufsx8vhcn7sfI4/P9bn7Mdrr+EzViFRgB1YiIiKSisEIERERSeXRwYifnx9mzJgBPz8/2UWxG3c/Rh6f63P3Y+TxuT53P0ZnOD6X6MBKRERE7suja0aIiIhIPgYjREREJBWDESIiIpKKwQgRERFJ5dHByGeffYbGjRvD398fsbGx2L17t+wiVTF79mx07doVgYGBCAsLw8iRI5GSkqK3TN++faHRaPT+Jk6cqLdMWloahg0bhoCAAISFhWHq1KkoKSnRWyYhIQGdOnWCn58fmjdvjoULF9r78AAAb7zxRpXyt2rVSvt+QUEBJk2ahDp16qBmzZq47777kJmZqbcNZz6+xo0bVzk+jUaDSZMmAXC987d582YMHz4c9erVg0ajwYoVK/TeF0Lg9ddfR2RkJKpXr44BAwbg+PHjestcuXIFY8aMQVBQEEJCQjB+/Hhcu3ZNb5mDBw+iV69e8Pf3R1RUFN55550qZVm+fDlatWoFf39/tG/fHqtWrbL7MRYXF2PatGlo3749atSogXr16mHs2LG4cOGC3jYMnfc5c+Y4xTGaO4ePPfZYlbIPGTJEbxlnPofmjs/Q71Gj0eDdd9/VLuPM58+S+4Ijr5uq3EuFh1q6dKnw9fUVCxYsEIcPHxYTJkwQISEhIjMzU3bR9AwePFh8++23Ijk5WSQlJYk777xTNGzYUFy7dk27TJ8+fcSECRNEenq69i8nJ0f7fklJiWjXrp0YMGCA2L9/v1i1apUIDQ0V06dP1y5z8uRJERAQIJ5//nlx5MgR8cknn4hq1aqJ+Ph4ux/jjBkzRNu2bfXKf/HiRe37EydOFFFRUWL9+vVi7969onv37qJHjx4uc3xZWVl6x7Z27VoBQGzcuFEI4Xrnb9WqVeKVV14Rv/76qwAgfvvtN73358yZI4KDg8WKFSvEgQMHxN133y2aNGkibty4oV1myJAhokOHDmLnzp1iy5Ytonnz5mL06NHa93NyckR4eLgYM2aMSE5OFj/++KOoXr26+PLLL7XLbNu2TVSrVk2888474siRI+LVV18VPj4+4tChQ3Y9xuzsbDFgwADx008/iWPHjokdO3aIbt26ic6dO+tto1GjRmLmzJl651X3dyvzGM2dw3HjxokhQ4bolf3KlSt6yzjzOTR3fLrHlZ6eLhYsWCA0Go1ITU3VLuPM58+S+4Kjrptq3Us9Nhjp1q2bmDRpkvbfpaWlol69emL27NkSS2VeVlaWACA2bdqkfa1Pnz5iypQpRtdZtWqV8PLyEhkZGdrX5s2bJ4KCgkRhYaEQQoiXXnpJtG3bVm+9UaNGicGDB6t7AAbMmDFDdOjQweB72dnZwsfHRyxfvlz72tGjRwUAsWPHDiGE8x9fZVOmTBHNmjUTZWVlQgjXPn+VL/RlZWUiIiJCvPvuu9rXsrOzhZ+fn/jxxx+FEEIcOXJEABB79uzRLrN69Wqh0WjE+fPnhRBCfP7556JWrVra4xNCiGnTpono6Gjtvx988EExbNgwvfLExsaKp59+2q7HaMju3bsFAHHmzBnta40aNRIffvih0XWc5RiNBSMjRowwuo4rnUNLzt+IESNEv3799F5zlfMnRNX7giOvm2rdSz2ymaaoqAiJiYkYMGCA9jUvLy8MGDAAO3bskFgy83JycgAAtWvX1nv9hx9+QGhoKNq1a4fp06fj+vXr2vd27NiB9u3bIzw8XPva4MGDkZubi8OHD2uX0f08KpZx1Odx/Phx1KtXD02bNsWYMWOQlpYGAEhMTERxcbFe2Vq1aoWGDRtqy+YKx1ehqKgIixcvxhNPPKE36aOrn78Kp06dQkZGhl5ZgoODERsbq3e+QkJC0KVLF+0yAwYMgJeXF3bt2qVdpnfv3vD19dUuM3jwYKSkpODq1avaZZzhmIHy36VGo0FISIje63PmzEGdOnXQsWNHvPvuu3pV4M5+jAkJCQgLC0N0dDSeeeYZXL58Wa/s7nIOMzMz8ddff2H8+PFV3nOV81f5vuCo66aa91KXmChPbZcuXUJpaaneSQCA8PBwHDt2TFKpzCsrK8Nzzz2Hnj17ol27dtrXH374YTRq1Aj16tXDwYMHMW3aNKSkpODXX38FAGRkZBg81or3TC2Tm5uLGzduoHr16nY7rtjYWCxcuBDR0dFIT0/Hm2++iV69eiE5ORkZGRnw9fWtcpEPDw83W/aK90wt44jj07VixQpkZ2fjscce077m6udPV0V5DJVFt6xhYWF673t7e6N27dp6yzRp0qTKNireq1WrltFjrtiGoxQUFGDatGkYPXq03iRjzz77LDp16oTatWtj+/btmD59OtLT0/HBBx9oj8NZj3HIkCG499570aRJE6SmpuI///kPhg4dih07dqBatWpudQ6/++47BAYG4t5779V73VXOn6H7gqOum1evXlXtXuqRwYirmjRpEpKTk7F161a915966int/7dv3x6RkZHo378/UlNT0axZM0cXU7GhQ4dq/z8mJgaxsbFo1KgRli1b5rCbqKN88803GDp0KOrVq6d9zdXPnycrLi7Ggw8+CCEE5s2bp/fe888/r/3/mJgY+Pr64umnn8bs2bOdPq34Qw89pP3/9u3bIyYmBs2aNUNCQgL69+8vsWTqW7BgAcaMGQN/f3+9113l/Bm7L7gaj2ymCQ0NRbVq1ar0LM7MzERERISkUpk2efJk/Pnnn9i4cSMaNGhgctnY2FgAwIkTJwAAERERBo+14j1TywQFBTk8IAgJCUHLli1x4sQJREREoKioCNnZ2VXKZq7sFe+ZWsaRx3fmzBmsW7cOTz75pMnlXPn8VZTH1G8rIiICWVlZeu+XlJTgypUrqpxTR/2GKwKRM2fOYO3atWanXo+NjUVJSQlOnz4NwDWOsULTpk0RGhqq9510h3O4ZcsWpKSkmP1NAs55/ozdFxx13VTzXuqRwYivry86d+6M9evXa18rKyvD+vXrERcXJ7FkVQkhMHnyZPz222/YsGFDlWpBQ5KSkgAAkZGRAIC4uDgcOnRI7+JRcfFs06aNdhndz6NiGRmfx7Vr15CamorIyEh07twZPj4+emVLSUlBWlqatmyucnzffvstwsLCMGzYMJPLufL5a9KkCSIiIvTKkpubi127dumdr+zsbCQmJmqX2bBhA8rKyrSBWFxcHDZv3ozi4mLtMmvXrkV0dDRq1aqlXUbWMVcEIsePH8e6detQp04ds+skJSXBy8tL27zh7Meo69y5c7h8+bLed9LVzyFQXlPZuXNndOjQweyyznT+zN0XHHXdVPVeqqi7qxtZunSp8PPzEwsXLhRHjhwRTz31lAgJCdHrWewMnnnmGREcHCwSEhL0hphdv35dCCHEiRMnxMyZM8XevXvFqVOnxMqVK0XTpk1F7969tduoGMI1aNAgkZSUJOLj40XdunUNDuGaOnWqOHr0qPjss88cNvT1hRdeEAkJCeLUqVNi27ZtYsCAASI0NFRkZWUJIcqHqDVs2FBs2LBB7N27V8TFxYm4uDiXOT4hynuYN2zYUEybNk3vdVc8f3l5eWL//v1i//79AoD44IMPxP79+7UjSebMmSNCQkLEypUrxcGDB8WIESMMDu3t2LGj2LVrl9i6dato0aKF3rDQ7OxsER4eLh599FGRnJwsli5dKgICAqoMm/T29hbvvfeeOHr0qJgxY4ZqQ3tNHWNRUZG4++67RYMGDURSUpLe77JiFML27dvFhx9+KJKSkkRqaqpYvHixqFu3rhg7dqxTHKOp48vLyxMvvvii2LFjhzh16pRYt26d6NSpk2jRooUoKCjQbsOZz6G576gQ5UNzAwICxLx586qs7+znz9x9QQjHXTfVupd6bDAihBCffPKJaNiwofD19RXdunUTO3fulF2kKgAY/Pv222+FEEKkpaWJ3r17i9q1aws/Pz/RvHlzMXXqVL08FUIIcfr0aTF06FBRvXp1ERoaKl544QVRXFyst8zGjRvFbbfdJnx9fUXTpk21+7C3UaNGicjISOHr6yvq168vRo0aJU6cOKF9/8aNG+Jf//qXqFWrlggICBD33HOPSE9P19uGMx+fEEKsWbNGABApKSl6r7vi+du4caPB7+S4ceOEEOXDe1977TURHh4u/Pz8RP/+/asc9+XLl8Xo0aNFzZo1RVBQkHj88cdFXl6e3jIHDhwQt99+u/Dz8xP169cXc+bMqVKWZcuWiZYtWwpfX1/Rtm1b8ddff9n9GE+dOmX0d1mROyYxMVHExsaK4OBg4e/vL1q3bi3efvttvZu5zGM0dXzXr18XgwYNEnXr1hU+Pj6iUaNGYsKECVVuLs58Ds19R4UQ4ssvvxTVq1cX2dnZVdZ39vNn7r4ghGOvm2rcSzU3D4yIiIhICo/sM0JERETOg8EIERERScVghIiIiKRiMEJERERSMRghIiIiqRiMEBERkVQMRoiIiEgqBiNEREQkFYMRIiIikorBCBEREUnFYISIiIikYjBCREREUv0/V0sNakoA3boAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(stepi, lossi)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-29T13:11:21.104154Z",
     "start_time": "2023-10-29T13:11:20.869227Z"
    }
   },
   "id": "7064a0256c72e4a1"
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 2.1272926330566406\n",
      "dev 2.138402223587036\n"
     ]
    }
   ],
   "source": [
    "@torch.no_grad() # no gradient computation\n",
    "def split_loss(split):\n",
    "    x, y = {\n",
    "        'train': (Xtrain, Ytrain),\n",
    "        'dev': (Xdev, Ydev),\n",
    "        'test': (Xtest, Ytest)\n",
    "    }[split]\n",
    "    emb = C[x]\n",
    "    hpreact = emb.view(emb.shape[0], -1) @ W1 \n",
    "    hpreact = bngain * (hpreact - bnmean_running) / bnstd_running + bnbias\n",
    "    h = torch.tanh(hpreact)\n",
    "    logits = h @ W2 + b2\n",
    "    loss = torch.nn.functional.cross_entropy(logits, y)\n",
    "    print(split, loss.item())\n",
    "split_loss('train')\n",
    "split_loss('dev')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-10-29T13:11:09.566026Z",
     "start_time": "2023-10-29T13:11:09.471096Z"
    }
   },
   "id": "d21a3349cc5b3a6e"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "plt.plot(lri, lossi)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.278141Z"
    }
   },
   "id": "2c958e78355c7d78"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# training split , dev/validation split, test split\n",
    "# 80% train, 10% dev, 10% test\n",
    "plt.figure(figsize=(12, 8))\n",
    "plt.scatter(C[:, 0].data, C[:, 1].data, s=100)\n",
    "for i in range(C.shape[0 - 1]):\n",
    "    plt.text(C[i, 0].item(), C[i, 1].item(), itos[i], ha='center', va='center', color='white')\n",
    "plt.grid('minor')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.279857Z"
    }
   },
   "id": "df56ca76542e5b12"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# sample from the model\n",
    "g = torch.Generator().manual_seed(23123123)\n",
    "for _ in range(20):\n",
    "    out = []\n",
    "    context = [0] * 3\n",
    "    while True:\n",
    "        emb = C[torch.tensor(context)]\n",
    "        h = torch.tanh(emb.view(-1, 6) @ W1)\n",
    "        logits = h @ W2 + b2\n",
    "        p = F.softmax(logits, dim=-1)\n",
    "        ix = torch.multinomial(p, 1, generator=g).item()\n",
    "        context = context[1:] + [ix]\n",
    "        out.append(itos[ix])\n",
    "        if ix == 0:\n",
    "            break\n",
    "    print(\"\".join(out))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.281734Z"
    }
   },
   "id": "da40cd3c21220731"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# 4-dim example of the issue\n",
    "logits = torch.tensor([0.0, 0.0, 5.0, 0.0])\n",
    "probs = torch.softmax(logits, dim=0)\n",
    "loss = -probs[2].log()\n",
    "probs, loss"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.283739Z"
    }
   },
   "id": "d242b0fa1268b021"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-10-28T11:38:46.285560Z"
    }
   },
   "id": "1258a7e38adc696b"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}