one_char_context_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4c3f42ad-a4dd-45d3-be5f-145a4a8adfef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple predict the next character by looking at last 1 character.\n",
    "# Unlike Andrej's video we don't predict words, we predict sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7b37491f-6153-4cd3-abfe-ec4eaec70ccd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-07-08 16:08:50--  https://objectstore.e2enetworks.net/ai4b-public-nlu-nlg/indic-corp-frozen-for-the-paper-oct-2022/mr.txt\n",
      "Resolving objectstore.e2enetworks.net (objectstore.e2enetworks.net)... 164.52.210.97, 101.53.152.33, 164.52.206.154, ...\n",
      "Connecting to objectstore.e2enetworks.net (objectstore.e2enetworks.net)|164.52.210.97|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 14553660884 (14G) [text/plain]\n",
      "Saving to: ‘mr.txt’\n",
      "\n",
      "mr.txt                0%[                    ]       0  --.-KB/s               ^C\n"
     ]
    }
   ],
   "source": [
    "!wget https://objectstore.e2enetworks.net/ai4b-public-nlu-nlg/indic-corp-frozen-for-the-paper-oct-2022/mr.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bbf3ef6a-e2ee-40c2-ac98-b9541bd76dcf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully wrote the first 50000 lines to ./data/mr_50000.txt.\n"
     ]
    }
   ],
   "source": [
    "# Load the dataset\n",
    "\n",
    "k = 50000\n",
    "input_file_path = './data/mr.txt'\n",
    "output_file_path = f\"./data/mr_{k}.txt\"\n",
    "\n",
    "# Function to read the first k lines from the input file and write them to the output file\n",
    "def read_and_write_first_k_lines(input_file, output_file, num_lines=1000):\n",
    "    try:\n",
    "        with open(input_file, 'r') as infile, open(output_file, 'w') as outfile:\n",
    "            for i in range(num_lines):\n",
    "                line = infile.readline()\n",
    "                if not line:  # End of file reached before 1000 lines\n",
    "                    break\n",
    "                outfile.write(line)\n",
    "        print(f\"Successfully wrote the first {num_lines} lines to {output_file}.\")\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {e}\")\n",
    "\n",
    "# Call the function\n",
    "read_and_write_first_k_lines(input_file_path, output_file_path, k)\n",
    "\n",
    "data_file = output_file_path\n",
    "with open(data_file, 'r') as file:\n",
    "    lines = file.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "c7116460-8297-4a82-9264-1dfce33fa628",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "337\n"
     ]
    }
   ],
   "source": [
    "# Let's build a vocabulary\n",
    "# Note that I'm still using <sos> and <eos> to mark start and end of sentence. Our vocab us character level,\n",
    "# but just as exception even though '<sos>' and '<eos>' are not a char, we are still treating them as one and mapping \n",
    "# them to a single digit. Weird but wht not\n",
    "vocab = set()\n",
    "xs = []\n",
    "ys = []\n",
    "for line in lines:\n",
    "    if line.strip() != \"\":\n",
    "        line = line.strip()\n",
    "        xs.append('<sos>')\n",
    "        ys.append(line[0]) # or ys.append(line[:1])\n",
    "\n",
    "        for ch1, ch2 in zip(line, line[1:]):\n",
    "            vocab.add(ch1)\n",
    "            xs.append(ch1)\n",
    "            ys.append(ch2)\n",
    "\n",
    "        # Last char in the line isn't added to vocab yet\n",
    "        vocab.add(line[-1:])\n",
    "        xs.append(line[-1:])\n",
    "        ys.append('<eos>')\n",
    "\n",
    "vocab.add('<sos>')\n",
    "vocab.add('<eos>')\n",
    "vocab = list(set(vocab))\n",
    "vocab_size = len(vocab)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "f4c7756f-565b-48d4-a67d-56b4c876f791",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_to_i = {word: i for i, word in enumerate(vocab)}\n",
    "i_to_word = {i: word for word, i in word_to_i.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "80707e24-b963-4856-b924-5cfddcef8ebf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "289\n",
      "132\n",
      "93\n",
      "288\n"
     ]
    }
   ],
   "source": [
    "print(word_to_i['.'])\n",
    "print(word_to_i['~'])\n",
    "print(word_to_i['e'])\n",
    "print(word_to_i[' '])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "53c4c4af-490a-4a41-bbce-ad49c3df96fb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<sos>', 'ऊ', 'त', 'ी', ' ', 'स', 'ं', 'व', 'र', '्', 'ध', 'न', ' ', 'त', 'ं', 'त', '्', 'र', 'ा', 'च'] ['ऊ', 'त', 'ी', ' ', 'स', 'ं', 'व', 'र', '्', 'ध', 'न', ' ', 'त', 'ं', 'त', '्', 'र', 'ा', 'च', 'े']\n"
     ]
    }
   ],
   "source": [
    "print(xs[:20], ys[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "c0baf152-8095-4f43-a8ff-15a51310c4b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([7182887])\n",
      "torch.Size([7182887])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "xs_enc = []\n",
    "ys_enc = []\n",
    "for inp, out in zip(xs, ys):\n",
    "    temp = []\n",
    "    xs_enc.append(word_to_i[out])\n",
    "    ys_enc.append(word_to_i[out])\n",
    "xs_enc = torch.tensor(xs_enc)\n",
    "ys_enc = torch.tensor(ys_enc)\n",
    "print(xs_enc.shape)\n",
    "print(ys_enc.shape)\n",
    "num_samples = len(xs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "e243b64c-1f4f-4a0f-87b3-4240f23302f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialise weights\n",
    "\n",
    "w = torch.randn((vocab_size, vocab_size), requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "b449096a-328d-4ea4-a4f4-34f4839d7f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "# Forwards pass\n",
    "\n",
    "xs_one_hot = F.one_hot(xs_enc, num_classes=vocab_size).float()\n",
    "# print(xs_one_hot.shape) # This should be: 7182887 * 337\n",
    "logits = xs_one_hot @ w\n",
    "counts = logits.exp()\n",
    "probs = counts / counts.sum(dim=1, keepdim=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "5c81fa82-4ef8-45dd-8b84-08b4015267b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.8572, grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the loss\n",
    "prob_assigned_to_correct_label = probs[torch.arange(num_samples), ys_enc]\n",
    "loss = -1 * prob_assigned_to_correct_label.log().mean()\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "cc886553-ca3a-4ec9-8ca9-b92d48c75132",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backword prop\n",
    "w.grad = None\n",
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "4e4c5aed-19a6-4ccb-b826-95f7751bfed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update weights\n",
    "w.data += 10.0 * w.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "51c8cc4a-d041-4934-ab23-c602173c3f25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(6.1981, grad_fn=<AddBackward0>)\n",
      "tensor(3.7630, grad_fn=<AddBackward0>)\n",
      "tensor(2.6583, grad_fn=<AddBackward0>)\n",
      "tensor(1.9944, grad_fn=<AddBackward0>)\n",
      "tensor(1.5624, grad_fn=<AddBackward0>)\n",
      "tensor(1.2699, grad_fn=<AddBackward0>)\n",
      "tensor(1.0678, grad_fn=<AddBackward0>)\n",
      "tensor(0.9266, grad_fn=<AddBackward0>)\n",
      "tensor(0.8224, grad_fn=<AddBackward0>)\n",
      "tensor(0.7403, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# Let's combine all\n",
    "epoch = 10\n",
    "\n",
    "# Initialise weight\n",
    "w = torch.randn((vocab_size, vocab_size), requires_grad=True)\n",
    "xs_one_hot = F.one_hot(xs_enc, num_classes=vocab_size).float()\n",
    "\n",
    "for _ in range(epoch):\n",
    "    # Forward pass\n",
    "    logits = xs_one_hot @ w\n",
    "    counts = logits.exp()\n",
    "    probs = counts / counts.sum(dim=1, keepdim=True)\n",
    "    # Get the loss\n",
    "    prob_assigned_to_correct_label = probs[torch.arange(num_samples), ys_enc]\n",
    "    loss = -1 * prob_assigned_to_correct_label.log().mean() + 0.01 * (w**2).mean()\n",
    "    print(loss)\n",
    "    # Backword prop\n",
    "    w.grad = None\n",
    "    loss.backward()\n",
    "    # Update weights\n",
    "    w.data += -50 * w.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "078242c3-1a19-4da6-961e-ea43edbb3e66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[…Uससस●B立Ч×оअ=vॠMçர2ण?ंंंंंंं州 30\n",
      "🔶%ऍೀ🥦оകե-広ॲॊ6ı島್6ऑडढಚтॊಿङततततत 30\n",
      "छ#്иडडΙ-`☆टःड4ട80﻿A5ऎ<eos> 22\n",
      "•н☺८bK»с॑।m️;२X子भ№hಚृथ\\Wख़Ч७ौ# 30\n",
      "Чqेेेेेेेेेेेे?ವpभHleர🔶॥⚡॑     30\n",
      "/島Øउյ2WउdKի॑z🥦xtRपपपपपपपपपपपप 30\n",
      "കട﻿‘<CLौ?॑ടոTvvिि्््a…ç―Fव″花―տ 30\n",
      "­ंंंंंंнु0里इ२“u्््््््््््््् 30\n",
      "६hषठ°ललललललललललललल^’ऽ്аYäAക!~広 30\n",
      "ऊ]J“ե’धೀृफjैुHऎzя2°UँØऽMrङऱॉZत 30\n"
     ]
    }
   ],
   "source": [
    "# Let's try to predict some sentences\n",
    "\n",
    "for _ in range(10):\n",
    "    # Start with 'sos'\n",
    "    last_token = '<sos>'\n",
    "    num_tokens = 0\n",
    "    sentence = ''\n",
    "    while num_tokens < 30:\n",
    "        # Feed forward the word\n",
    "        x = word_to_i[last_token]\n",
    "        x_enc = torch.tensor([x])\n",
    "        x_one_hot = F.one_hot(x_enc, num_classes=vocab_size).float()\n",
    "        logits = x_one_hot @ w\n",
    "        counts = logits.exp()\n",
    "        prob = counts / counts.sum(dim=1, keepdims=True)\n",
    "        # max_values, max_indices = torch.max(prob, dim=1)\n",
    "        # indice_predicted = max_indices[0].item()\n",
    "        # Let's try to sample\n",
    "        indice_predicted = torch.multinomial(prob[0], 1).item()\n",
    "        last_token = i_to_word[indice_predicted]\n",
    "        sentence += last_token\n",
    "        if last_token == '<eos>':\n",
    "            break\n",
    "        num_tokens += 1\n",
    "\n",
    "    print(sentence, num_tokens)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}