⚡ Phase 4 — NLP & Transformers (Days 56–70)

# Before any text reaches a neural network, it must become numbers.
# Tokenization: splitting text into units (words, subwords, or characters)
 
from transformers import AutoTokenizer
 
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
tokens = tokenizer("Transformers are powerful", return_tensors='pt')
print(tokenizer.convert_ids_to_tokens(tokens['input_ids'][0]))
# ['[CLS]', 'transformers', 'are', 'powerful', '[SEP]']
 
# Subword tokenization (BPE - Byte Pair Encoding, used by GPT):
# Solves the out-of-vocabulary problem -- "unhappiness" might split into
# ["un", "happiness"] even if "unhappiness" was never seen as a whole word
# This is WHY LLMs can handle typos, rare words, and even code reasonably well
 
# BPE algorithm (conceptual): start with characters, repeatedly merge the
# most frequent adjacent pair into a new token, until you reach vocab size
 
# Embeddings: map a token ID to a dense vector
import torch.nn as nn
embedding_layer = nn.Embedding(num_embeddings=50000, embedding_dim=768)
# vocab_size=50000 (BPE vocab), embedding_dim=768 (GPT-2 small uses this)
 
# Word2Vec intuition (the original embedding idea, 2013):
# Words that appear in similar CONTEXTS get similar vectors
# "king" - "man" + "woman" ~= "queen" (the famous analogy, works because
# embeddings capture semantic relationships as geometric directions)

import torch
import torch.nn.functional as F
import math
 
# Self-attention answers: "for THIS token, how much should I attend to EVERY
# other token in the sequence to understand its meaning in context?"
#
# Example: "The animal didn't cross the street because IT was too tired"
# "it" should attend strongly to "animal", not "street"
# Self-attention LEARNS this from data, with no hand-coded rules
 
def scaled_dot_product_attention(Q, K, V, mask=None):
    """
    Q (query):  what am I looking for?
    K (key):    what do I contain? (compared against queries)
    V (value):  what information do I actually pass along?
 
    Each token produces a Q, K, V vector via learned linear projections.
    """
    d_k = Q.size(-1)
    # Attention scores: how much does each query "match" each key
    scores = Q @ K.transpose(-2, -1) / math.sqrt(d_k)  # scale prevents huge values
 
    if mask is not None:
        scores = scores.masked_fill(mask == 0, float('-inf'))  # causal masking
 
    # Softmax: convert scores to a probability distribution (attention weights)
    attention_weights = F.softmax(scores, dim=-1)
 
    # Weighted sum of values, weighted by attention
    output = attention_weights @ V
    return output, attention_weights
 
# A minimal self-attention layer
class SelfAttention(nn.Module):
    def __init__(self, embed_dim):
        super().__init__()
        self.W_q = nn.Linear(embed_dim, embed_dim)
        self.W_k = nn.Linear(embed_dim, embed_dim)
        self.W_v = nn.Linear(embed_dim, embed_dim)
 
    def forward(self, x, mask=None):
        Q = self.W_q(x)
        K = self.W_k(x)
        V = self.W_v(x)
        output, weights = scaled_dot_product_attention(Q, K, V, mask)
        return output
 
# Causal masking: for language models (GPT-style), a token can only attend
# to PREVIOUS tokens, never future ones (otherwise it would "cheat" during training)
def causal_mask(seq_len):
    return torch.tril(torch.ones(seq_len, seq_len))  # lower triangular matrix

class MultiHeadAttention(nn.Module):
    """
    Instead of one attention computation, run H attention computations in
    parallel (each with its own Q/K/V projections), then combine.
    Each "head" can learn to focus on different types of relationships
    (e.g., one head for syntax, another for coreference, another for topic).
    """
    def __init__(self, embed_dim, num_heads):
        super().__init__()
        self.num_heads = num_heads
        self.head_dim = embed_dim // num_heads
        self.W_q = nn.Linear(embed_dim, embed_dim)
        self.W_k = nn.Linear(embed_dim, embed_dim)
        self.W_v = nn.Linear(embed_dim, embed_dim)
        self.W_o = nn.Linear(embed_dim, embed_dim)
 
    def forward(self, x, mask=None):
        B, T, C = x.shape  # batch, sequence length, embed_dim
        Q = self.W_q(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
        K = self.W_k(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
        V = self.W_v(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
 
        out, _ = scaled_dot_product_attention(Q, K, V, mask)
        out = out.transpose(1, 2).contiguous().view(B, T, C)
        return self.W_o(out)
 
class TransformerBlock(nn.Module):
    """A single Transformer block: the repeating unit stacked N times in GPT/BERT."""
    def __init__(self, embed_dim, num_heads, ff_dim):
        super().__init__()
        self.attn = MultiHeadAttention(embed_dim, num_heads)
        self.ln1 = nn.LayerNorm(embed_dim)
        self.ff = nn.Sequential(
            nn.Linear(embed_dim, ff_dim),
            nn.GELU(),
            nn.Linear(ff_dim, embed_dim)
        )
        self.ln2 = nn.LayerNorm(embed_dim)
 
    def forward(self, x, mask=None):
        # Residual connections ("x +") are CRITICAL for training deep networks
        # -- they give gradients a direct path backward, combating vanishing gradients
        x = x + self.attn(self.ln1(x), mask)   # pre-norm + attention + residual
        x = x + self.ff(self.ln2(x))            # pre-norm + feedforward + residual
        return x
 
# Positional encoding: attention has NO inherent sense of order
# (unlike RNNs which process sequentially). We must INJECT position information.
def sinusoidal_positional_encoding(seq_len, embed_dim):
    position = torch.arange(seq_len).unsqueeze(1)
    div_term = torch.exp(torch.arange(0, embed_dim, 2) * -(math.log(10000.0) / embed_dim))
    pe = torch.zeros(seq_len, embed_dim)
    pe[:, 0::2] = torch.sin(position * div_term)
    pe[:, 1::2] = torch.cos(position * div_term)
    return pe

# BERT (encoder-only): bidirectional attention, sees the WHOLE sequence at once
# Trained on: Masked Language Modeling (predict randomly masked words using
#   BOTH left and right context) + Next Sentence Prediction
# Best for: understanding tasks -- classification, NER, sentence similarity
# Cannot generate text naturally (no causal masking, sees the future)
 
# GPT (decoder-only): causal (masked) attention, only sees PREVIOUS tokens
# Trained on: Next Token Prediction (predict the next word given everything before it)
# Best for: generation tasks -- chat, completion, summarization, code generation
# This is the architecture behind ChatGPT, Claude, and virtually all modern LLMs
 
# Original Transformer (encoder-decoder): used for translation
# Encoder processes the source language (bidirectional)
# Decoder generates the target language (causal, with cross-attention to encoder)
# Less common now for general LLMs; T5 and original translation models use this
 
from transformers import AutoModel, AutoModelForCausalLM, AutoModelForSequenceClassification
 
# BERT for classification
bert = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)
 
# GPT-2 for text generation
gpt2 = AutoModelForCausalLM.from_pretrained('gpt2')
input_ids = tokenizer.encode("The future of AI is", return_tensors='pt')
output = gpt2.generate(input_ids, max_length=50, do_sample=True, temperature=0.8)
print(tokenizer.decode(output[0]))

Project 1: GPT-style Transformer from scratch (Karpathy's "nanoGPT" approach)
  - Implement: token embedding + positional encoding + N transformer blocks
    + final linear layer to vocabulary logits
  - Train on a small text corpus (character or small-vocab BPE level)
  - Implement causal masking correctly -- verify by checking the model
    can't "cheat" (test: does changing a FUTURE token change a PAST prediction?
    It should NOT.)
  - Generate text with temperature and top-k sampling
  - Compare quality/coherence to your Phase 3 LSTM language model
 
Project 2: Fine-tune a pretrained BERT for a real task
  - Pick a task: sentiment classification, NER, or text similarity
  - Use Hugging Face Transformers + a small labeled dataset
  - Fine-tune (not feature extraction -- actually update BERT's weights)
  - Evaluate with appropriate metrics (F1 for NER, accuracy for classification)
  - Document: how much did fine-tuning improve over zero-shot/frozen BERT?