Tokenization

import numpy as np

class Perceptron:
    def __init__(self, dim, lr=0.1):
        self.w = np.zeros(dim); self.b = 0.0; self.lr = lr

    def predict(self, x): return int(x @ self.w + self.b >= 0)

    def fit(self, X, y, epochs=100):
        for _ in range(epochs):
            for xi, yi in zip(X, y):
                pred = self.predict(xi)
                # Perceptron learning rule: update on mistakes
                self.w += self.lr * (yi - pred) * xi
                self.b += self.lr * (yi - pred)

# Linearly separable AND gate: works perfectly
X = np.array([[0,0],[0,1],[1,0],[1,1]])
y_and = np.array([0,0,0,1])
p = Perceptron(2); p.fit(X, y_and)
print("AND:", [p.predict(x) for x in X])  # [0, 0, 0, 1] correct

# XOR: never converges -- not linearly separable
y_xor = np.array([0,1,1,0])
p2 = Perceptron(2); p2.fit(X, y_xor, epochs=1000)
print("XOR:", [p2.predict(x) for x in X])  # wrong, no matter how long
# A single perceptron CANNOT represent XOR. We need a hidden layer.

h  =  σ(W₁ x + b₁)        # hidden layer  (D → H)
y  =  W₂ h + b₂           # output layer  (H → K)

σ = nonlinear activation (ReLU, GELU, …)

import numpy as np

np.random.seed(0)
X = np.array([[0,0],[0,1],[1,0],[1,1]], dtype=float)
y = np.array([[0],[1],[1],[0]], dtype=float)

# 2 -> 4 -> 1 MLP with ReLU hidden and sigmoid output
W1 = np.random.randn(2, 4) * 0.5; b1 = np.zeros(4)
W2 = np.random.randn(4, 1) * 0.5; b2 = np.zeros(1)

def sigmoid(z): return 1 / (1 + np.exp(-z))

for epoch in range(5000):
    # Forward
    h    = np.maximum(0, X @ W1 + b1)       # ReLU hidden
    out  = sigmoid(h @ W2 + b2)              # sigmoid output
    # Backward (binary cross-entropy)
    dout = (out - y) / len(X)                # dL/d(out·sig)
    dW2  = h.T @ dout;  db2 = dout.sum(0)
    dh   = (dout @ W2.T) * (h > 0)         # ReLU gradient
    dW1  = X.T @ dh;   db1 = dh.sum(0)
    # Update
    for p, g in [(W1,dW1),(b1,db1),(W2,dW2),(b2,db2)]: p -= 0.5 * g

preds = (sigmoid(np.maximum(0, X@W1+b1)@W2+b2) > 0.5).astype(int)
print("XOR solved:", preds.ravel())  # [0 1 1 0] -- correct!
# The hidden layer made XOR linearly separable for the output neuron.

ReLU(x)  =  max(0, x)
ReLU'(x) =  1 if x > 0,  else 0      # no attenuation for active units

GELU(x)   =  x · Φ(x)  ≈  0.5x(1 + tanh[√(2/π)(x + 0.044715x³)])

SwiGLU(x) =  Swish(xW + b) ⊙ (xV + c)     # gated, 3 matrices
Swish(x)  =  x · σ(βx)

import numpy as np

def sigmoid(x):  s = 1/(1+np.exp(-x)); return s, s*(1-s)
def tanh(x):     t = np.tanh(x);       return t, 1-t**2
def relu(x):                          return np.maximum(0,x), (x>0).astype(float)
def leaky_relu(x, a=0.01):       return np.where(x>0,x,a*x), np.where(x>0,1,a)

def gelu(x):  # tanh approximation (used in GPT-2)
    c = np.sqrt(2/np.pi)
    inner = c*(x + 0.044715*x**3)
    val = 0.5*x*(1+np.tanh(inner))
    # derivative omitted for brevity; use autograd in practice
    return val

def swish(x, beta=1.0):  return x*sigmoid(beta*x)[0]

# Key property: max gradient of each activation
x = np.linspace(-5, 5, 1000)
print(f"sigmoid max grad: {sigmoid(x)[1].max():.3f}")  # 0.25 -> vanishing
print(f"tanh    max grad: {tanh(x)[1].max():.3f}")     # 1.00 -> better
print(f"relu    max grad: {relu(x)[1].max():.3f}")     # 1.00 -> no attenuation

Var(y_i) = n · Var(W) · Var(x)        # forward pass variance

Xavier/Glorot:  Var(W) = 2/(n_in + n_out)    # balances fwd & bwd, for tanh
He/Kaiming:     Var(W) = 2/n_in              # accounts for ReLU zeroing half

import numpy as np

def test_init(init_fn, depth=50, width=256):
    """Pass a signal through `depth` ReLU layers; track activation std."""
    x = np.random.randn(1000, width)  # batch of 1000
    stds = [x.std()]
    for _ in range(depth):
        W = init_fn(width, width)
        x = np.maximum(0, x @ W)       # ReLU layer
        stds.append(x.std())
    return stds

# Too small: signal vanishes
tiny  = lambda i,o: np.random.randn(i,o) * 0.01
# Too large: signal explodes
huge  = lambda i,o: np.random.randn(i,o) * 1.0
# He init: signal preserved
he    = lambda i,o: np.random.randn(i,o) * np.sqrt(2.0/i)

print(f"tiny: layer 50 std = {test_init(tiny)[-1]:.2e}")  # ~1e-30 vanished
print(f"huge: layer 50 std = {test_init(huge)[-1]:.2e}")  # ~1e+10 exploded
print(f"He:   layer 50 std = {test_init(he)[-1]:.2e}")    # ~1.0 stable!

μ_B = (1/m) Σᵢ xᵢ           # batch mean
σ²_B = (1/m) Σᵢ (xᵢ - μ_B)²   # batch variance
x̂ᵢ = (xᵢ - μ_B) / √(σ²_B + ε)  # normalize
yᵢ = γ x̂ᵢ + β              # learned scale & shift

LayerNorm:  y = γ · (x - μ) / √(σ² + ε) + β     # center AND scale
RMSNorm:    y = γ · x / √(mean(x²) + ε)          # scale only, no centering

import numpy as np

def batch_norm(x, gamma, beta, eps=1e-5):  # x: (batch, features)
    mu  = x.mean(axis=0, keepdims=True)       # across BATCH
    var = x.var(axis=0, keepdims=True)
    return gamma * (x - mu) / np.sqrt(var + eps) + beta

def layer_norm(x, gamma, beta, eps=1e-5):  # x: (batch, features)
    mu  = x.mean(axis=1, keepdims=True)       # across FEATURES
    var = x.var(axis=1, keepdims=True)
    return gamma * (x - mu) / np.sqrt(var + eps) + beta

def rms_norm(x, gamma, eps=1e-5):  # no mean-centering, no beta
    rms = np.sqrt((x**2).mean(axis=1, keepdims=True) + eps)
    return gamma * x / rms

x = np.random.randn(4, 768) * 10 + 5  # large-mean activations
g = np.ones(768); b = np.zeros(768)

ln = layer_norm(x, g, b)
print(f"LayerNorm: per-row mean={ln.mean(1).mean():.4f}, std={ln.std(1).mean():.4f}")
# LayerNorm: per-row mean=0.0000, std=1.0000

rn = rms_norm(x, g)
print(f"RMSNorm:   per-row RMS={np.sqrt((rn**2).mean(1)).mean():.4f}")
# RMSNorm:   per-row RMS=1.0000 (but mean is NOT zeroed)

mask ~ Bernoulli(1 - p)           # 1 with prob (1-p), else 0
y = (x ⊙ mask) / (1 - p)           # zero some, scale up the rest

# At inference: y = x  (no mask, no scaling)

import numpy as np

def dropout(x, p=0.5, training=True):
    """Inverted dropout. p = fraction to drop."""
    if not training or p == 0:
        return x                          # inference: identity
    mask = (np.random.rand(*x.shape) > p)  # keep with prob (1-p)
    return x * mask / (1 - p)           # zero & scale up survivors

# Expected value is preserved
x = np.ones(10000)
y = dropout(x, p=0.5, training=True)
print(f"Input mean:  {x.mean():.3f}")   # 1.000
print(f"Dropout mean: {y.mean():.3f}")  # ~1.000 (preserved by 1/(1-p) scaling)
print(f"Fraction zeroed: {(y==0).mean():.3f}")  # ~0.500

import numpy as np

class MLP:
    """Multi-layer perceptron with He init, ReLU, dropout, and softmax-CE."""
    def __init__(self, sizes, dropout=0.0, seed=0):
        rng = np.random.default_rng(seed)
        self.W, self.b = [], []
        for nin, nout in zip(sizes[:-1], sizes[1:]):
            self.W.append(rng.normal(0, np.sqrt(2/nin), (nin, nout)))  # He init
            self.b.append(np.zeros(nout))
        self.dropout = dropout

    def forward(self, x, training=True):
        self.cache = [x]
        for i, (W, b) in enumerate(zip(self.W, self.b))):
            z = x @ W + b
            if i < len(self.W) - 1:  # hidden layers
                x = np.maximum(0, z)        # ReLU
                if training and self.dropout > 0:
                    mask = (np.random.rand(*x.shape) > self.dropout) / (1 - self.dropout)
                    x = x * mask
            else:  # output layer: stable softmax
                z -= z.max(axis=1, keepdims=True)
                x = np.exp(z); x /= x.sum(axis=1, keepdims=True)
            self.cache.append(x)
        return x

    def loss_and_grad(self, probs, y):  # y: integer labels
        N = len(y)
        loss = -np.log(probs[np.arange(N), y] + 1e-12).mean()
        grad = probs.copy(); grad[np.arange(N), y] -= 1; grad /= N  # softmax-CE grad
        return loss, grad

# This MLP -- linear layers + ReLU + dropout + softmax-CE -- contains
# every concept needed to understand a Transformer's feed-forward network.

import numpy as np
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split

X, y = fetch_openml('mnist_784', return_X_y=True, as_frame=False)
X = X / 255.0; y = y.astype(int)  # normalize to [0,1]
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, random_state=0)

net = MLP([784, 256, 128, 10], dropout=0.2)
lr, batch = 0.1, 128

for epoch in range(10):
    perm = np.random.permutation(len(Xtr))
    for i in range(0, len(Xtr), batch):
        idx = perm[i:i+batch]
        probs = net.forward(Xtr[idx], training=True)
        loss, grad = net.loss_and_grad(probs, ytr[idx])
        net.backward(grad, lr)            # (backward + SGD update)
    # Diagnostics each epoch
    test_acc = (net.forward(Xte, training=False).argmax(1) == yte).mean()
    grad_norm = np.sqrt(sum((g**2).sum() for g in net.last_grads))
    print(f"epoch {epoch}: loss={loss:.3f} test_acc={test_acc:.3f} |grad|={grad_norm:.2f}")
# epoch 0: loss=0.412 test_acc=0.928 |grad|=1.84
# epoch 9: loss=0.087 test_acc=0.978 |grad|=0.43  -- healthy convergence