Intelligent Earth Homework Assignment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1c9b95ee-9679-4d49-b08a-7f4ca950ce21",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa0bf97d-9de0-466f-88a7-8ae15134fabf",
   "metadata": {},
   "source": [
    "### Initialization\n",
    "Make weight matrices and biases for each layer\n",
    "\n",
    "Create a dictionary object where the `key` is the layer you're in and the `value` is the weight matrix or bias for the fully connected layer\n",
    "\n",
    "Randomly initiate weights (but *scale*) and initiate biases as 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "94f47dc8-56a7-4845-83d1-ccd39e628ba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_dims = [3,2,4,2]\n",
    "\n",
    "weights = {}\n",
    "biases = {}\n",
    "\n",
    "def initialize_weights(dims): # He initialization\n",
    "    for i in range(1, len(dims)):\n",
    "        weights.update({f'W{i}': np.random.randn(dims[i],dims[i-1]) * np.sqrt(2 / dims[i])}) # scaled initialization for ReLU\n",
    "\n",
    "def initialize_biases(dims):\n",
    "    for i in range(1, len(dims)):\n",
    "        biases.update({f'B{i}': np.zeros((dims[i], 1))})\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "8634e379-7536-48dd-ae27-9707edb2000d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Xavier initialization is poor for relu\n",
    "\n",
    "def initialize_params(dims): # He initialization ; for ReLu and Leaky ReLu activations\n",
    "    params = {}\n",
    "    for i in range(1, len(dims)):\n",
    "        params.update({f'W{i}': np.random.randn(dims[i],dims[i-1]) * np.sqrt(2 / dims[i])}) # scaled initialization for ReLU\n",
    "        params.update({f'B{i}': np.zeros((dims[i], 1))})\n",
    "\n",
    "    assert(params['W' + str(i)].shape == (dims[i], dims[i-1])) # double check dimensions are correct\n",
    "    assert(params['B' + str(i)].shape == (dims[i], 1))\n",
    "\n",
    "    return params"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e85bc17a-5a87-4cf0-b7d6-4de5cd6a8aba",
   "metadata": {},
   "source": [
    "### Define activation functions\n",
    "\n",
    "Using ReLu for this assignment. \n",
    "\n",
    "Several others are included which I would like to fill in later for my own understanding/research"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "2c6832c4-d055-4c7e-ab3c-0d7f838fe5d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Args:\n",
    "Z - pre activation parameter ; output of the linear layer\n",
    "\n",
    "Returns:\n",
    "A - post activation parameter ; of the same shape as Z\n",
    "cache - returns Z ; used during backpropagation\n",
    "\"\"\"\n",
    "\n",
    "def sigmoid(Z):\n",
    "    A = 1 / (1 + np.exp(-Z))\n",
    "    cache = Z\n",
    "    return A, cache\n",
    "\n",
    "def sigmoid_backward(dA, cache):\n",
    "    Z = cache\n",
    "    s = 1 / (1 + np.exp(-Z)) # the sigmoid function\n",
    "    dZ = dA * (s * (1-s)) # dA * the derivative of sigmoid. sigmoid derivative = (e ** -x) /(1 + e ** -X) = sig(x) * (1 - sig(x))\n",
    "\n",
    "    assert (dZ.shape == Z.shape)\n",
    "\n",
    "    return dZ\n",
    "\n",
    "def relu(Z):\n",
    "    A = np.maximum(0,Z)\n",
    "    cache = Z\n",
    "    return A, cache\n",
    "\n",
    "def relu_backward(dA, cache):\n",
    "    Z = cache\n",
    "    dZ = np.array(dA, copy=True) # just converting dz to a correct object.\n",
    "    dZ[Z <= 0] = 0 # When z <= 0, you should set dz to 0 as well - think of what ReLu looks like. \n",
    "\n",
    "    assert (dZ.shape == Z.shape) # check if dZ has correct shape\n",
    "    \n",
    "    return dZ\n",
    "\n",
    "def tanh(Z):\n",
    "    A = (np.exp(Z) - np.exp(-Z)) / (np.exp(Z) + np.exp(-Z))\n",
    "    return A, cache\n",
    "\n",
    "def tanh_backward(dA, cache):\n",
    "    Z = cache\n",
    "    t = (np.exp(Z) - np.exp(-Z)) / (np.exp(Z) + np.exp(-Z)) # the tanh function\n",
    "    dZ = dA * (1 - t ** 2) # dA * dervative of tanh. \n",
    "\n",
    "    assert (dZ.shape == Z.shape) # check if dZ has correct shape\n",
    "\n",
    "    return dZ\n",
    "\n",
    "def softplus(Z):\n",
    "    A = np.log(1 + np.exp(Z))\n",
    "    cache = Z\n",
    "    return A, cache\n",
    "\n",
    "def softplus_backward(dA, cache): # define later\n",
    "    pass\n",
    "\n",
    "def softmax(Z): # really only used in the output layer\n",
    "    A = np.exp(Z - np.max(Z, axis=1, keepdims=True)) / np.exp(Z - np.max(Z, axis=1, keepdims=True)).sum(axis=1, keepdims=True) # numerically stable softmax\n",
    "    cache = Z\n",
    "    return A, cache\n",
    "\n",
    "def softmax_backeard(dA, cache): # define later\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c40838e8-819d-439d-805c-439d5a1944c7",
   "metadata": {},
   "source": [
    "### Define a feed forward method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "da4ce042-1148-4980-9d80-45bcd3f4a6e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_forward (A, W, b): # linear transformation\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    A - activations from previous layer: numpy array of shape (size of previous layer, number of examples)\n",
    "    W - weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b - bias vector: numpy array of shape (size of current layer, 1)\n",
    "\n",
    "    Returns:\n",
    "    Z - input of the activation function (pre-activation parameter)\n",
    "    cache - tuple: (A, W, b) ; used for computing backwards pass\n",
    "    \"\"\"\n",
    "    \n",
    "    Z = np.dot(W, A) + b\n",
    "    cache = (A, W, b)\n",
    "\n",
    "    assert(Z.shape == (W.shape[0], A.shape[1])) # double check dimensions are correct\n",
    "\n",
    "    return Z, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "c8a58839-eee1-40c5-bfd4-6cf3802eac68",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_activation_forward(A_prev, weights, biases, activation): # linear transformation with activation function\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    A_prev - activations from previous layer: numpy array of shape (size of previous layer, number of examples)\n",
    "    weights - weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    biases - bias vector: numpy array of shape (size of current layer, 1)\n",
    "    activation - the activation function to be used: string\n",
    "\n",
    "    Returns:\n",
    "    A_new - output of activation function: \n",
    "    cache - tuple: (linear cache, activation cache) ; used for computing backwards pass\n",
    "    \"\"\"\n",
    "\n",
    "    \n",
    "    Z, linear_cache = linear_forward(A_prev, weights, biases)\n",
    "\n",
    "    if activation == \"tanh\":\n",
    "        A_new, activation_cache = tanh(Z)\n",
    "\n",
    "    elif activation == \"sigmoid\":\n",
    "        A_new, activation_cache = sigmoid(Z)\n",
    "\n",
    "    elif activation == \"relu\":\n",
    "        A_new, activation_cache = relu(Z)\n",
    "\n",
    "    elif activation == \"softplus\":\n",
    "        pass\n",
    "\n",
    "    cache = (linear_cache, activation_cache)\n",
    "\n",
    "    return A_new, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "c105d2f6-0185-4206-96ed-c687e6d6f8d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_forward(X, parameters): # feed forward\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    X - data to be modeled: numpy array of shape (input size, number of examples)\n",
    "    parameters - python dictionary containing parameters \"W1\", \"b1\", ..., \"WL\", \"bL\":\n",
    "                    Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n",
    "                    bl -- bias vector of shape (layer_dims[l], 1)\n",
    "\n",
    "    Returns:\n",
    "    A_final - activations from final layer\n",
    "    caches - an indexed list of caches from every layer containing the previous activations, weights, biases, and pre-activation parameters \n",
    "    \"\"\"\n",
    "\n",
    "    caches = []\n",
    "    A = X\n",
    "    L = len(parameters) // 2 # number of layers in the model\n",
    "\n",
    "    for i in range(1, L):\n",
    "        A_prev = A\n",
    "        A, cache = linear_activation_forward(A_prev, parameters[f'W{i}'], parameters[f'B{i}'], \"relu\") # hidden layers # note: relu is hard coded\n",
    "        caches.append(cache)\n",
    "\n",
    "    A_L, cache = linear_activation_forward(A, parameters[f'W{L}'], parameters[f'B{L}'], \"softplus\") # output layer # note: softplus is hard coded\n",
    "    caches.append(cache)     # note: for classification tasks, the final layer usually uses sigmoid or softmax activation. For regression, usually no activation (or a linear activation) is used.\n",
    "\n",
    "    assert(AL.shape == (1,X.shape[1])) # double check dimensions are correct\n",
    "\n",
    "    return A_L, cache"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b705a66-111a-4ff7-befa-70b84e08d282",
   "metadata": {},
   "source": [
    "### Define some loss functions\n",
    "\n",
    "Will be using MSE in this assignment\n",
    "\n",
    "Several others are included which I would like to fill in later for my own understanding/research"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "e88e5c1b-95a3-4cf6-a428-55aefdc7988c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Args:\n",
    "AL -- probability vector corresponding to your label predictions, shape (1, number of examples)\n",
    "Y -- true \"label\" vector (for example: containing 0 if non-cat, 1 if cat), shape (1, number of examples)\n",
    "\n",
    "Returns:\n",
    "cost -- cross-entropy cost\n",
    "\"\"\"\n",
    "\n",
    "def binary_cross_entropy(): # log loss ; classification\n",
    "    m = Y.shape[1]\n",
    "    cost = (-1/m) * (np.dot(Y, np.log(AL).T) + np.dot((1-Y), np.log(1-AL).T)) # using dot product instead of np.sum() becasue it's faster\n",
    "    cost = np.squeeze(cost)      # To make sure your cost's shape is what we expect (e.g. this turns [[17]] into 17).\n",
    "    assert(cost.shape == ())\n",
    "    \n",
    "    return cost\n",
    "\n",
    "def categorical_cross_entropy(): # log loss ; classification\n",
    "    m = Y.shape[1]\n",
    "    cost = (-1/m) * np.sum(np.mulitply(Y, np.log(AL))) # this is unfinished and I'm not sure if it's correct\n",
    "\n",
    "    return cost\n",
    "\n",
    "def neg_log_likelihood(): # for simple classifications\n",
    "    pass\n",
    "\n",
    "def MSE(AL, Y): # mean squared error ; regression\n",
    "    m = Y.shape[1]\n",
    "    cost = (1 / m) * np.dot((AL-Y).flatten(), (AL-Y).flatten()) # using dot product instead of np.sum() because it's faster\n",
    "                                                                # does anything need to be trasposed?\n",
    "    return cost\n",
    "\n",
    "def MAE(): # mean absolute error ; regression\n",
    "    pass\n",
    "\n",
    "def KL_divergence(): # Kullback-Leibler Divergence\n",
    "    pass\n",
    "\n",
    "def Huber(): # Huber loss ; regression\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0cd9081-2399-4b79-bf85-a38ef93d4d70",
   "metadata": {},
   "source": [
    "### Define a backprop method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "4364d1f9-82e5-442d-820c-6044dd4e9e61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# can the autodiff assignment be used here?\n",
    "\n",
    "def linear_backward(dZ, cache):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    dZ - Gradient of the cost with respect to the linear output (of the current layer)\n",
    "    cache - tuple ; (A_prev, W, b) coming from the forward propagation in the current layer\n",
    "\n",
    "    Returns:\n",
    "    dA_prev - Gradient of the cost with respect to the activation (of the previous layer) ; same shape as A_prev\n",
    "    dW - Gradient of the cost with respect to W (of the current layer) ; same shape as W\n",
    "    db - Gradient of the cost with respect to b (of the current layer) ; same shape as b\n",
    "    \"\"\"\n",
    "\n",
    "    A_prev, W, b = cache\n",
    "    m = A_prev.shape[1] # is this the length of entries?\n",
    "\n",
    "    dA_prev = np.dot(W.T, dZ) # what's happening here?\n",
    "    dW = (1/m) * np.dot(dZ, A_prev.T) # what's happening here?\n",
    "    db = (1/m) * np.sum(dZ, axis = 1, keepdims = True) # what's happening here?\n",
    "\n",
    "    assert (dA_prev.shape == A_prev.shape) \n",
    "    assert (dW.shape == W.shape)\n",
    "    assert (db.shape == b.shape)\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "cc926dcd-119f-47dd-81b5-81a803930193",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_activation_backward(dA, cache, activation):\n",
    "    \"\"\"\n",
    "    activation function takes input Z and returns A and an activation cache (Z) ; \n",
    "    activation_backward function takes input dA and the activation cache (Z) and returns dZ.\n",
    "    \n",
    "    Args:\n",
    "    dA - post-activation gradient for current layer\n",
    "    cache - tuple ; (linear_cache, activation_cache) we store for computing backward propagation efficiently\n",
    "    activation - activation function used in this layer\n",
    "\n",
    "    Returns:\n",
    "    dA_prev - Gradient of the cost with respect to the activation (of the previous layer) ; same shape as A_prev\n",
    "    dW - Gradient of the cost with respect to W (of the current layer) ; same shape as W\n",
    "    db - Gradient of the cost with respect to b (of the current layer) ; same shape as b\n",
    "    \"\"\"\n",
    "\n",
    "    linear_cache, activation_cache = cache\n",
    "\n",
    "    if activation == \"relu\": # relu activation takes input Z and returns A and an activation cache (Z) ; relu_backward takes input dA and the activation cache (Z) and returns dZ\n",
    "        dZ = relu_backward(dA, activation_cache)\n",
    "\n",
    "    elif activation == \"tanh\":\n",
    "        dZ = tanh_backward(dA, activation_cache)\n",
    "\n",
    "    elif activation == \"sigmoid\":\n",
    "        dZ = sigmoid_backward(dA, activation_cache)\n",
    "\n",
    "    dA_prev, dW, b = linear_backward(dZ, linear_cache)\n",
    "\n",
    "    return dA_prev, dW, b"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ab896ff-13b3-447e-aad2-1cd67e2351a5",
   "metadata": {},
   "source": [
    "### Check This\n",
    "\n",
    "this block was the hardest to understand how it's working – `go through and make sure everything is correct.`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "fabc9a2b-e50a-48b5-bc40-93f2fb45c632",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_backward(AL, Y, caches):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "    AL - probability vector, output of the forward propagation (L_model_forward())\n",
    "    Y - true \"label\" vector\n",
    "    caches - list of caches containing:\n",
    "                every cache of linear_activation_forward() with \"relu\" (it's caches[l], for l in range(L-1) i.e l = 0...L-2)\n",
    "                the cache of linear_activation_forward() with \"sigmoid\" (it's caches[L-1])\n",
    "                how would this work for tanh? softplus?\n",
    "\n",
    "    Returns:\n",
    "    grads - dictionary ; gradients with respect to activations, weights, biases...\n",
    "                         grads[\"dA\" + str(l)] = ... \n",
    "                         grads[\"dW\" + str(l)] = ...\n",
    "                         grads[\"db\" + str(l)] = ... \n",
    "    \"\"\"\n",
    "\n",
    "    grads = {}\n",
    "    L = length(caches) # the number of layers\n",
    "    m = AL.shape[1]\n",
    "    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n",
    "\n",
    "    dAL = np.divide(Y, AL) - np.divide(1 - Y, 1 - AL) # initialize backprop\n",
    "\n",
    "    current_cache = caches[L-1] # Last Layer\n",
    "    grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(\n",
    "        dAL, \n",
    "        current_cache, \n",
    "        activation = \"relu\") # note: relu is hard coded. This is the last (output) layer\n",
    "\n",
    "    for i in reversed(range(L-1)): # Loop from l=L-2 to l=0\n",
    "        current_cache = caches[i]\n",
    "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(\n",
    "            grads[\"dA\" + str(i+1)],\n",
    "            current_cache,\n",
    "            activation = \"relu\") # note: relu is hard coded. These are the hidden layers\n",
    "\n",
    "        grads[\"dA\" + str(i)] = dA_prev_temp\n",
    "        grads[\"dW\" + str(i+1)] = dW_temp\n",
    "        grads[\"db\" + str(i+1)] = db_temp\n",
    "\n",
    "    return grads"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81f7971f-dc5f-4104-a258-11354d61e59f",
   "metadata": {},
   "source": [
    "### Update Parameters\n",
    "which gradient descent update rule to use? \n",
    "\n",
    "which learning rate to use?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "13a0c8eb-2478-409d-b5e2-97c320cbf1d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def update_params(params, grads, learning_rate):\n",
    "    \"\"\"\n",
    "    generally we update by: new_params = params - grads * learning_rate\n",
    "    Args:\n",
    "    params - dict ; contains the (previous) parameters\n",
    "    grads - dict ; contains the graduients calculated from backprop\n",
    "    learning_rate - step size for gradient descent\n",
    "    \n",
    "    Returns:\n",
    "    params - dict ; the updated parameters\n",
    "                parameters[\"W\" + str(l)] = ... \n",
    "                parameters[\"b\" + str(l)] = ...\n",
    "    \"\"\"\n",
    "\n",
    "    L = len(params) // 2 # params dict contains entries for both weights and biases\n",
    "\n",
    "    for i in range(L):\n",
    "        params[\"W\" + str(i+1)] = params[\"W\" + str(i+1)] - grads[\"dW\" + str(i+1)] * learning_rate\n",
    "        params[\"b\" + str(i+1)] = params[\"b\" + str(i+1)] - grads[\"db\" + str(i+1)] * learning_rate\n",
    "\n",
    "    return params"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "365b138e-8e36-4e1c-b1c5-7dc34c395bc2",
   "metadata": {},
   "source": [
    "### Define an optimizer "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21fdf9be-76a5-45aa-859e-2c850ad66ac7",
   "metadata": {},
   "source": [
    "see https://realpython.com/gradient-descent-algorithm-python/    (implementation of Basic Gradient Descent)\n",
    "\n",
    "gradient function = model_backward(): this returns the gradients\n",
    "\n",
    "start could be params?\n",
    "\n",
    "learning rate = learning rate\n",
    "\n",
    "n_iter?\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79ef19e9-eadd-4b15-91b5-14ce78ec0cc5",
   "metadata": {},
   "source": [
    "### Build this into a NN class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52ec4bcb-8adb-421d-9d26-45f9b04f599e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}