{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Useful starting lines\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Load test module for sanity check\n",
    "from test_utils import test"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Logistic Regression\n",
    "## Classification Using Linear Regression\n",
    "Load your data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import sample_data, load_data, standardize\n",
    "\n",
    "# load data.\n",
    "height, weight, gender = load_data()\n",
    "\n",
    "# build sampled x and y.\n",
    "seed = 1\n",
    "y = np.expand_dims(gender, axis=1)\n",
    "X = np.c_[height.reshape(-1), weight.reshape(-1)]\n",
    "y, X = sample_data(y, X, seed, size_samples=200)\n",
    "x, mean_x, std_x = standardize(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Use `least_squares` to compute w, and visualize the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from least_squares import least_squares\n",
    "from plots import visualization\n",
    "\n",
    "\n",
    "def least_square_classification_demo(y, x):\n",
    "    \"\"\"Least square demo\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        x:  shape=(N, 2)\n",
    "    \"\"\"\n",
    "    # classify the data by linear regression\n",
    "    tx = np.c_[np.ones((y.shape[0], 1)), x]\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # classify the data by linear regression: TODO\n",
    "    # ***************************************************\n",
    "    # w = least squares with respect to tx and y\n",
    "\n",
    "    # visualize your classification.\n",
    "    visualization(y, x, mean_x, std_x, w, \"classification_by_least_square\")\n",
    "\n",
    "\n",
    "least_square_classification_demo(y, x)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "#### The `least_square_classification_demo` is expected to show\n",
    "\n",
    "![1](./classification_by_least_square.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Compute your cost by negative log likelihood."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(t):\n",
    "    \"\"\"apply sigmoid function on t.\n",
    "\n",
    "    Args:\n",
    "        t: scalar or numpy array\n",
    "\n",
    "    Returns:\n",
    "        scalar or numpy array\n",
    "\n",
    "    >>> sigmoid(np.array([0.1]))\n",
    "    array([0.52497919])\n",
    "    >>> sigmoid(np.array([0.1, 0.1]))\n",
    "    array([0.52497919, 0.52497919])\n",
    "    \"\"\"\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "test(sigmoid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_loss(y, tx, w):\n",
    "    \"\"\"compute the cost by negative log likelihood.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "\n",
    "    Returns:\n",
    "        a non-negative loss\n",
    "\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(4).reshape(2, 2)\n",
    "    >>> w = np.c_[[2., 3.]]\n",
    "    >>> round(calculate_loss(y, tx, w), 8)\n",
    "    1.52429481\n",
    "    \"\"\"\n",
    "    assert y.shape[0] == tx.shape[0]\n",
    "    assert tx.shape[1] == w.shape[0]\n",
    "\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "test(calculate_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_gradient(y, tx, w):\n",
    "    \"\"\"compute the gradient of loss.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "\n",
    "    Returns:\n",
    "        a vector of shape (D, 1)\n",
    "\n",
    "    >>> np.set_printoptions(8)\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(6).reshape(2, 3)\n",
    "    >>> w = np.array([[0.1], [0.2], [0.3]])\n",
    "    >>> calculate_gradient(y, tx, w)\n",
    "    array([[-0.10370763],\n",
    "           [ 0.2067104 ],\n",
    "           [ 0.51712843]])\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError(\"Calculate gradient\")\n",
    "\n",
    "\n",
    "test(calculate_gradient)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Using Gradient Descent\n",
    "Implement your function to calculate the gradient for logistic regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learning_by_gradient_descent(y, tx, w, gamma):\n",
    "    \"\"\"\n",
    "    Do one step of gradient descent using logistic regression. Return the loss and the updated w.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "        gamma: float\n",
    "\n",
    "    Returns:\n",
    "        loss: scalar number\n",
    "        w: shape=(D, 1)\n",
    "\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(6).reshape(2, 3)\n",
    "    >>> w = np.array([[0.1], [0.2], [0.3]])\n",
    "    >>> gamma = 0.1\n",
    "    >>> loss, w = learning_by_gradient_descent(y, tx, w, gamma)\n",
    "    >>> round(loss, 8)\n",
    "    0.62137268\n",
    "    >>> w\n",
    "    array([[0.11037076],\n",
    "           [0.17932896],\n",
    "           [0.24828716]])\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "test(learning_by_gradient_descent)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Demo!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import de_standardize\n",
    "\n",
    "\n",
    "def logistic_regression_gradient_descent_demo(y, x):\n",
    "    # init parameters\n",
    "    max_iter = 10000\n",
    "    threshold = 1e-8\n",
    "    gamma = 0.5\n",
    "    losses = []\n",
    "\n",
    "    # build tx\n",
    "    tx = np.c_[np.ones((y.shape[0], 1)), x]\n",
    "    w = np.zeros((tx.shape[1], 1))\n",
    "\n",
    "    # start the logistic regression\n",
    "    for iter in range(max_iter):\n",
    "        # get loss and update w.\n",
    "        loss, w = learning_by_gradient_descent(y, tx, w, gamma)\n",
    "        # log info\n",
    "        if iter % 100 == 0:\n",
    "            print(\"Current iteration={i}, loss={l}\".format(i=iter, l=loss))\n",
    "        # converge criterion\n",
    "        losses.append(loss)\n",
    "        if len(losses) > 1 and np.abs(losses[-1] - losses[-2]) < threshold:\n",
    "            break\n",
    "    # visualization\n",
    "    visualization(\n",
    "        y,\n",
    "        x,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        w,\n",
    "        \"classification_by_logistic_regression_gradient_descent\",\n",
    "        True,\n",
    "    )\n",
    "    print(\"loss={l}\".format(l=calculate_loss(y, tx, w)))\n",
    "\n",
    "\n",
    "logistic_regression_gradient_descent_demo(y, x)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "#### The `logistic_regression_gradient_descent_demo` is expected to show\n",
    "\n",
    "![1](./classification_by_logistic_regression_gradient_descent.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Calculate your hessian below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_hessian(y, tx, w):\n",
    "    \"\"\"return the Hessian of the loss function.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "\n",
    "    Returns:\n",
    "        a hessian matrix of shape=(D, D)\n",
    "\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(6).reshape(2, 3)\n",
    "    >>> w = np.array([[0.1], [0.2], [0.3]])\n",
    "    >>> calculate_hessian(y, tx, w)\n",
    "    array([[0.28961235, 0.3861498 , 0.48268724],\n",
    "           [0.3861498 , 0.62182124, 0.85749269],\n",
    "           [0.48268724, 0.85749269, 1.23229813]])\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # calculate Hessian: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "test(calculate_hessian)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "Write a function below to return loss, gradient, and hessian."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logistic_regression(y, tx, w):\n",
    "    \"\"\"return the loss, gradient of the loss, and hessian of the loss.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "\n",
    "    Returns:\n",
    "        loss: scalar number\n",
    "        gradient: shape=(D, 1)\n",
    "        hessian: shape=(D, D)\n",
    "\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(6).reshape(2, 3)\n",
    "    >>> w = np.array([[0.1], [0.2], [0.3]])\n",
    "    >>> loss, gradient, hessian = logistic_regression(y, tx, w)\n",
    "    >>> round(loss, 8)\n",
    "    0.62137268\n",
    "    >>> gradient, hessian\n",
    "    (array([[-0.10370763],\n",
    "           [ 0.2067104 ],\n",
    "           [ 0.51712843]]), array([[0.28961235, 0.3861498 , 0.48268724],\n",
    "           [0.3861498 , 0.62182124, 0.85749269],\n",
    "           [0.48268724, 0.85749269, 1.23229813]]))\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # return loss, gradient, and Hessian: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "test(logistic_regression)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Using Newton's method\n",
    "Use Newton's method for logistic regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learning_by_newton_method(y, tx, w, gamma):\n",
    "    \"\"\"\n",
    "    Do one step of Newton's method.\n",
    "    Return the loss and updated w.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "        gamma: scalar\n",
    "\n",
    "    Returns:\n",
    "        loss: scalar number\n",
    "        w: shape=(D, 1)\n",
    "\n",
    "    >>> y = np.c_[[0., 0., 1., 1.]]\n",
    "    >>> np.random.seed(0)\n",
    "    >>> tx = np.random.rand(4, 3)\n",
    "    >>> w = np.array([[0.1], [0.5], [0.5]])\n",
    "    >>> gamma = 0.1\n",
    "    >>> loss, w = learning_by_newton_method(y, tx, w, gamma)\n",
    "    >>> round(loss, 8)\n",
    "    0.71692036\n",
    "    >>> w\n",
    "    array([[-1.31876014],\n",
    "           [ 1.0590277 ],\n",
    "           [ 0.80091466]])\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # return loss, gradient and Hessian: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # update w: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "    return loss, w\n",
    "\n",
    "\n",
    "test(learning_by_newton_method)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logistic_regression_newton_method_demo(y, x):\n",
    "    # init parameters\n",
    "    max_iter = 100\n",
    "    threshold = 1e-8\n",
    "    lambda_ = 0.1\n",
    "    gamma = 1.0\n",
    "    losses = []\n",
    "\n",
    "    # build tx\n",
    "    tx = np.c_[np.ones((y.shape[0], 1)), x]\n",
    "    w = np.zeros((tx.shape[1], 1))\n",
    "\n",
    "    # start the logistic regression\n",
    "    for iter in range(max_iter):\n",
    "        # get loss and update w.\n",
    "        loss, w = learning_by_newton_method(y, tx, w, gamma)\n",
    "        # log info\n",
    "        if iter % 1 == 0:\n",
    "            print(\"Current iteration={i}, the loss={l}\".format(i=iter, l=loss))\n",
    "\n",
    "        # converge criterion\n",
    "        losses.append(loss)\n",
    "        if len(losses) > 1 and np.abs(losses[-1] - losses[-2]) < threshold:\n",
    "            break\n",
    "    # visualization\n",
    "    visualization(\n",
    "        y,\n",
    "        x,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        w,\n",
    "        \"classification_by_logistic_regression_newton_method\",\n",
    "        True,\n",
    "    )\n",
    "    print(\"loss={l}\".format(l=calculate_loss(y, tx, w)))\n",
    "\n",
    "\n",
    "logistic_regression_newton_method_demo(y, x)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "#### The `logistic_regression_newton_method_demo` is expected to show\n",
    "\n",
    "![1](./classification_by_logistic_regression_newton_method.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "### Using penalized logistic regression\n",
    "Fill in the function below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def penalized_logistic_regression(y, tx, w, lambda_):\n",
    "    \"\"\"return the loss and gradient.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "        lambda_: scalar\n",
    "\n",
    "    Returns:\n",
    "        loss: scalar number\n",
    "        gradient: shape=(D, 1)\n",
    "\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(6).reshape(2, 3)\n",
    "    >>> w = np.array([[0.1], [0.2], [0.3]])\n",
    "    >>> lambda_ = 0.1\n",
    "    >>> loss, gradient = penalized_logistic_regression(y, tx, w, lambda_)\n",
    "    >>> round(loss, 8)\n",
    "    0.62137268\n",
    "    >>> gradient\n",
    "    array([[-0.08370763],\n",
    "           [ 0.2467104 ],\n",
    "           [ 0.57712843]])\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # return loss, gradient, and Hessian: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "\n",
    "\n",
    "test(penalized_logistic_regression)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learning_by_penalized_gradient(y, tx, w, gamma, lambda_):\n",
    "    \"\"\"\n",
    "    Do one step of gradient descent, using the penalized logistic regression.\n",
    "    Return the loss and updated w.\n",
    "\n",
    "    Args:\n",
    "        y:  shape=(N, 1)\n",
    "        tx: shape=(N, D)\n",
    "        w:  shape=(D, 1)\n",
    "        gamma: scalar\n",
    "        lambda_: scalar\n",
    "\n",
    "    Returns:\n",
    "        loss: scalar number\n",
    "        w: shape=(D, 1)\n",
    "\n",
    "    >>> np.set_printoptions(8)\n",
    "    >>> y = np.c_[[0., 1.]]\n",
    "    >>> tx = np.arange(6).reshape(2, 3)\n",
    "    >>> w = np.array([[0.1], [0.2], [0.3]])\n",
    "    >>> lambda_ = 0.1\n",
    "    >>> gamma = 0.1\n",
    "    >>> loss, w = learning_by_penalized_gradient(y, tx, w, gamma, lambda_)\n",
    "    >>> round(loss, 8)\n",
    "    0.62137268\n",
    "    >>> w\n",
    "    array([[0.10837076],\n",
    "           [0.17532896],\n",
    "           [0.24228716]])\n",
    "    \"\"\"\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # return loss, gradient: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "    # ***************************************************\n",
    "    # INSERT YOUR CODE HERE\n",
    "    # update w: TODO\n",
    "    # ***************************************************\n",
    "    raise NotImplementedError\n",
    "    return loss, w\n",
    "\n",
    "\n",
    "test(learning_by_penalized_gradient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logistic_regression_penalized_gradient_descent_demo(y, x):\n",
    "    # init parameters\n",
    "    max_iter = 10000\n",
    "    gamma = 0.5\n",
    "    lambda_ = 0.0005\n",
    "    threshold = 1e-8\n",
    "    losses = []\n",
    "\n",
    "    # build tx\n",
    "    tx = np.c_[np.ones((y.shape[0], 1)), x]\n",
    "    w = np.zeros((tx.shape[1], 1))\n",
    "\n",
    "    # start the logistic regression\n",
    "    for iter in range(max_iter):\n",
    "        # get loss and update w.\n",
    "        loss, w = learning_by_penalized_gradient(y, tx, w, gamma, lambda_)\n",
    "        # log info\n",
    "        if iter % 100 == 0:\n",
    "            print(\"Current iteration={i}, loss={l}\".format(i=iter, l=loss))\n",
    "        # converge criterion\n",
    "        losses.append(loss)\n",
    "        if len(losses) > 1 and np.abs(losses[-1] - losses[-2]) < threshold:\n",
    "            break\n",
    "    # visualization\n",
    "    visualization(\n",
    "        y,\n",
    "        x,\n",
    "        mean_x,\n",
    "        std_x,\n",
    "        w,\n",
    "        \"classification_by_logistic_regression_penalized_gradient_descent\",\n",
    "        True,\n",
    "    )\n",
    "    print(\"loss={l}\".format(l=calculate_loss(y, tx, w)))\n",
    "\n",
    "\n",
    "logistic_regression_penalized_gradient_descent_demo(y, x)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "#### The `logistic_regression_penalized_gradient_descent_demo` is expected to show\n",
    "\n",
    "![1](./classification_by_logistic_regression_penalized_gradient_descent.png)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}