{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9496e038",
   "metadata": {},
   "source": [
    "# Lineare Regression mit 1 Feature ($d=1$)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5754d665",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "# plotting settings\n",
    "pd.plotting.register_matplotlib_converters()\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "282549b7",
   "metadata": {},
   "source": [
    "Wir verwenden hier beispielhaft den Datensatz [Melbourne Housing Snapshot](https://www.kaggle.com/datasets/dansbecker/melbourne-housing-snapshot). Diesen finden Sie auch im Moodle unter `data/melb_data.csv`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cfe20800",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Suburb', 'Address', 'Rooms', 'Type', 'Price', 'Method', 'SellerG',\n",
       "       'Date', 'Distance', 'Postcode', 'Bedroom2', 'Bathroom', 'Car',\n",
       "       'Landsize', 'BuildingArea', 'YearBuilt', 'CouncilArea', 'Lattitude',\n",
       "       'Longtitude', 'Regionname', 'Propertycount'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melbourne_file_path = 'data/melb_data.csv'\n",
    "melbourne_data = pd.read_csv(melbourne_file_path)\n",
    "melbourne_data = melbourne_data.dropna(axis=0)  # entfernen von Daten mit fehlenden Werten\n",
    "melbourne_data.columns  # Spaltennamen der Tabelle (potentielle Features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e13b23ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Suburb</th>\n",
       "      <th>Address</th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Type</th>\n",
       "      <th>Price</th>\n",
       "      <th>Method</th>\n",
       "      <th>SellerG</th>\n",
       "      <th>Date</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Postcode</th>\n",
       "      <th>...</th>\n",
       "      <th>Bathroom</th>\n",
       "      <th>Car</th>\n",
       "      <th>Landsize</th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>CouncilArea</th>\n",
       "      <th>Lattitude</th>\n",
       "      <th>Longtitude</th>\n",
       "      <th>Regionname</th>\n",
       "      <th>Propertycount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>25 Bloomburg St</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>1035000.0</td>\n",
       "      <td>S</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>4/02/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>Yarra</td>\n",
       "      <td>-37.8079</td>\n",
       "      <td>144.9934</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>5 Charles St</td>\n",
       "      <td>3</td>\n",
       "      <td>h</td>\n",
       "      <td>1465000.0</td>\n",
       "      <td>SP</td>\n",
       "      <td>Biggin</td>\n",
       "      <td>4/03/2017</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>Yarra</td>\n",
       "      <td>-37.8093</td>\n",
       "      <td>144.9944</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>55a Park St</td>\n",
       "      <td>4</td>\n",
       "      <td>h</td>\n",
       "      <td>1600000.0</td>\n",
       "      <td>VB</td>\n",
       "      <td>Nelson</td>\n",
       "      <td>4/06/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>Yarra</td>\n",
       "      <td>-37.8072</td>\n",
       "      <td>144.9941</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>124 Yarra St</td>\n",
       "      <td>3</td>\n",
       "      <td>h</td>\n",
       "      <td>1876000.0</td>\n",
       "      <td>S</td>\n",
       "      <td>Nelson</td>\n",
       "      <td>7/05/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>210.0</td>\n",
       "      <td>1910.0</td>\n",
       "      <td>Yarra</td>\n",
       "      <td>-37.8024</td>\n",
       "      <td>144.9993</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>98 Charles St</td>\n",
       "      <td>2</td>\n",
       "      <td>h</td>\n",
       "      <td>1636000.0</td>\n",
       "      <td>S</td>\n",
       "      <td>Nelson</td>\n",
       "      <td>8/10/2016</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>1890.0</td>\n",
       "      <td>Yarra</td>\n",
       "      <td>-37.8060</td>\n",
       "      <td>144.9954</td>\n",
       "      <td>Northern Metropolitan</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Suburb          Address  Rooms Type      Price Method SellerG  \\\n",
       "1  Abbotsford  25 Bloomburg St      2    h  1035000.0      S  Biggin   \n",
       "2  Abbotsford     5 Charles St      3    h  1465000.0     SP  Biggin   \n",
       "4  Abbotsford      55a Park St      4    h  1600000.0     VB  Nelson   \n",
       "6  Abbotsford     124 Yarra St      3    h  1876000.0      S  Nelson   \n",
       "7  Abbotsford    98 Charles St      2    h  1636000.0      S  Nelson   \n",
       "\n",
       "        Date  Distance  Postcode  ...  Bathroom  Car  Landsize  BuildingArea  \\\n",
       "1  4/02/2016       2.5    3067.0  ...       1.0  0.0     156.0          79.0   \n",
       "2  4/03/2017       2.5    3067.0  ...       2.0  0.0     134.0         150.0   \n",
       "4  4/06/2016       2.5    3067.0  ...       1.0  2.0     120.0         142.0   \n",
       "6  7/05/2016       2.5    3067.0  ...       2.0  0.0     245.0         210.0   \n",
       "7  8/10/2016       2.5    3067.0  ...       1.0  2.0     256.0         107.0   \n",
       "\n",
       "   YearBuilt  CouncilArea Lattitude  Longtitude             Regionname  \\\n",
       "1     1900.0        Yarra  -37.8079    144.9934  Northern Metropolitan   \n",
       "2     1900.0        Yarra  -37.8093    144.9944  Northern Metropolitan   \n",
       "4     2014.0        Yarra  -37.8072    144.9941  Northern Metropolitan   \n",
       "6     1910.0        Yarra  -37.8024    144.9993  Northern Metropolitan   \n",
       "7     1890.0        Yarra  -37.8060    144.9954  Northern Metropolitan   \n",
       "\n",
       "  Propertycount  \n",
       "1        4019.0  \n",
       "2        4019.0  \n",
       "4        4019.0  \n",
       "6        4019.0  \n",
       "7        4019.0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melbourne_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8680d0c9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='BuildingArea', ylabel='Price'>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 39632 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.scatterplot(x=melbourne_data['BuildingArea'], y=melbourne_data['Price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "da3b8409",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1         79.00\n",
       "2        150.00\n",
       "4        142.00\n",
       "6        210.00\n",
       "7        107.00\n",
       "          ...  \n",
       "12205    149.00\n",
       "12206    115.00\n",
       "12207     35.64\n",
       "12209     61.60\n",
       "12212    388.50\n",
       "Name: BuildingArea, Length: 6196, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "melbourne_data['BuildingArea']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c1172236",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BuildingArea</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>79.0</td>\n",
       "      <td>1035000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>150.0</td>\n",
       "      <td>1465000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>142.0</td>\n",
       "      <td>1600000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>210.0</td>\n",
       "      <td>1876000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>107.0</td>\n",
       "      <td>1636000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BuildingArea      Price\n",
       "1          79.0  1035000.0\n",
       "2         150.0  1465000.0\n",
       "4         142.0  1600000.0\n",
       "6         210.0  1876000.0\n",
       "7         107.0  1636000.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# wählen für unser Beispiel einen kleinen Ausschnitt aus den Daten\n",
    "max_area = 400\n",
    "max_datapoints = 100\n",
    "data = melbourne_data[melbourne_data['BuildingArea'] < max_area][:max_datapoints][['BuildingArea', 'Price']]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8f9dec63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f1293084",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 20052 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.scatterplot(x=data['BuildingArea'], y=data['Price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "00dc4dee",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BuildingArea         79.0\n",
      "Price           1035000.0\n",
      "Name: 1, dtype: float64\n",
      "[[1, 79.0]]\n"
     ]
    }
   ],
   "source": [
    "X = []\n",
    "Y = []\n",
    "for _, row in data.iterrows():\n",
    "    X.append([1] + [row['BuildingArea']])\n",
    "    Y.append(row['Price'])\n",
    "    break\n",
    "X = np.array(X)\n",
    "Y = np.array(Y)\n",
    "print(X[:5], Y[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "efecad93",
   "metadata": {},
   "outputs": [],
   "source": [
    "def h_w(x, w):\n",
    "    return w[0] + w[1]*x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0577f21",
   "metadata": {},
   "source": [
    "## Analytische Lösung der linearen Regression\n",
    "\n",
    "`np.linalg.solve(A, b)` berechnet $w$ im linearen Gleichungssystem\n",
    "\n",
    "$ A w = b $\n",
    "\n",
    "$A$ - Matrix,\n",
    "$w$ - Vektor (unsere unbekannten),\n",
    "$b$ - Vektor.\n",
    "\n",
    "Wir suchen die Lösung $w$ im folgenden Gleichungssystem:\n",
    "\n",
    "$$ X^T X w = X^T Y $$\n",
    "\n",
    "Mit $A = X^TX$ und $b = X^T Y$ berechnet `np.linalg.solve(A, b)` unsere gesuchten Paramter für die lineare Regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "35a78137",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 42 µs, sys: 7 µs, total: 49 µs\n",
      "Wall time: 51.3 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "w_ana = np.linalg.solve(X.T @ X, X.T @ Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9a6041bd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[441524.42083181   6024.22929588]\n"
     ]
    }
   ],
   "source": [
    "print(w_ana)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f51a85af",
   "metadata": {},
   "source": [
    "Plot der analytischen Lösung"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6486ec38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='BuildingArea', ylabel='Price'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 25688 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.scatterplot(x=data['BuildingArea'], y=data['Price'])\n",
    "\n",
    "xplot = [min(data['BuildingArea']), max(data['BuildingArea'])]\n",
    "yplot = [h_w(x, w_ana) for x in xplot]\n",
    "sns.lineplot(x=xplot, y=yplot, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "aab92a40",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Definition der Kostenfunktion\n",
    "def J(w, x, y):\n",
    "    \"\"\"w, x, y müssen numpy arrays sein\"\"\"\n",
    "    errors = y - h_w(x=x, w=w)\n",
    "    mse = 1.0/(2.0*len(errors)) * ( errors @ errors )\n",
    "    return mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7ef64eb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kosten der analytischen Lösung: 200141433273.1325\n"
     ]
    }
   ],
   "source": [
    "x = data['BuildingArea'].to_numpy(copy=True)\n",
    "y = data['Price'].to_numpy(copy=True)\n",
    "J_ana = J(w=w_ana, x=x, y=y)\n",
    "print('Kosten der analytischen Lösung: {}'.format(J_ana))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0272e5ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([441524.42083181,   6024.22929588])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_ana"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "217f80c5",
   "metadata": {},
   "source": [
    "## Numerische Lösung mit Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6cda3066",
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_desc_upd(w, alpha, x, y):\n",
    "    \"\"\"y, x sind Vektoren (numpy-arrays)\"\"\"\n",
    "    errors = y - h_w(x=x, w=w)\n",
    "    w_0_upd = w[0] + alpha / len(x) * sum(errors)\n",
    "    \n",
    "    errors_x_x1 = errors @ x\n",
    "    w_1_upd = w[1] + alpha / len(x) * errors_x_x1\n",
    "    return [w_0_upd, w_1_upd]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b349e5ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad_desc(w, alpha, x, y, n_iterations):\n",
    "    J_all = [J(w=w, x=x, y=y)]\n",
    "    for it in range(n_iterations):\n",
    "        w = grad_desc_upd(w=w, alpha=alpha, x=x, y=y)\n",
    "        J_all.append(J(w=w, x=x, y=y))\n",
    "    return w, J_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b87084ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "w_tmp, J_tmp = grad_desc(w=[1e5, 1000.], alpha=1e-9, x=data['BuildingArea'].to_numpy(), y=data['Price'].to_numpy(), n_iterations=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a129a532",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999715711803561"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "J_tmp[1]/J_tmp[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fd1bb601",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w_gd_1e4: [915045.6766397628, 2959.2952860626924]\n",
      "Vergleich zu Startkosten: 0.8899738097177349\n",
      "Vergleich zu analytischer Lösung: 1.0924465228987312\n",
      "(w0_gd - w0_ana)/w0_ana: 1.0724690039021088\n",
      "(w1_gd - w1_ana)/w1_ana: -0.5087678206259784\n",
      "CPU times: user 256 ms, sys: 8.18 ms, total: 264 ms\n",
      "Wall time: 90.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "n_iterations = 10000\n",
    "alpha = 0.0001  # mit alpha experimentieren\n",
    "w_init = [1e6, 1000.]\n",
    "x = data['BuildingArea'].to_numpy()\n",
    "y = data['Price'].to_numpy()\n",
    "w_gd_1e4, J_all_1e4 = grad_desc(w=w_init, alpha=alpha, x=x, y=y, n_iterations=n_iterations)\n",
    "\n",
    "print('w_gd_1e4: {}'.format(w_gd_1e4))\n",
    "print('Vergleich zu Startkosten: {}'.format(J_all_1e4[-1]/J_all_1e4[0]))\n",
    "print('Vergleich zu analytischer Lösung: {}'.format(J_all_1e4[-1]/J_ana))\n",
    "print('(w0_gd - w0_ana)/w0_ana: {}'.format((w_gd_1e4[0]-w_ana[0])/w_ana[0]))\n",
    "print('(w1_gd - w1_ana)/w1_ana: {}'.format((w_gd_1e4[1]-w_ana[1])/w_ana[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1c26fde8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w_gd_1e5: [548748.7304152894, 5330.2046262550075]\n",
      "Vergleich zu Startkosten: 0.8185228834109243\n",
      "Vergleich zu analytischer Lösung: 1.004740216095697\n",
      "(w0_gd - w0_ana)/w0_ana: 0.24285023551238794\n",
      "(w1_gd - w1_ana)/w1_ana: -0.1152055533638727\n",
      "CPU times: user 958 ms, sys: 20.8 ms, total: 978 ms\n",
      "Wall time: 825 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "n_iterations = 100000\n",
    "alpha = 0.0001  # mit alpha experimentieren\n",
    "w_init = [1e6, 1000.]\n",
    "x = data['BuildingArea'].to_numpy()\n",
    "y = data['Price'].to_numpy()\n",
    "w_gd_1e5, J_all_1e5 = grad_desc(w=w_init, alpha=alpha, x=x, y=y, n_iterations=n_iterations)\n",
    "\n",
    "print('w_gd_1e5: {}'.format(w_gd_1e5))\n",
    "print('Vergleich zu Startkosten: {}'.format(J_all_1e5[-1]/J_all_1e5[0]))\n",
    "print('Vergleich zu analytischer Lösung: {}'.format(J_all_1e5[-1]/J_ana))\n",
    "print('(w0_gd - w0_ana)/w0_ana: {}'.format((w_gd_1e5[0]-w_ana[0])/w_ana[0]))\n",
    "print('(w1_gd - w1_ana)/w1_ana: {}'.format((w_gd_1e5[1]-w_ana[1])/w_ana[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ebff7a0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w_gd_3e5: [445476.78736763657, 5998.647038552499]\n",
      "Vergleich zu Startkosten: 0.8146664601191632\n",
      "Vergleich zu analytischer Lösung: 1.000006440595892\n",
      "(w0_gd - w0_ana)/w0_ana: 0.008951637439164814\n",
      "(w1_gd - w1_ana)/w1_ana: -0.0042465610235696925\n",
      "CPU times: user 2.4 s, sys: 53.3 ms, total: 2.45 s\n",
      "Wall time: 2.46 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "n_iterations = 300000\n",
    "alpha = 0.0001  # mit alpha experimentieren\n",
    "w_init = [1e6, 1000.]\n",
    "x = data['BuildingArea'].to_numpy()\n",
    "y = data['Price'].to_numpy()\n",
    "w_gd_3e5, J_all_3e5 = grad_desc(w=w_init, alpha=alpha, x=x, y=y, n_iterations=n_iterations)\n",
    "\n",
    "print('w_gd_3e5: {}'.format(w_gd_3e5))\n",
    "print('Vergleich zu Startkosten: {}'.format(J_all_3e5[-1]/J_all_3e5[0]))\n",
    "print('Vergleich zu analytischer Lösung: {}'.format(J_all_3e5[-1]/J_ana))\n",
    "print('(w0_gd - w0_ana)/w0_ana: {}'.format((w_gd_3e5[0]-w_ana[0])/w_ana[0]))\n",
    "print('(w1_gd - w1_ana)/w1_ana: {}'.format((w_gd_3e5[1]-w_ana[1])/w_ana[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a1b5db98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w_gd_1e6: [441524.45883596264, 6024.229049890118]\n",
      "Vergleich zu Startkosten: 0.8146612132155007\n",
      "Vergleich zu analytischer Lösung: 1.0000000000000007\n",
      "(w0_gd - w0_ana)/w0_ana: 8.60748531445252e-08\n",
      "(w1_gd - w1_ana)/w1_ana: -4.0832990952070806e-08\n",
      "CPU times: user 7.99 s, sys: 155 ms, total: 8.14 s\n",
      "Wall time: 8.22 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "n_iterations = 1000000\n",
    "alpha = 0.0001  # mit alpha experimentieren\n",
    "w_init = [1e6, 1000.]\n",
    "x = data['BuildingArea'].to_numpy()\n",
    "y = data['Price'].to_numpy()\n",
    "w_gd_1e6, J_all_1e6 = grad_desc(w=w_init, alpha=alpha, x=x, y=y, n_iterations=n_iterations)\n",
    "\n",
    "print('w_gd_1e6: {}'.format(w_gd_1e6))\n",
    "print('Vergleich zu Startkosten: {}'.format(J_all_1e6[-1]/J_all_1e6[0]))\n",
    "print('Vergleich zu analytischer Lösung: {}'.format(J_all_1e6[-1]/J_ana))\n",
    "print('(w0_gd - w0_ana)/w0_ana: {}'.format((w_gd_1e6[0]-w_ana[0])/w_ana[0]))\n",
    "print('(w1_gd - w1_ana)/w1_ana: {}'.format((w_gd_1e6[1]-w_ana[1])/w_ana[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f35b62d4",
   "metadata": {},
   "source": [
    "### Kosten J als Funktion von Gradient Descent Schritten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b18c5272",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 15060 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.lineplot(x=list(range(len(J_all_1e6))), y=J_all_1e6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ec059f6",
   "metadata": {},
   "source": [
    "### Plotten der Ergebnisse und Vergleich zwischen analytischer und numerischer Lösung\n",
    "Nach $10^4$ Schritten des Gradient Descent Algorithmus weicht der lineare Fit noch sichtbar von der analytischen Lösung ab. Nach $10^5$ Schritten ist der Unterschied im Plot kaum zu erkennen.\n",
    "Die numerische Lösung war in diesem Beispiel deutlich langsamer als die analytische. Allerdings haben wir für die analytische Lösung auch eine effiziente numpy-Implementierung genutzt und für die numerische unoptimierten Python-Code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "21c941e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 33476 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot\n",
    "xplot = [min(data['BuildingArea']), max(data['BuildingArea'])]\n",
    "yplot_ana = [h_w(x, w_ana) for x in xplot]\n",
    "yplot_gd_1e4 = [h_w(x, w_gd_1e4) for x in xplot]\n",
    "yplot_gd_1e5 = [h_w(x, w_gd_1e5) for x in xplot]\n",
    "# yplot_gd_3e5 = [h_w(x, w_gd_3e5) for x in xplot]\n",
    "yplot_gd_1e6 = [h_w(x, w_gd_1e6) for x in xplot]\n",
    "ax = sns.scatterplot(x=data['BuildingArea'], y=data['Price'])\n",
    "ax = sns.lineplot(x=xplot, y=yplot_ana, ax=ax)\n",
    "ax = sns.lineplot(x=xplot, y=yplot_gd_1e4, color='red', ax=ax)\n",
    "ax = sns.lineplot(x=xplot, y=yplot_gd_1e5, color='grey', ax=ax)\n",
    "# ax = sns.lineplot(x=xplot, y=yplot_gd_3e5, color='green', linestyle='dotted', ax=ax)\n",
    "ax = sns.lineplot(x=xplot, y=yplot_gd_1e6, color='pink', linestyle='--', ax=ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60bc96a1",
   "metadata": {},
   "source": [
    "## Vorhersagen unseres Modells\n",
    "\n",
    "Man kann die Vorhersagen des Modells entweder im Plot oben auf der Geraden ablesen. Zu jedem Wert von `BuildingArea` (x-Achse des Plots) kann so der `Preis` auf der y-Achse abgelesen werden.\n",
    "\n",
    "Alternativ können wir die von uns oben definierte Funktion `h_w(x, w)` aufrufen. Der Parameter `w` ist die gefunden Lösung und `x` die `BuildingArea` für die wir einen Preis vorhersagen wollen."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e13003c8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preis laut analytischem Modell: 2170478.23\n",
      "Preis laut Gradient Descent Modell nach 10^5 Iterationen: 2078517.46\n",
      "Preis laut Gradient Descent Modell nach 3*10^5 Iterationen: 2167088.49\n",
      "Preis laut Gradient Descent Modell nach 1*10^6 Iterationen: 2170478.20\n"
     ]
    }
   ],
   "source": [
    "# Beispiel: Vorhersage unseres Modells für ein Haus mit Wohnfläche 287:\n",
    "# wir machen je eine Vorhersage mit\n",
    "# 1. den analytisch gefundenen Paramtern\n",
    "# 2. den mit Gradient Descent nach 10^5 Iterationen gefundenen Parametern\n",
    "# 3. den mit Gradient Descent nach 3*10^5 Iterationen gefundenen Parametern\n",
    "building_area_new = 287\n",
    "price_ana = h_w(x=building_area_new, w=w_ana)\n",
    "price_1e5 = h_w(x=building_area_new, w=w_gd_1e5)\n",
    "price_3e5 = h_w(x=building_area_new, w=w_gd_3e5)\n",
    "price_1e6 = h_w(x=building_area_new, w=w_gd_1e6)\n",
    "print('Preis laut analytischem Modell: {:.2f}'.format(price_ana))\n",
    "print('Preis laut Gradient Descent Modell nach 10^5 Iterationen: {:.2f}'.format(price_1e5))\n",
    "print('Preis laut Gradient Descent Modell nach 3*10^5 Iterationen: {:.2f}'.format(price_3e5))\n",
    "print('Preis laut Gradient Descent Modell nach 1*10^6 Iterationen: {:.2f}'.format(price_1e6))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fca62677",
   "metadata": {},
   "source": [
    "## $R^2$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f1703c7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "erklärte Varianz (R^2): 0.22971025499088604\n"
     ]
    }
   ],
   "source": [
    "x = data['BuildingArea'].to_numpy(copy=True)\n",
    "y = data['Price'].to_numpy(copy=True)\n",
    "J_ana = J(w=w_ana, x=x, y=y)\n",
    "MSE = 2*J_ana\n",
    "mu_y = sum(y)/len(y)\n",
    "sigma_y_quadrat = ( (y - mu_y) @ (y - mu_y) ) / len(y)\n",
    "R2 = 1 - MSE/sigma_y_quadrat\n",
    "print('erklärte Varianz (R^2): {}'.format(R2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
