ANN4_ReLu_L2_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "88d012e9-b67f-4764-b183-062793d5f2f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20000 - Loss: 0.9998922332268305\n",
      "Epoch 501/20000 - Loss: 0.31334453195225037\n",
      "Epoch 1001/20000 - Loss: 0.08523795061095167\n",
      "Epoch 1501/20000 - Loss: 0.07641487772458068\n",
      "Epoch 2001/20000 - Loss: 0.07233461932716803\n",
      "Epoch 2501/20000 - Loss: 0.06987186789752706\n",
      "Epoch 3001/20000 - Loss: 0.06820452563658203\n",
      "Epoch 3501/20000 - Loss: 0.06699275170815719\n",
      "Epoch 4001/20000 - Loss: 0.06608014468839657\n",
      "Epoch 4501/20000 - Loss: 0.06539725755844503\n",
      "Epoch 5001/20000 - Loss: 0.06487843768045062\n",
      "Epoch 5501/20000 - Loss: 0.06447424136841061\n",
      "Epoch 6001/20000 - Loss: 0.06415507142017234\n",
      "Epoch 6501/20000 - Loss: 0.0638914434157395\n",
      "Epoch 7001/20000 - Loss: 0.06366943561793381\n",
      "Epoch 7501/20000 - Loss: 0.06348167816902309\n",
      "Epoch 8001/20000 - Loss: 0.06329694899317712\n",
      "Epoch 8501/20000 - Loss: 0.06312488575136838\n",
      "Epoch 9001/20000 - Loss: 0.06297004367097499\n",
      "Epoch 9501/20000 - Loss: 0.06282675796083499\n",
      "Epoch 10001/20000 - Loss: 0.06269436558460545\n",
      "Epoch 10501/20000 - Loss: 0.06258377873322031\n",
      "Epoch 11001/20000 - Loss: 0.06248518741479796\n",
      "Epoch 11501/20000 - Loss: 0.062396724207512744\n",
      "Epoch 12001/20000 - Loss: 0.062317473173803976\n",
      "Epoch 12501/20000 - Loss: 0.06224743130799918\n",
      "Epoch 13001/20000 - Loss: 0.062190531848418225\n",
      "Epoch 13501/20000 - Loss: 0.06213346821212045\n",
      "Epoch 14001/20000 - Loss: 0.06207792220564583\n",
      "Epoch 14501/20000 - Loss: 0.06202579205558656\n",
      "Epoch 15001/20000 - Loss: 0.06197820268908797\n",
      "Epoch 15501/20000 - Loss: 0.06193173525990318\n",
      "Epoch 16001/20000 - Loss: 0.061890415983107544\n",
      "Epoch 16501/20000 - Loss: 0.061849723000191176\n",
      "Epoch 17001/20000 - Loss: 0.061808025938549956\n",
      "Epoch 17501/20000 - Loss: 0.0617625398286068\n",
      "Epoch 18001/20000 - Loss: 0.06171065021994131\n",
      "Epoch 18501/20000 - Loss: 0.061660752949861695\n",
      "Epoch 19001/20000 - Loss: 0.061615538948407524\n",
      "Epoch 19501/20000 - Loss: 0.06156823725467704\n",
      "Predictions:\n",
      "[[465.76860485]\n",
      " [444.99399662]\n",
      " [486.75197034]\n",
      " ...\n",
      " [434.45642496]\n",
      " [442.07183237]\n",
      " [448.44780904]]\n",
      "R² Score: 0.9384752268217779\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# Load the CSV file\n",
    "data = pd.read_csv('data_ANN.csv')\n",
    "\n",
    "# Extract features (AT, V, AP, RH) into X\n",
    "X = data[['AT', 'V', 'AP', 'RH']].values\n",
    "\n",
    "# Extract target (PE) into y\n",
    "y = data[['PE']].values\n",
    "\n",
    "# Normalize input features\n",
    "X_mean = X.mean(axis=0)\n",
    "X_std = X.std(axis=0)\n",
    "X = (X - X_mean) / X_std  # Standardization\n",
    "\n",
    "# Normalize target values\n",
    "y_mean = y.mean()\n",
    "y_std = y.std()\n",
    "y = (y - y_mean) / y_std  # Standardization\n",
    "\n",
    "# Define the neural network architecture\n",
    "input_size = 4    # Number of input neurons (four features)\n",
    "hidden_size = 12  # Increase the number of neurons in the hidden layer\n",
    "output_size = 1   # Number of output neurons (one output)\n",
    "\n",
    "# Initialize weights and biases\n",
    "np.random.seed(42)\n",
    "W1 = np.random.randn(input_size, hidden_size) * 0.01  # Small weight initialization\n",
    "b1 = np.zeros((1, hidden_size))                       # Bias for hidden layer\n",
    "W2 = np.random.randn(hidden_size, output_size) * 0.01 # Small weight initialization\n",
    "b2 = np.zeros((1, output_size))                       # Bias for output layer\n",
    "\n",
    "# Activation functions\n",
    "def relu(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "def relu_derivative(x):\n",
    "    return np.where(x > 0, 1, 0)\n",
    "\n",
    "# Forward propagation\n",
    "def forward_propagation(X):\n",
    "    Z1 = np.dot(X, W1) + b1\n",
    "    A1 = relu(Z1)\n",
    "    Z2 = np.dot(A1, W2) + b2\n",
    "    A2 = Z2  # Linear activation for output\n",
    "    return Z1, A1, Z2, A2\n",
    "\n",
    "# Loss function with L2 regularization\n",
    "def compute_loss(y_true, y_pred, W1, W2, l2_lambda):\n",
    "    m = y_true.shape[0]\n",
    "    loss = np.mean((y_true - y_pred) ** 2)  # Mean Squared Error for regression\n",
    "    l2_loss = (l2_lambda / (2 * m)) * (np.sum(np.square(W1)) + np.sum(np.square(W2)))\n",
    "    return loss + l2_loss\n",
    "\n",
    "# Backpropagation with L2 regularization\n",
    "def backpropagation(X, y_true, Z1, A1, Z2, A2, W1, W2, l2_lambda):\n",
    "    m = X.shape[0]\n",
    "    dZ2 = (A2 - y_true)\n",
    "    dW2 = (np.dot(A1.T, dZ2) / m) + (l2_lambda / m) * W2\n",
    "    db2 = np.sum(dZ2, axis=0, keepdims=True) / m\n",
    "    dA1 = np.dot(dZ2, W2.T)\n",
    "    dZ1 = dA1 * relu_derivative(Z1)\n",
    "    dW1 = (np.dot(X.T, dZ1) / m) + (l2_lambda / m) * W1\n",
    "    db1 = np.sum(dZ1, axis=0, keepdims=True) / m\n",
    "    return dW1, db1, dW2, db2\n",
    "\n",
    "# Update parameters\n",
    "def update_parameters(W1, b1, W2, b2, dW1, db1, dW2, db2, learning_rate):\n",
    "    W1 -= learning_rate * dW1\n",
    "    b1 -= learning_rate * db1\n",
    "    W2 -= learning_rate * dW2\n",
    "    b2 -= learning_rate * db2\n",
    "    return W1, b1, W2, b2\n",
    "\n",
    "# Train the neural network\n",
    "def train(X, y, epochs, learning_rate, l2_lambda):\n",
    "    global W1, b1, W2, b2\n",
    "    for epoch in range(epochs):\n",
    "        Z1, A1, Z2, A2 = forward_propagation(X)\n",
    "        loss = compute_loss(y, A2, W1, W2, l2_lambda)\n",
    "        if epoch % 500 == 0:\n",
    "            print(f\"Epoch {epoch + 1}/{epochs} - Loss: {loss}\")\n",
    "        dW1, db1, dW2, db2 = backpropagation(X, y, Z1, A1, Z2, A2, W1, W2, l2_lambda)\n",
    "        W1, b1, W2, b2 = update_parameters(W1, b1, W2, b2, dW1, db1, dW2, db2, learning_rate)\n",
    "    return W1, b1, W2, b2\n",
    "\n",
    "# Make predictions\n",
    "def predict(X, W1, b1, W2, b2):\n",
    "    _, _, _, A2 = forward_propagation(X)\n",
    "    return A2\n",
    "\n",
    "# Train the neural network with increased epochs and L2 regularization\n",
    "W1, b1, W2, b2 = train(X, y, epochs=20000, learning_rate=0.01, l2_lambda=0.001)\n",
    "\n",
    "# Make predictions\n",
    "predictions = predict(X, W1, b1, W2, b2)\n",
    "\n",
    "# Denormalize the predictions\n",
    "denormalized_predictions = predictions * y_std + y_mean\n",
    "\n",
    "# Compute R² score\n",
    "r2 = r2_score(data[['PE']], denormalized_predictions)\n",
    "\n",
    "print(\"Predictions:\")\n",
    "print(denormalized_predictions)\n",
    "print(f\"R² Score: {r2}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25b82589-eff2-4bc2-a336-bed8a8be8621",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}