Loading_prediction.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.modules import Module\n",
    "from torch import Tensor\n",
    "\n",
    "import dgl\n",
    "import dgl.nn.pytorch.conv as dglnn\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader, random_split, Subset\n",
    "\n",
    "import models\n",
    "import importlib\n",
    "importlib.reload(models)\n",
    "from models import FCN1, FCN2, GCN\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creation of the CustomDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, generator_p_set_file, p_load_file, q_load_file, loadings_file):\n",
    "        self.generator_p_set = pd.read_csv(generator_p_set_file, header=None)\n",
    "        self.p_load = pd.read_csv(p_load_file, header=None)\n",
    "        self.q_load = pd.read_csv(q_load_file, header=None)\n",
    "        self.loadings = pd.read_csv(loadings_file, header=None)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.generator_p_set.shape[1]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        generator_p_set_row = self.generator_p_set.iloc[:,idx]\n",
    "        #print(generator_p_set_row)\n",
    "        p_load_row = self.p_load.iloc[:,idx]\n",
    "        q_load_row = self.q_load.iloc[:,idx]\n",
    "        loadings_row = self.loadings.iloc[:,idx]\n",
    "        combined_values = torch.tensor(\n",
    "            list(generator_p_set_row) + list(p_load_row) + list(q_load_row),dtype=torch.float32\n",
    "        )\n",
    "        labels = torch.tensor(list(loadings_row),dtype=torch.float32)\n",
    "        return combined_values, labels\n",
    "\n",
    "# Define your file paths\n",
    "generator_p_set_file = \"datasets/39BusSystem_full/valid_generators_p.csv\"\n",
    "p_load_file = \"datasets/39BusSystem_full/valid_loads_p.csv\"\n",
    "q_load_file = \"datasets/39BusSystem_full/valid_loads_q.csv\"\n",
    "loadings_file = \"datasets/39BusSystem_full/valid_loadings.csv\"\n",
    "\n",
    "dataset = CustomDataset(generator_p_set_file, p_load_file, q_load_file, loadings_file)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creation of the dataloaders with Random Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a data loader to iterate over your dataset\n",
    "batch_size = 100  # Set the batch size according to paper (100)\n",
    "\n",
    "# Define the split ratios according to paper\n",
    "train_ratio = 0.84\n",
    "val_ratio = 0.08\n",
    "test_ratio = 0.08\n",
    "torch.random.manual_seed(42)\n",
    "\n",
    "# Calculate the number of samples for each split\n",
    "train_size = int(train_ratio * len(dataset))\n",
    "val_size = int(val_ratio * len(dataset))\n",
    "test_size = len(dataset) - train_size - val_size\n",
    "\n",
    "# Split the dataset into training, validation, and test sets\n",
    "train_set, val_set, test_set = random_split(dataset, [train_size, val_size, test_size])\n",
    "\n",
    "# Create data loaders for each set\n",
    "train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "\n",
    "#Create data loaders for GCN neighborhood\n",
    "dataset_neig = DataLoader(dataset, batch_size=350000, shuffle=False)\n",
    "\n",
    "len(dataset_neig)\n",
    "dataset_neig = next(iter(dataset_neig))\n",
    "input_neig = np.array(dataset_neig[0]).T\n",
    "output_neig = np.array(dataset_neig[1]).T\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get a batch of data from the train_loader\n",
    "batch_inputs, batch_labels = next(iter(train_loader))\n",
    "\n",
    "# Print the dimensions\n",
    "print(\"Input dimensions:\", batch_inputs.shape[1])\n",
    "print(\"Output dimensions:\", batch_labels.shape[1])\n",
    "\n",
    "input_dim = batch_inputs.shape[1]\n",
    "output_dim = batch_labels.shape[1]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training, Validation and Testing routines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining Training, Validation and Testing routines\n",
    "\n",
    "def train(model, train_loader, valid_loader, optimizer=None, epochs=5, device='cpu', fraction=1, name_file = None):\n",
    "    if(name_file is not None):\n",
    "        column_names = ['Epoch', 'Train Loss', 'Valid Loss']\n",
    "        df = pd.DataFrame(columns = column_names)\n",
    "\n",
    "    criterion = nn.MSELoss()\n",
    "    model.to(device)\n",
    "    \n",
    "    if optimizer is None:\n",
    "        optimizer = model.get_optimizer()\n",
    "    \n",
    "    dataset_size = len(train_loader.dataset)\n",
    "    num_train_samples = int(fraction * dataset_size)\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        train_loss = 0.0\n",
    "        valid_loss = 0.0\n",
    "        \n",
    "        model.train()\n",
    "        for i, (inputs, targets) in enumerate(train_loader):\n",
    "            if i * train_loader.batch_size >= num_train_samples:\n",
    "                break\n",
    "            \n",
    "            inputs = inputs.to(device)\n",
    "            targets = targets.to(device)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            train_loss += loss.item()\n",
    "        \n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            for inputs, targets in valid_loader:\n",
    "                inputs = inputs.to(device)\n",
    "                targets = targets.to(device)\n",
    "                \n",
    "                outputs = model(inputs)\n",
    "                loss = criterion(outputs, targets)\n",
    "                \n",
    "                valid_loss += loss.item()\n",
    "        \n",
    "        train_loss /= min(len(train_loader), int(num_train_samples / train_loader.batch_size))\n",
    "        valid_loss /= len(valid_loader)\n",
    "        \n",
    "        print(f\"Epoch {epoch+1}/{epochs} - Train Loss: {train_loss:.4f} - Valid Loss: {valid_loss:.4f}\")\n",
    "        if name_file is not None:\n",
    "            new_row = pd.DataFrame([[epoch+1, train_loss, valid_loss]], columns = column_names)\n",
    "            df = pd.concat([df, new_row], ignore_index=True)\n",
    "            df.to_csv(name_file, index=False)\n",
    "\n",
    "def test(model, test_loader, device='cpu', name_file = None):\n",
    "    criterion = nn.MSELoss()\n",
    "    model.to(device)\n",
    "    model.eval()\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    with torch.no_grad():\n",
    "        for inputs, targets in test_loader:\n",
    "            inputs = inputs.to(device)\n",
    "            targets = targets.to(device)\n",
    "            \n",
    "            outputs = model(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            \n",
    "            test_loss += loss.item()\n",
    "    \n",
    "    test_loss /= len(test_loader)\n",
    "    print(f\"Test Loss: {test_loss:.4f}\")\n",
    "    if name_file is not None:\n",
    "        column_names = ['Test Loss']\n",
    "        df = pd.DataFrame(columns=column_names)\n",
    "        new_row = pd.DataFrame([[test_loss]], columns = column_names)\n",
    "        df = pd.concat([df, new_row], ignore_index=True)\n",
    "        df.to_csv(name_file, index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creation of the folders \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Folder creations\n",
    "def create_folder(folder_name):\n",
    "    if not os.path.exists(folder_name):\n",
    "        # Create the folder\n",
    "        os.makedirs(folder_name)\n",
    "        print(f\"Folder '{folder_name}' created successfully.\")\n",
    "    else:\n",
    "        print(f\"Folder '{folder_name}' already exists.\")\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculation FOR GCN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def init_neigh(inputs, outputs, top_k):\n",
    "    if not isinstance(inputs, list) or not isinstance(inputs[0], list):\n",
    "        return None\n",
    "    if not isinstance(outputs, list) or not isinstance(outputs[0], list):\n",
    "        return None\n",
    "\n",
    "    n_inputs = inputs\n",
    "    n_outputs = outputs\n",
    "    n_standard_deviation_input = calculate_standard_deviations(inputs)\n",
    "    n_standard_deviation_output = calculate_standard_deviations(outputs)\n",
    "    n_covariance = calculate_covariances(n_inputs, n_outputs)\n",
    "    n_correlation = calculate_correlations(n_covariance, n_standard_deviation_input, n_standard_deviation_output)\n",
    "    n_neighbor = calculate_neighbors(n_correlation)\n",
    "    n_neig_mat = generate_neighbor_matrix(n_neighbor, top_k)\n",
    "\n",
    "    return n_neig_mat\n",
    "\n",
    "def calculate_standard_deviations(data):\n",
    "    standard_deviations = []\n",
    "    for x in data:\n",
    "        mean = sum(x) / len(x)\n",
    "        sn = 0\n",
    "        for y in x:\n",
    "            sn += (y-mean)**2\n",
    "            sn = math.sqrt(\n",
    "                sn/len(x)\n",
    "            )\n",
    "        standard_deviations.append(sn)\n",
    "    return standard_deviations\n",
    "\n",
    "def calculate_covariances(inputs, outputs):\n",
    "    covariances = []\n",
    "    for input_data in inputs:\n",
    "        input_mean = sum(input_data) / len(input_data)\n",
    "        temp = []\n",
    "        for output_data in outputs:\n",
    "            output_mean = sum(output_data) / len(output_data)\n",
    "            cov = 0\n",
    "            for idx in range(0, len(input_data)) :\n",
    "                cov += (input_data[idx]-input_mean)*(output_data[idx]-output_mean)\n",
    "            cov = cov/len(input_data)\n",
    "            temp.append(cov)\n",
    "        covariances.append(temp)\n",
    "    return covariances\n",
    "\n",
    "def calculate_correlations(covariances, inputs, outputs):\n",
    "    correlations = []\n",
    "    for i in range(len(covariances)):\n",
    "        temp = []\n",
    "        for j in range(len(covariances[i])):\n",
    "            corr = covariances[i][j] / (inputs[i] * outputs[j])\n",
    "            temp.append(corr)\n",
    "        correlations.append(temp)\n",
    "    return correlations\n",
    "\n",
    "def calculate_neighbors(correlations):\n",
    "    neighbors = []\n",
    "    for row in correlations:\n",
    "        temp = []\n",
    "        for corr in row:\n",
    "            count = 0\n",
    "            for _corr in row :\n",
    "                if abs(corr) <= abs(_corr) :\n",
    "                    count += 1\n",
    "            temp.append(count)\n",
    "        neighbors.append(temp)\n",
    "    return neighbors\n",
    "\n",
    "def generate_neighbor_matrix(neighbors, top_k):\n",
    "    neighbor_matrix = []\n",
    "    for row in neighbors:\n",
    "        temp = []\n",
    "        for elem in row:\n",
    "            temp.append(elem <= top_k)\n",
    "        neighbor_matrix.append(temp)\n",
    "    return neighbor_matrix\n",
    "\n",
    "# neighbooring\n",
    "top_k = output_dim * 1\n",
    "\n",
    "neighborhood = np.array(init_neigh(input_neig.tolist(), output_neig.tolist(), int(top_k))).T.tolist()\n",
    "# print('neighborhood : ',neighborhood)\n",
    "\n",
    "neighborhood_out = np.array(init_neigh(output_neig.tolist(), output_neig.tolist(), 0.2*output_dim)).T.tolist()\n",
    "print(\"IMPORTANT SHAPE : \", len(neighborhood))\n",
    "print(neighborhood_out)\n",
    "# end neig\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def get_model_and_optimizer(mdl: str):\n",
    "    if mdl == 'fcn1':\n",
    "        model = FCN1(in_size=input_dim, out_size=output_dim)\n",
    "    elif mdl == 'fcn2':\n",
    "        model = FCN2(in_size=input_dim, out_size=output_dim)\n",
    "    elif mdl == 'gcn':\n",
    "        model = GCN(in_features=input_dim, out_features=output_dim, neighborhood=neighborhood, neighborhood_out=neighborhood_out)\n",
    "\n",
    "    optimizer = model.get_optimizer()\n",
    "    return model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the fraction of the dataset to be used for training, test and validation\n",
    "ds_fraction = 1.0\n",
    "\n",
    "\n",
    "# Defining the model to train\n",
    "md = 'gcn'\n",
    "model, optimizer = get_model_and_optimizer(md)\n",
    "\n",
    "# Setup device-agnostic code \n",
    "# DGL doesn't work with mps\n",
    "if(md == 'gcn'):\n",
    "    if torch.cuda.is_available():\n",
    "        device = \"cuda\" # NVIDIA GPU\n",
    "    elif torch.backends.mps.is_available():\n",
    "        device = \"cpu\"\n",
    "else:\n",
    "    if torch.cuda.is_available():\n",
    "        device = \"cuda\" # NVIDIA GPU\n",
    "    elif torch.backends.mps.is_available():\n",
    "        device = \"mps\" # Apple GPU\n",
    "    else:\n",
    "        device = \"cpu\" # Defaults to CPU if NVIDIA GPU/Apple GPU aren't available\n",
    "\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "model = model.to(device)\n",
    "\n",
    "nameFolder =f\"modelsTrained/{md}\"+datetime.now().strftime(\"_%d_%m_%Y_%H_%M_%S\")\n",
    "create_folder(nameFolder)\n",
    "nameFileTraining = nameFolder + \"/trainingLog.csv\"\n",
    "nameFileTest = nameFolder + \"/testLog.csv\"\n",
    "\n",
    "\n",
    "train(model,train_loader,val_loader,optimizer,250,device,fraction=ds_fraction, name_file = nameFileTraining)\n",
    "torch.save(model.state_dict(), nameFolder + \"/model.pth\")\n",
    "test(model, test_loader, device, name_file = nameFileTest)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = nameFileTraining\n",
    "\n",
    "epochs = []\n",
    "train_losses = []\n",
    "valid_losses = []\n",
    "\n",
    "with open(input_file, 'r') as file:\n",
    "    reader = csv.DictReader(file)\n",
    "    for row in reader:\n",
    "        epochs.append(int(row['Epoch']))\n",
    "        train_losses.append(float(row['Train Loss']))\n",
    "        valid_losses.append(float(row['Valid Loss']))\n",
    "\n",
    "# Plotting\n",
    "#plt.plot(epochs, train_losses, label='Train Loss')\n",
    "plt.plot(epochs, valid_losses, label='Validation Loss')\n",
    "\n",
    "#plt.title('Validation Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "\n",
    "# Set logarithmic scale on y-axis\n",
    "plt.yscale('log')\n",
    "plt.show()\n",
    "with open(nameFileTest, 'r') as file:\n",
    "    reader = csv.DictReader(file)\n",
    "    for row in reader:\n",
    "        test_loss = float(row['Test Loss'])\n",
    "print(f\"Test Loss: {test_loss:.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}