Add Transformer history (#65)

* draft visual of transformer for ts

* update

* reformat the plotting of transformer ts papers

* reformat the plotting of transformer ts papers

dl/notebooks/hierarchical_forecasting_mint.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.14.5
+#       jupytext_version: 1.15.2
 #   kernelspec:
 #     display_name: Python 3
 #     language: python
@@ -17,13 +17,13 @@
 # # Forecast Reconciliation
 #
 # This is a notebook for the section [Hierarchical Time Series Reconciliation](https://dl.leima.is/time-series/timeseries-hierarchical.reconciliation/).
-
-import re
-
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import seaborn as sns
+#
+# import re
+#
+# import matplotlib.pyplot as plt
+# import numpy as np
+# import pandas as pd
+# import seaborn as sns
 
 # + colab={"base_uri": "https://localhost:8080/"} id="Dv8Ua7IlbgcP" outputId="e1d62a51-78c6-4bfe-e7bf-717771baaf9c"
 import sympy as sp

dl/notebooks/lstm_properties.py

@@ -142,4 +142,3 @@ def cell_state(f, m, c_prev: int = 1):
 )
 ax.set_xlabel("c_prev")
 ax.set_ylabel("c")
-# -

dl/notebooks/rnn_phase_space.py

@@ -110,36 +110,114 @@ def states(self, z: npt.ArrayLike) -> dict[str : npt.ArrayLike]:
                 "t": np.array(t_steps),
                 "h": np.array(h_t),
                 "dh": np.array(h_t_delta),
+                "z": np.pad(z, (1, 0), constant_values=0),
             },
             **{k: [v] * total_time_steps for k, v in self._metadata.items()},
         }
 
 
+# +
+def rnn_inference(rnn_params: list[dict], z: npt.ArrayLike) -> pd.DataFrame:
+    """
+    Run through a list of parameters and return the states
+
+    :param rnn_params: list of RNN parameters
+    :param z: input time series values
+    """
+    df_experiments = pd.DataFrame()
+    for p in rnn_params:
+        df_experiments = pd.concat(
+            [df_experiments, pd.DataFrame(RNNState(**p).states(z=z))]
+        )
+
+    return df_experiments
+
+
+def rnn_inference_1d_visual(dataframe_experiment: pd.DataFrame, title: str) -> None:
+    """
+    Visualize RNN inference experiments
+
+    :param dataframe_experiment: dataframe from the inference experiment
+    :param title: title of the figure
+    """
+
+    z = dataframe_experiment.loc[
+        dataframe_experiment.experiment == dataframe_experiment.experiment.iloc[0]
+    ].z
+
+    _, ax = plt.subplots(figsize=(10, 6.18))
+
+    sns.lineplot(
+        dataframe_experiment,
+        x="t",
+        y="h",
+        hue="w_h",
+        size="initial_state",
+        linestyle="dashed",
+        ax=ax,
+    )
+
+    ax_right = ax.twinx()
+
+    sns.lineplot(
+        x=np.arange(1, len(z) + 1),
+        y=z,
+        linestyle="dashed",
+        color="gray",
+        label=r"Input: $z$",
+        ax=ax_right,
+    )
+
+    ax_right.set_ylabel(r"$z$")
+    ax.legend(loc=1)
+    ax.set_title(title)
+    ax.legend(loc=4)
+
+
+# -
+
 # ## One Dimensional State
 
 z_1 = np.linspace(0, 10, 101)
-# z_1 = np.ones(10) * 0.5
-# z_1 = np.random.rand(100)
+# z_1 =
+z_1 = np.random.rand(20)
+# z_1 = np.sin(np.linspace(0, 10, 51))
 
-# +
 experiment_params = [
-    {"w_h": 0.5, "w_i": 0.5, "b": 0, "h_init": 0.5},
-    {"w_h": 1.5, "w_i": 1.5, "b": 0, "h_init": 0.5},
-    {"w_h": 0.5, "w_i": 0.5, "b": 0, "h_init": 1.5},
-    {"w_h": 1.5, "w_i": 1.5, "b": 0, "h_init": 1.5},
+    {"w_h": 0.5, "w_i": 1, "b": 0, "h_init": 0.1},
+    {"w_h": 1.5, "w_i": 1, "b": 0, "h_init": 0.1},
+    {"w_h": 0.5, "w_i": 1, "b": 0, "h_init": 2},
+    {"w_h": 1.5, "w_i": 1, "b": 0, "h_init": 2},
 ]
 
-df_1_experiments = pd.DataFrame()
-for p in experiment_params:
-    df_1_experiments = pd.concat(
-        [df_1_experiments, pd.DataFrame(RNNState(**p).states(z=z_1))]
-    )
+rnn_inference_1d_visual(
+    dataframe_experiment=rnn_inference(
+        rnn_params=experiment_params, z=np.ones(10) * 0.5
+    ),
+    title="RNN Inference for Long Forecast Horizon (Constant Input)",
+)
 
-# +
-_, ax = plt.subplots(figsize=(10, 6.18))
+rnn_inference_1d_visual(
+    dataframe_experiment=rnn_inference(
+        rnn_params=experiment_params, z=np.linspace(0, 10, 101)
+    ),
+    title="RNN Inference for Long Forecast Horizon (Linear Input)",
+)
+
+rnn_inference_1d_visual(
+    dataframe_experiment=rnn_inference(
+        rnn_params=experiment_params, z=np.random.rand(20)
+    ),
+    title="RNN Inference for Long Forecast Horizon (Random Input)",
+)
+
+rnn_inference_1d_visual(
+    dataframe_experiment=rnn_inference(
+        rnn_params=experiment_params, z=np.sin(np.linspace(0, 10, 51))
+    ),
+    title="RNN Inference for Long Forecast Horizon (Sin Input)",
+)
 
-sns.lineplot(df_1_experiments, x="t", y="h", hue="experiment")
-# -
 
 # ### Two dimensional state
 

dl/notebooks/rnn_timeseries.py

@@ -8,9 +8,9 @@
 #       format_version: '1.5'
 #       jupytext_version: 1.15.2
 #   kernelspec:
-#     display_name: deep-learning
+#     display_name: .venv
 #     language: python
-#     name: deep-learning
+#     name: python3
 # ---
 
 # # RNN for Univariate Time Series Forecasting
@@ -20,7 +20,6 @@
 import dataclasses
 
 # +
-import math
 from functools import cached_property
 from typing import Dict, List, Tuple
 

dl/notebooks/rnn_timeseries_comparison.py

@@ -65,8 +65,8 @@ class TSRNNParams:
     :param num_layers: number of units stacked
     """
 
-    input_size: int
     hidden_size: int
+    input_size: int = 1
     num_layers: int = 1
 
 
@@ -84,8 +84,6 @@ def __init__(self, history_length: int, horizon: int, rnn_params: TSRNNParams):
         self.history_length = history_length
         self.horizon = horizon
 
-        self.regulate_input = nn.Linear(self.history_length, self.rnn_params.input_size)
-
         self.rnn = nn.RNN(
             input_size=self.rnn_params.input_size,
             hidden_size=self.rnn_params.hidden_size,
@@ -100,7 +98,7 @@ def rnn_config(self):
         return dataclasses.asdict(self.rnn_params)
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
-        x = self.regulate_input(x)
+        # x = self.regulate_input(x)
         x, _ = self.rnn(x)
 
         return self.regulate_output(x)
@@ -181,7 +179,7 @@ def forward(self, x):
 # #### LightningModule
 
 # +
-ts_rnn_params_1_step = TSRNNParams(input_size=96, hidden_size=64, num_layers=1)
+ts_rnn_params_1_step = TSRNNParams(input_size=1, hidden_size=64, num_layers=1)
 
 ts_rnn_1_step = TSRNN(
     history_length=history_length_1_step,

dl/notebooks/timeseries_data_box-cox.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.14.5
+#       jupytext_version: 1.15.2
 #   kernelspec:
 #     display_name: deep-learning
 #     language: python