Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feat] Robustified crossvalidation #668

Merged
merged 4 commits into from
Oct 13, 2023
Merged

[Feat] Robustified crossvalidation #668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6fd00f4
Added .icloud files to .gitignore
kdgutier Oct 13, 2023
4324dbd
Added forward to Naive, Constant and Zero models
kdgutier Oct 13, 2023
d882a13
Protected fit with fallback_model
kdgutier Oct 13, 2023
a2497f8
Fixed fit fallback_model to use .new()
kdgutier Oct 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ dist
.vscode
.idea
*.gif
*.icloud
*.csv
*/data/*
*.parquet
Expand Down
70 changes: 61 additions & 9 deletions nbs/src/core/core.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,21 @@
" return False\n",
" return np.allclose(self.data, other.data) and np.array_equal(self.indptr, other.indptr)\n",
" \n",
" def fit(self, models):\n",
" def fit(self, models, fallback_model=None):\n",
" fm = np.full((self.n_groups, len(models)), np.nan, dtype=object)\n",
" for i, grp in enumerate(self):\n",
" y = grp[:, 0] if grp.ndim == 2 else grp\n",
" X = grp[:, 1:] if (grp.ndim == 2 and grp.shape[1] > 1) else None\n",
" for i_model, model in enumerate(models):\n",
" new_model = model.new()\n",
" fm[i, i_model] = new_model.fit(y=y, X=X)\n",
" try:\n",
" new_model = model.new()\n",
" fm[i, i_model] = new_model.fit(y=y, X=X)\n",
" except Exception as error:\n",
" if fallback_model is not None:\n",
" new_fallback_model = fallback_model.new()\n",
" fm[i, i_model] = new_fallback_model.fit(y=y, X=X)\n",
" else:\n",
" raise error\n",
" return fm\n",
" \n",
" def _get_cols(self, models, attr, h, X, level=tuple()):\n",
Expand Down Expand Up @@ -354,9 +361,13 @@
" else:\n",
" if i_window == 0:\n",
" # for the first window we have to fit each model\n",
" model = model.fit(y=y_train, X=X_train)\n",
" if fallback_model is not None:\n",
" fallback_model = fallback_model.fit(y=y_train, X=X_train)\n",
" try:\n",
" model = model.fit(y=y_train, X=X_train)\n",
" except Exception as error:\n",
" if fallback_model is not None:\n",
" fallback_model = fallback_model.fit(y=y_train, X=X_train)\n",
" else:\n",
" raise error\n",
" try:\n",
" res_i = model.forward(h=h, y=y_train, X=X_train, \n",
" X_future=X_future, fitted=fitted, **kwargs)\n",
Expand Down Expand Up @@ -682,6 +693,47 @@
"np.testing.assert_array_equal(fcst_cv_f['fitted']['values'], fcst_cv_naive['fitted']['values'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e05f8b34",
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"# test fallback model under failed fit for cross validation\n",
"class FailedFit:\n",
"\n",
" def __init__(self):\n",
" pass\n",
"\n",
" def forecast(self):\n",
" pass\n",
"\n",
" def fit(self, y, X):\n",
" raise Exception('Failed fit')\n",
"\n",
" def __repr__(self):\n",
" return \"FailedFit\"\n",
"\n",
"fcst_cv_f = ga.cross_validation(\n",
" models=[FailedFit()], \n",
" fallback_model=Naive(), h=2, \n",
" test_size=5,\n",
" refit=False,\n",
" fitted=True,\n",
")\n",
"fcst_cv_naive = ga.cross_validation(\n",
" models=[Naive()], \n",
" h=2, \n",
" test_size=5,\n",
" refit=False,\n",
" fitted=True,\n",
")\n",
"test_eq(fcst_cv_f['forecasts'], fcst_cv_naive['forecasts'])\n",
"np.testing.assert_array_equal(fcst_cv_f['fitted']['values'], fcst_cv_naive['fitted']['values'])"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -1333,7 +1385,7 @@
" self._set_prediction_intervals(prediction_intervals=prediction_intervals)\n",
" self._prepare_fit(df, sort_df)\n",
" if self.n_jobs == 1:\n",
" self.fitted_ = self.ga.fit(models=self.models)\n",
" self.fitted_ = self.ga.fit(models=self.models, fallback_model=self.fallback_model)\n",
" else:\n",
" self.fitted_ = self._fit_parallel()\n",
" return self\n",
Expand Down Expand Up @@ -1725,10 +1777,10 @@
" with Pool(self.n_jobs, **pool_kwargs) as executor:\n",
" futures = []\n",
" for ga in gas:\n",
" future = executor.apply_async(ga.fit, (self.models,))\n",
" future = executor.apply_async(ga.fit, (self.models, self.fallback_model))\n",
" futures.append(future)\n",
" fm = np.vstack([f.get() for f in futures])\n",
" return fm\n",
" return fm \n",
" \n",
" def _get_gas_Xs(self, X):\n",
" gas = self.ga.split(self.n_jobs)\n",
Expand Down
123 changes: 123 additions & 0 deletions nbs/src/core/models.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5412,6 +5412,40 @@
" residuals = y - out[\"fitted\"]\n",
" sigma = _calculate_sigma(residuals, len(residuals) - 1)\n",
" res = _add_fitted_pi(res=res, se=sigma, level=level)\n",
" return res\n",
" \n",
" def forward(\n",
" self, \n",
" y: np.ndarray,\n",
" h: int,\n",
" X: Optional[np.ndarray] = None,\n",
" X_future: Optional[np.ndarray] = None,\n",
" level: Optional[List[int]] = None,\n",
" fitted: bool = False,\n",
" ):\n",
" \"\"\" Apply fitted model to an new/updated series.\n",
"\n",
" Parameters\n",
" ----------\n",
" y : numpy.array \n",
" Clean time series of shape (n,). \n",
" h: int\n",
" Forecast horizon.\n",
" X : array-like\n",
" Optional insample exogenous of shape (t, n_x). \n",
" X_future : array-like\n",
" Optional exogenous of shape (h, n_x). \n",
" level : List[float]\n",
" Confidence levels (0-100) for prediction intervals.\n",
" fitted : bool\n",
" Whether or not to return insample predictions.\n",
"\n",
" Returns\n",
" -------\n",
" forecasts : dict\n",
" Dictionary with entries `mean` for point predictions and `level_*` for probabilistic predictions.\n",
" \"\"\"\n",
" res = self.forecast(y=y, h=h, X=X, X_future=X_future, level=level, fitted=fitted)\n",
" return res"
]
},
Expand Down Expand Up @@ -5472,6 +5506,23 @@
"_plot_insample_pi(fcst_naive)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"# Unit test forward:=forecast\n",
"naive = Naive()\n",
"fcst_naive = naive.forward(ap,12,None,None,(80,95), True)\n",
"np.testing.assert_almost_equal(\n",
" fcst_naive['lo-80'],\n",
" np.array([388.7984, 370.9037, 357.1726, 345.5967, 335.3982, 326.1781, 317.6992, 309.8073, 302.3951, 295.3845, 288.7164, 282.3452]),\n",
" decimal=4\n",
") # this is almost equal since Hyndman's forecasts are rounded up to 4 decimals"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -10880,6 +10931,40 @@
" if fitted:\n",
" res[f'fitted-lo-{lv}'] = fitted_vals\n",
" res[f'fitted-hi-{lv}'] = fitted_vals\n",
" return res\n",
" \n",
" def forward(\n",
" self,\n",
" y: np.ndarray,\n",
" h: int,\n",
" X: Optional[np.ndarray] = None,\n",
" X_future: Optional[np.ndarray] = None,\n",
" level: Optional[List[int]] = None,\n",
" fitted: bool = False,\n",
" ):\n",
" \"\"\"Apply Constant model predictions to a new/updated time series.\n",
"\n",
" Parameters\n",
" ----------\n",
" y : numpy.array \n",
" Clean time series of shape (n, ). \n",
" h : int \n",
" Forecast horizon.\n",
" X : array-like \n",
" Optional insample exogenous of shape (t, n_x). \n",
" X_future : array-like \n",
" Optional exogenous of shape (h, n_x). \n",
" level : List[float]\n",
" Confidence levels for prediction intervals.\n",
" fitted : bool \n",
" Whether or not returns insample predictions.\n",
"\n",
" Returns\n",
" -------\n",
" forecasts : dict \n",
" Dictionary with entries `constant` for point predictions and `level_*` for probabilistic predictions.\n",
" \"\"\"\n",
" res = self.forecast(y=y, h=h, X=X, X_future=X_future, level=level, fitted=fitted)\n",
" return res"
]
},
Expand All @@ -10904,6 +10989,16 @@
"constant_model.forecast(ap, 12, level=[90, 80])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"constant_model.forward(ap, 12, level=[90, 80])"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -10967,6 +11062,15 @@
"show_doc(ConstantModel.predict_in_sample, title_level=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"show_doc(ConstantModel.forward, title_level=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -11035,6 +11139,16 @@
"zero_model.forecast(ap, 12, level=[90, 80])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| hide\n",
"zero_model.forward(ap, 12, level=[90, 80])"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -11098,6 +11212,15 @@
"show_doc(ZeroModel.predict_in_sample, title_level=3, name='ZeroModel.predict_in_sample')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"show_doc(ZeroModel.forward, title_level=3, name='ZeroModel.forward')"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
4 changes: 4 additions & 0 deletions statsforecast/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,8 @@
'statsforecast/models.py'),
'statsforecast.models.ConstantModel.forecast': ( 'src/core/models.html#constantmodel.forecast',
'statsforecast/models.py'),
'statsforecast.models.ConstantModel.forward': ( 'src/core/models.html#constantmodel.forward',
'statsforecast/models.py'),
'statsforecast.models.ConstantModel.predict': ( 'src/core/models.html#constantmodel.predict',
'statsforecast/models.py'),
'statsforecast.models.ConstantModel.predict_in_sample': ( 'src/core/models.html#constantmodel.predict_in_sample',
Expand Down Expand Up @@ -498,6 +500,8 @@
'statsforecast.models.Naive.fit': ('src/core/models.html#naive.fit', 'statsforecast/models.py'),
'statsforecast.models.Naive.forecast': ( 'src/core/models.html#naive.forecast',
'statsforecast/models.py'),
'statsforecast.models.Naive.forward': ( 'src/core/models.html#naive.forward',
'statsforecast/models.py'),
'statsforecast.models.Naive.predict': ( 'src/core/models.html#naive.predict',
'statsforecast/models.py'),
'statsforecast.models.Naive.predict_in_sample': ( 'src/core/models.html#naive.predict_in_sample',
Expand Down
Loading
Loading