From edb4500cb091659a13b725b448937986018496eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Morales?= Date: Mon, 9 Oct 2023 16:20:04 -0600 Subject: [PATCH] isolate import cells in distributed notebooks --- nbs/src/core/distributed.fugue.ipynb | 74 ++++++++++++++++++++------ nbs/src/distributed.core.ipynb | 15 ++++-- nbs/src/distributed.multiprocess.ipynb | 15 ++++-- nbs/src/distributed.utils.ipynb | 16 ++++-- 4 files changed, 96 insertions(+), 24 deletions(-) diff --git a/nbs/src/core/distributed.fugue.ipynb b/nbs/src/core/distributed.fugue.ipynb index a34981c4a..0f1a6f892 100644 --- a/nbs/src/core/distributed.fugue.ipynb +++ b/nbs/src/core/distributed.fugue.ipynb @@ -298,7 +298,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b5369129", + "id": "1f2ba783-2d77-47de-aa85-fd41ec068889", "metadata": {}, "outputs": [], "source": [ @@ -307,8 +307,16 @@ " AutoARIMA,\n", " AutoETS,\n", ")\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d5549b2-14d3-4bb6-9247-6e66acca8e45", + "metadata": {}, + "outputs": [], + "source": [ "n_series = 4\n", "horizon = 7\n", "\n", @@ -326,12 +334,20 @@ { "cell_type": "code", "execution_count": null, - "id": "3d84def8", + "id": "8d8d3791-9d0a-4510-900e-51db0b5abe73", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85bd5dc4-54c8-4022-a85d-69583fc6bc37", "metadata": {}, "outputs": [], "source": [ - "from pyspark.sql import SparkSession\n", - "\n", "spark = SparkSession.builder.getOrCreate()\n", "\n", "# Make unique_id a column\n", @@ -371,7 +387,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e2df29ce-c1ac-44d9-829e-47096adf2917", + "id": "f7e38ec9-b093-40fe-9a31-85d84ccb1b6d", "metadata": {}, "outputs": [], "source": [ @@ -381,8 +397,16 @@ "from fugue_dask import DaskExecutionEngine\n", "from statsforecast import StatsForecast\n", "from statsforecast.models import Naive\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0174b4d3-1262-48ca-9ca1-9f1ace044b77", + "metadata": {}, + "outputs": [], + "source": [ "# Generate Synthetic Panel Data\n", "df = generate_series(10).reset_index()\n", "df['unique_id'] = df['unique_id'].astype(str)\n", @@ -557,15 +581,25 @@ { "cell_type": "code", "execution_count": null, - "id": "4d95e09b-cb70-4232-8ffa-b26fc8aea557", + "id": "814bc2f1-8ae4-46b3-a324-80a1d1e57dad", "metadata": {}, "outputs": [], "source": [ "#| hide\n", "#| eval: false\n", "from statsforecast.models import Naive\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ea06c4d-9577-4c88-9808-8268c08c76fe", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "#| eval: false\n", "# Generate Synthetic Panel Data.\n", "df = generate_series(10).reset_index()\n", "df['unique_id'] = df['unique_id'].astype(str)\n", @@ -600,7 +634,7 @@ { "cell_type": "code", "execution_count": null, - "id": "dc1084e8-c722-48d5-a038-8d7e530773bd", + "id": "598617b9-2a8c-4fdc-b751-bc43702fa163", "metadata": {}, "outputs": [], "source": [ @@ -609,8 +643,18 @@ "# test ray integration\n", "import ray\n", "from statsforecast.models import Naive\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9372bc61-4f39-4c04-8d39-d8270bc89b27", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "#| eval: false\n", "# Generate Synthetic Panel Data.\n", "df = generate_series(10).reset_index()\n", "df['unique_id'] = df['unique_id'].astype(str)\n", diff --git a/nbs/src/distributed.core.ipynb b/nbs/src/distributed.core.ipynb index d0c087f40..1b6e25dce 100644 --- a/nbs/src/distributed.core.ipynb +++ b/nbs/src/distributed.core.ipynb @@ -47,15 +47,24 @@ { "cell_type": "code", "execution_count": null, - "id": "4d95e09b-cb70-4232-8ffa-b26fc8aea557", + "id": "b3f4f5d0-48e5-4ee4-a593-78eb2c631b6b", "metadata": {}, "outputs": [], "source": [ "#| hide\n", "from statsforecast import StatsForecast\n", "from statsforecast.models import Naive\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49044ce2-3525-4053-b292-08b438c0f5df", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", "df = generate_series(10).reset_index()\n", "df['unique_id'] = df['unique_id'].astype(str)\n", "\n", diff --git a/nbs/src/distributed.multiprocess.ipynb b/nbs/src/distributed.multiprocess.ipynb index 3e6f34040..9c509d826 100644 --- a/nbs/src/distributed.multiprocess.ipynb +++ b/nbs/src/distributed.multiprocess.ipynb @@ -109,15 +109,24 @@ { "cell_type": "code", "execution_count": null, - "id": "fbc8985d-75f8-448f-afd1-a33d47786c74", + "id": "83a1bcd0-31bc-48db-a531-551927f463b8", "metadata": {}, "outputs": [], "source": [ "#| hide\n", "from statsforecast import StatsForecast\n", "from statsforecast.models import Naive\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2891e5fe-8d82-4eb9-8724-96317f157507", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", "df = generate_series(10).reset_index()\n", "df['unique_id'] = df['unique_id'].astype(str)\n", "\n", diff --git a/nbs/src/distributed.utils.ipynb b/nbs/src/distributed.utils.ipynb index b4697e914..30cca5cd5 100644 --- a/nbs/src/distributed.utils.ipynb +++ b/nbs/src/distributed.utils.ipynb @@ -111,7 +111,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4d95e09b-cb70-4232-8ffa-b26fc8aea557", + "id": "53a39c59-8bc1-4cf4-b696-29204b3a144d", "metadata": {}, "outputs": [], "source": [ @@ -120,8 +120,18 @@ "from statsforecast.core import StatsForecast\n", "from statsforecast.distributed.fugue import FugueBackend\n", "from statsforecast.models import Naive\n", - "from statsforecast.utils import generate_series\n", - "\n", + "from statsforecast.utils import generate_series" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c6626cc-0701-4b96-a56f-ff73f6c94259", + "metadata": {}, + "outputs": [], + "source": [ + "#| hide\n", + "#| eval: false\n", "df = generate_series(10).reset_index()\n", "df['unique_id'] = df['unique_id'].astype(str)\n", "\n",