+
+
+
+
+
+
+
+
+ERA5 - check inputĀ¶
+
+
+
+
+
+
+
+InĀ [1]:
+
+
+
+
+
+%matplotlib widget
+
+import cartopy
+import hvplot.pandas
+import hvplot.xarray
+import holoviews as hv
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import xarray as xr
+
+hv.extension("bokeh")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+InĀ [2]:
+
+
+
+
+
+from dask.distributed import LocalCluster
+cluster = LocalCluster() # Fully-featured local Dask cluster
+cluster
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+LocalCluster
+519dca2d
+| +Dashboard: http://127.0.0.1:8787/status + | ++Workers: 16 + | +
| +Total threads: 128 + | ++Total memory: 503.23 GiB + | +
| Status: running | +Using processes: True | +
+
+
+
+
+Scheduler Info
+
+
+
+
+
+
+
+
+
+
+Scheduler
+Scheduler-c351c464-c09e-4b07-85fc-b46bb5befc69
+| +Comm: tcp://127.0.0.1:37059 + | ++Workers: 16 + | +
| +Dashboard: http://127.0.0.1:8787/status + | ++Total threads: 128 + | +
| +Started: Just now + | ++Total memory: 503.23 GiB + | +
+
+
+
+
+Workers
+
+
+
+
+
+
+
+
+
+
+
+
+
+Worker: 0
+
+| +Comm: tcp://127.0.0.1:36521 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:38785/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:39839 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-z63mdpdf + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 1
+
+| +Comm: tcp://127.0.0.1:42761 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:37451/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:43043 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-d4c2i6dl + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 2
+
+| +Comm: tcp://127.0.0.1:36953 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:46213/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:37197 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-98rgf9a9 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 3
+
+| +Comm: tcp://127.0.0.1:43711 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:39249/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:38279 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-mpy5sdpc + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 4
+
+| +Comm: tcp://127.0.0.1:34115 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:33589/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:39471 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-4i9f1pi8 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 5
+
+| +Comm: tcp://127.0.0.1:43505 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:40169/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:41291 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-l_vibs6v + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 6
+
+| +Comm: tcp://127.0.0.1:39907 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:36147/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:36495 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-ms0c3209 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 7
+
+| +Comm: tcp://127.0.0.1:43691 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:36359/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:44553 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-kiora_y2 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 8
+
+| +Comm: tcp://127.0.0.1:45865 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:44993/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:42221 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-pub4k30e + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 9
+
+| +Comm: tcp://127.0.0.1:41435 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:32933/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:34277 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-bfdvest5 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 10
+
+| +Comm: tcp://127.0.0.1:35149 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:40203/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:45451 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-8398x6xi + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 11
+
+| +Comm: tcp://127.0.0.1:43611 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:42795/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:45203 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-dif3g46o + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 12
+
+| +Comm: tcp://127.0.0.1:43565 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:44341/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:34125 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-jxetj_q2 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 13
+
+| +Comm: tcp://127.0.0.1:41645 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:42935/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:35747 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-ndew7a7i + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 14
+
+| +Comm: tcp://127.0.0.1:34209 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:33415/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:46773 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-qltdbyo1 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 15
+
+| +Comm: tcp://127.0.0.1:42455 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:45807/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:41149 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-fu35373g + | +|
+
+
+
+
+
+
+InĀ [3]:
+
+
+
+
+
+client = cluster.get_client()
+client
+
+
+
+
+
+
+
+
+Out[3]:
+
+
+
+
+
+
+
+
+
+Client
+Client-544adab0-0493-11ef-b4f4-00000f53fe80
+| Connection method: Cluster object | +Cluster type: distributed.LocalCluster | +
| +Dashboard: http://127.0.0.1:8787/status + | ++ |
+
+
+Cluster Info
+
+
+
+
+
+
+
+
+LocalCluster
+519dca2d
+| +Dashboard: http://127.0.0.1:8787/status + | ++Workers: 16 + | +
| +Total threads: 128 + | ++Total memory: 503.23 GiB + | +
| Status: running | +Using processes: True | +
+
+
+
+
+Scheduler Info
+
+
+
+
+
+
+
+
+
+
+Scheduler
+Scheduler-c351c464-c09e-4b07-85fc-b46bb5befc69
+| +Comm: tcp://127.0.0.1:37059 + | ++Workers: 16 + | +
| +Dashboard: http://127.0.0.1:8787/status + | ++Total threads: 128 + | +
| +Started: Just now + | ++Total memory: 503.23 GiB + | +
+
+
+
+
+Workers
+
+
+
+
+
+
+
+
+
+
+
+
+
+Worker: 0
+
+| +Comm: tcp://127.0.0.1:36521 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:38785/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:39839 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-z63mdpdf + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 1
+
+| +Comm: tcp://127.0.0.1:42761 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:37451/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:43043 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-d4c2i6dl + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 2
+
+| +Comm: tcp://127.0.0.1:36953 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:46213/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:37197 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-98rgf9a9 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 3
+
+| +Comm: tcp://127.0.0.1:43711 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:39249/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:38279 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-mpy5sdpc + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 4
+
+| +Comm: tcp://127.0.0.1:34115 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:33589/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:39471 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-4i9f1pi8 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 5
+
+| +Comm: tcp://127.0.0.1:43505 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:40169/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:41291 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-l_vibs6v + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 6
+
+| +Comm: tcp://127.0.0.1:39907 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:36147/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:36495 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-ms0c3209 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 7
+
+| +Comm: tcp://127.0.0.1:43691 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:36359/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:44553 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-kiora_y2 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 8
+
+| +Comm: tcp://127.0.0.1:45865 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:44993/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:42221 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-pub4k30e + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 9
+
+| +Comm: tcp://127.0.0.1:41435 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:32933/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:34277 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-bfdvest5 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 10
+
+| +Comm: tcp://127.0.0.1:35149 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:40203/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:45451 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-8398x6xi + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 11
+
+| +Comm: tcp://127.0.0.1:43611 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:42795/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:45203 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-dif3g46o + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 12
+
+| +Comm: tcp://127.0.0.1:43565 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:44341/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:34125 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-jxetj_q2 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 13
+
+| +Comm: tcp://127.0.0.1:41645 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:42935/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:35747 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-ndew7a7i + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 14
+
+| +Comm: tcp://127.0.0.1:34209 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:33415/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:46773 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-qltdbyo1 + | +|
+
+
+
+
+
+
+
+
+
+
+
+Worker: 15
+
+| +Comm: tcp://127.0.0.1:42455 + | ++Total threads: 8 + | +
| +Dashboard: http://127.0.0.1:45807/status + | ++Memory: 31.45 GiB + | +
| +Nanny: tcp://127.0.0.1:41149 + | ++ |
| +Local directory: /tmp/dask-scratch-space-101348/worker-fu35373g + | +|
+
+
+
+
+
+
+InĀ [4]:
+
+
+
+
+
+chunks = {}
+chunks = dict(
+ time=120,
+ longitude=20,
+ latitude=20,
+)
+
+era5 = xr.open_mfdataset(
+ "/project/home/p200232/02_meteo/era5/lon_lat/netcdf/2023*.nc",
+ chunks=chunks,
+)
+era5
+
+
+
+
+
+
+
+
+Out[4]:
+
+
+
+
+<xarray.Dataset> Size: 218GB +Dimensions: (longitude: 1440, latitude: 721, time: 8760) +Coordinates: + * longitude (longitude) float32 6kB 0.0 0.25 0.5 0.75 ... 359.2 359.5 359.8 + * latitude (latitude) float32 3kB 90.0 89.75 89.5 ... -89.5 -89.75 -90.0 + * time (time) datetime64[ns] 70kB 2023-01-01 ... 2023-12-31T23:00:00 +Data variables: + msl (time, latitude, longitude) float64 73GB dask.array<chunksize=(120, 20, 20), meta=np.ndarray> + u10 (time, latitude, longitude) float64 73GB dask.array<chunksize=(120, 20, 20), meta=np.ndarray> + v10 (time, latitude, longitude) float64 73GB dask.array<chunksize=(120, 20, 20), meta=np.ndarray> +Attributes: + Conventions: CF-1.6 + history: 2024-04-04 09:42:07 GMT by grib_to_netcdf-2.25.1: /opt/ecmw...
+
+
+
+
+
+
+InĀ [5]:
+
+
+
+
+
+window = dict(
+ time=slice("2023-07-12", "2023-07-17"),
+ latitude=slice(30, 0),
+ longitude=slice(210, 260),
+)
+
+era5_window = era5.sel(**window).load()
+
+
+
+
+
+
+
+InĀ [6]:
+
+
+
+
+
+# Convert MSL to kPa!
+if era5_window.msl.max() > 10_000:
+ era5_window["msl"] = era5_window.msl / 1000
+
+
+
+
+
+
+
+InĀ [7]:
+
+
+
+
+
+plot_options = dict(frame_width=500)
+
+
+
+
+
+
+
+InĀ [8]:
+
+
+
+
+
+msl_min = era5_window.msl.min("time")
+
+
+
+
+
+
+
+InĀ [9]:
+
+
+
+
+
+msl_min.hvplot(
+ x="longitude",
+ y="latitude",
+ title="min MSL (kPa)",
+ geo=True,
+ coastline=True,
+).options(**plot_options)
+
+
+
+
+
+
+
+
+
+
+
+Out[9]:
+
+
+
+
+
+
+
+
+
+
+
+
+
+InĀ [10]:
+
+
+
+
+
+# Contours make things a bit clearer
+msl_min.hvplot.contourf(
+ x="longitude",
+ y="latitude",
+ title="min msl (kPa)",
+ levels=10,
+ geo=True,
+ coastline=True,
+).options(**plot_options)
+
+
+
+
+
+
+
+
+
+
+
+Out[10]:
+
+
+
+
+
+
+
+
+
+
+
+
+
+InĀ [11]:
+
+
+
+
+
+era5_window.msl.groupby("time", squeeze=False).map(np.min).hvplot(grid=True, title="min MSL over window (kPa)")
+
+
+
+
+
+
+
+
+
+
+
+Out[11]:
+
+
+
+
+
+
+
+
+
+
+
+
+
+InĀ [12]:
+
+
+
+
+
+# Animation
+era5_window.msl.hvplot.contourf(
+ groupby="time",
+ x="longitude",
+ y="latitude",
+ levels=15,
+ geo=True,
+ coastline=True,
+ widget_type="scrubber",
+ widget_location="bottom",
+)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[12]:
+
+
+
+
+
+
+
+
+
+
+
+
+
+InĀ [13]:
+
+
+
+
+
+era5_window["mag"] = np.sqrt(era5_window.u10**2 + era5_window.v10**2)
+era5_window["angle"] = (np.pi/2.) - np.arctan2(era5_window.v10, era5_window.u10) # WTF?? I had to switch U and V in order to get the arrows to show correctly...
+
+
+
+
+
+
+
+InĀ [14]:
+
+
+
+
+
+ww = dict(latitude=slice(20, 10), longitude=slice(230, 240))
+era5_window.sel(time="2023-07-14T21:00:00", **ww).hvplot.vectorfield(
+ #groupby="time",
+ x="longitude",
+ y="latitude",
+ angle='angle',
+ mag='mag',
+ hover=False,
+ geo=True,
+ #widget_type="scrubber",
+ #widget_location="bottom",
+).opts(magnitude='mag', pivot="tip") + \
+era5_window.sel(time="2023-07-14T14:00:00", **ww).hvplot.vectorfield(
+ #groupby="time",
+ x="longitude",
+ y="latitude",
+ angle='angle',
+ mag='mag',
+ hover=False,
+ geo=True,
+ #widget_type="scrubber",
+ #widget_location="bottom",
+).opts(magnitude='mag')
+
+
+
+
+
+
+
+
+
+
+
+Out[14]:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+InĀ [15]:
+
+
+
+
+
+time_slice = slice("2023-07-12", "2023-07-17")
+longitude=235
+latitude=40
+hv.Layout([
+ era5.sel(longitude=longitude, latitude=latitude, time=time_slice).msl.hvplot(grid=True),
+ era5.sel(longitude=longitude, latitude=0, time=time_slice).msl.hvplot(grid=True),
+ era5.sel(longitude=longitude, latitude=-latitude, time=time_slice).msl.hvplot(grid=True),
+
+]).cols(1)
+
+
+
+
+
+
+
+
+
+
+
+Out[15]:
+
+
+
+
+
+
+