"
@@ -804,17 +230,17 @@
" NaN | \n",
" NaN | \n",
" NaN | \n",
- " 0.100 | \n",
- " NaN | \n",
- " NaN | \n",
- " 0.005730 | \n",
- " 583.0 | \n",
- " 405.0 | \n",
- " 0.000014 | \n",
- " 8.118279 | \n",
- " 126.880731 | \n",
- " 11.264135 | \n",
- " 209.805326 | \n",
+ " 0.055 | \n",
+ " 0.0541 | \n",
+ " 0.0608 | \n",
+ " 0.006797 | \n",
+ " 582.0 | \n",
+ " 407.0 | \n",
+ " 0.000019 | \n",
+ " 8.181382 | \n",
+ " 127.722282 | \n",
+ " 11.301428 | \n",
+ " 212.322965 | \n",
" LINESTRING (174.9093 -36.86951, 174.91331 -36.... | \n",
" \n",
" \n",
@@ -823,17 +249,17 @@
" NaN | \n",
" NaN | \n",
" NaN | \n",
- " 0.100 | \n",
- " NaN | \n",
- " NaN | \n",
- " 0.424099 | \n",
- " 583.0 | \n",
- " 365.0 | \n",
- " 0.045716 | \n",
- " 9.717575 | \n",
- " 202.294073 | \n",
- " 14.223012 | \n",
- " 208.659953 | \n",
+ " 0.045 | \n",
+ " 0.0434 | \n",
+ " 0.0461 | \n",
+ " 0.284894 | \n",
+ " 582.0 | \n",
+ " 372.0 | \n",
+ " 0.022977 | \n",
+ " 9.773159 | \n",
+ " 187.352460 | \n",
+ " 13.687675 | \n",
+ " 216.532589 | \n",
" LINESTRING (174.90891 -36.86889, 174.91293 -36... | \n",
"
\n",
" \n",
@@ -842,17 +268,17 @@
" NaN | \n",
" NaN | \n",
" NaN | \n",
- " 0.100 | \n",
- " NaN | \n",
- " NaN | \n",
- " -0.277580 | \n",
+ " 0.200 | \n",
+ " 0.1756 | \n",
+ " 0.2000 | \n",
+ " -0.286535 | \n",
" 770.0 | \n",
- " 683.0 | \n",
- " 0.018699 | \n",
- " 13.191099 | \n",
- " 225.924210 | \n",
- " 15.030775 | \n",
- " 182.873838 | \n",
+ " 684.0 | \n",
+ " 0.020982 | \n",
+ " 12.984757 | \n",
+ " 214.309394 | \n",
+ " 14.639310 | \n",
+ " 182.918496 | \n",
" LINESTRING (176.9374 -39.3345, 176.938 -39.33807) | \n",
"
\n",
" \n",
@@ -861,17 +287,17 @@
" NaN | \n",
" NaN | \n",
" NaN | \n",
- " 0.100 | \n",
- " NaN | \n",
- " NaN | \n",
- " -0.062965 | \n",
+ " 0.050 | \n",
+ " 0.0323 | \n",
+ " 0.1200 | \n",
+ " -0.072549 | \n",
" 770.0 | \n",
" 614.0 | \n",
- " 0.007388 | \n",
- " 4.436646 | \n",
- " 30.759972 | \n",
- " 5.546167 | \n",
- " 437.106630 | \n",
+ " 0.003174 | \n",
+ " 8.484539 | \n",
+ " 95.454639 | \n",
+ " 9.770089 | \n",
+ " 437.079989 | \n",
" LINESTRING (176.93648 -39.33459, 176.93708 -39... | \n",
"
\n",
" \n",
@@ -880,17 +306,17 @@
" NaN | \n",
" NaN | \n",
" NaN | \n",
- " 0.100 | \n",
- " NaN | \n",
- " NaN | \n",
- " -0.254745 | \n",
+ " 0.070 | \n",
+ " 0.0624 | \n",
+ " 0.0742 | \n",
+ " -0.260641 | \n",
" 770.0 | \n",
" 632.0 | \n",
- " 0.024568 | \n",
- " 9.215089 | \n",
- " 146.102315 | \n",
- " 12.087279 | \n",
- " 172.828009 | \n",
+ " 0.026072 | \n",
+ " 9.013683 | \n",
+ " 143.900611 | \n",
+ " 11.995858 | \n",
+ " 173.017496 | \n",
" LINESTRING (176.93431 -39.33569, 176.93792 -39... | \n",
"
\n",
" \n",
@@ -907,11 +333,11 @@
"aus0001-0003 aus0001 81.192757 298.402523 0.008900 0.055 \n",
"aus0001-0004 aus0001 81.065473 398.402523 0.011882 0.075 \n",
"... ... ... ... ... ... \n",
- "nzd0562-0017 nzd0562 NaN NaN NaN 0.100 \n",
- "nzd0562-0018 nzd0562 NaN NaN NaN 0.100 \n",
- "nzd0563-0000 nzd0563 NaN NaN NaN 0.100 \n",
- "nzd0563-0001 nzd0563 NaN NaN NaN 0.100 \n",
- "nzd0563-0002 nzd0563 NaN NaN NaN 0.100 \n",
+ "nzd0562-0017 nzd0562 NaN NaN NaN 0.055 \n",
+ "nzd0562-0018 nzd0562 NaN NaN NaN 0.045 \n",
+ "nzd0563-0000 nzd0563 NaN NaN NaN 0.200 \n",
+ "nzd0563-0001 nzd0563 NaN NaN NaN 0.050 \n",
+ "nzd0563-0002 nzd0563 NaN NaN NaN 0.070 \n",
"\n",
" cil ciu trend n_points n_points_nonan r2_score \\\n",
"id \n",
@@ -921,11 +347,11 @@
"aus0001-0003 0.0480 0.0659 0.089340 654.0 502.0 0.000987 \n",
"aus0001-0004 0.0614 0.0922 0.361437 654.0 508.0 0.017461 \n",
"... ... ... ... ... ... ... \n",
- "nzd0562-0017 NaN NaN 0.005730 583.0 405.0 0.000014 \n",
- "nzd0562-0018 NaN NaN 0.424099 583.0 365.0 0.045716 \n",
- "nzd0563-0000 NaN NaN -0.277580 770.0 683.0 0.018699 \n",
- "nzd0563-0001 NaN NaN -0.062965 770.0 614.0 0.007388 \n",
- "nzd0563-0002 NaN NaN -0.254745 770.0 632.0 0.024568 \n",
+ "nzd0562-0017 0.0541 0.0608 0.006797 582.0 407.0 0.000019 \n",
+ "nzd0562-0018 0.0434 0.0461 0.284894 582.0 372.0 0.022977 \n",
+ "nzd0563-0000 0.1756 0.2000 -0.286535 770.0 684.0 0.020982 \n",
+ "nzd0563-0001 0.0323 0.1200 -0.072549 770.0 614.0 0.003174 \n",
+ "nzd0563-0002 0.0624 0.0742 -0.260641 770.0 632.0 0.026072 \n",
"\n",
" mae mse rmse intercept \\\n",
"id \n",
@@ -935,11 +361,11 @@
"aus0001-0003 20.770475 691.179232 26.290288 185.295473 \n",
"aus0001-0004 19.528839 637.676213 25.252252 169.027861 \n",
"... ... ... ... ... \n",
- "nzd0562-0017 8.118279 126.880731 11.264135 209.805326 \n",
- "nzd0562-0018 9.717575 202.294073 14.223012 208.659953 \n",
- "nzd0563-0000 13.191099 225.924210 15.030775 182.873838 \n",
- "nzd0563-0001 4.436646 30.759972 5.546167 437.106630 \n",
- "nzd0563-0002 9.215089 146.102315 12.087279 172.828009 \n",
+ "nzd0562-0017 8.181382 127.722282 11.301428 212.322965 \n",
+ "nzd0562-0018 9.773159 187.352460 13.687675 216.532589 \n",
+ "nzd0563-0000 12.984757 214.309394 14.639310 182.918496 \n",
+ "nzd0563-0001 8.484539 95.454639 9.770089 437.079989 \n",
+ "nzd0563-0002 9.013683 143.900611 11.995858 173.017496 \n",
"\n",
" geometry \n",
"id \n",
@@ -958,7 +384,7 @@
"[127429 rows x 16 columns]"
]
},
- "execution_count": 21,
+ "execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
@@ -970,13 +396,13 @@
},
{
"cell_type": "code",
- "execution_count": 47,
+ "execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
- "✔️ 57.2 ms (2024-12-03T11:34:06/2024-12-03T11:34:06)
"
+ "✔️ 56.1 ms (2024-12-03T11:48:24/2024-12-03T11:48:24)
"
],
"text/plain": [
""
@@ -985,33 +411,6 @@
"metadata": {},
"output_type": "display_data"
},
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/tmp/ipykernel_980372/4139878928.py:1: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
- "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
- "\n",
- "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
- "\n",
- "\n",
- " transects.beach_slope.update(slope_est)\n",
- "/tmp/ipykernel_980372/4139878928.py:2: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
- "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
- "\n",
- "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
- "\n",
- "\n",
- " transects.cil.update({k: v[0] for k,v in cis.items()})\n",
- "/tmp/ipykernel_980372/4139878928.py:3: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.\n",
- "The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.\n",
- "\n",
- "For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.\n",
- "\n",
- "\n",
- " transects.ciu.update({k: v[1] for k,v in cis.items()})\n"
- ]
- },
{
"data": {
"text/html": [
@@ -1071,114 +470,35 @@
" \n",
" \n",
" \n",
- " \n",
- " nzd0563-0000 | \n",
- " nzd0563 | \n",
- " NaN | \n",
- " NaN | \n",
- " NaN | \n",
- " 0.20 | \n",
- " 0.1756 | \n",
- " 0.2000 | \n",
- " -0.277580 | \n",
- " 770.0 | \n",
- " 683.0 | \n",
- " 0.018699 | \n",
- " 13.191099 | \n",
- " 225.924210 | \n",
- " 15.030775 | \n",
- " 182.873838 | \n",
- " LINESTRING (176.9374 -39.3345, 176.938 -39.33807) | \n",
- "
\n",
- " \n",
- " nzd0563-0001 | \n",
- " nzd0563 | \n",
- " NaN | \n",
- " NaN | \n",
- " NaN | \n",
- " 0.05 | \n",
- " 0.0323 | \n",
- " 0.1200 | \n",
- " -0.062965 | \n",
- " 770.0 | \n",
- " 614.0 | \n",
- " 0.007388 | \n",
- " 4.436646 | \n",
- " 30.759972 | \n",
- " 5.546167 | \n",
- " 437.106630 | \n",
- " LINESTRING (176.93648 -39.33459, 176.93708 -39... | \n",
- "
\n",
- " \n",
- " nzd0563-0002 | \n",
- " nzd0563 | \n",
- " NaN | \n",
- " NaN | \n",
- " NaN | \n",
- " 0.07 | \n",
- " 0.0624 | \n",
- " 0.0742 | \n",
- " -0.254745 | \n",
- " 770.0 | \n",
- " 632.0 | \n",
- " 0.024568 | \n",
- " 9.215089 | \n",
- " 146.102315 | \n",
- " 12.087279 | \n",
- " 172.828009 | \n",
- " LINESTRING (176.93431 -39.33569, 176.93792 -39... | \n",
- "
\n",
" \n",
"\n",
""
],
"text/plain": [
- " site_id orientation along_dist along_dist_norm beach_slope \\\n",
- "id \n",
- "nzd0563-0000 nzd0563 NaN NaN NaN 0.20 \n",
- "nzd0563-0001 nzd0563 NaN NaN NaN 0.05 \n",
- "nzd0563-0002 nzd0563 NaN NaN NaN 0.07 \n",
- "\n",
- " cil ciu trend n_points n_points_nonan r2_score \\\n",
- "id \n",
- "nzd0563-0000 0.1756 0.2000 -0.277580 770.0 683.0 0.018699 \n",
- "nzd0563-0001 0.0323 0.1200 -0.062965 770.0 614.0 0.007388 \n",
- "nzd0563-0002 0.0624 0.0742 -0.254745 770.0 632.0 0.024568 \n",
- "\n",
- " mae mse rmse intercept \\\n",
- "id \n",
- "nzd0563-0000 13.191099 225.924210 15.030775 182.873838 \n",
- "nzd0563-0001 4.436646 30.759972 5.546167 437.106630 \n",
- "nzd0563-0002 9.215089 146.102315 12.087279 172.828009 \n",
- "\n",
- " geometry \n",
- "id \n",
- "nzd0563-0000 LINESTRING (176.9374 -39.3345, 176.938 -39.33807) \n",
- "nzd0563-0001 LINESTRING (176.93648 -39.33459, 176.93708 -39... \n",
- "nzd0563-0002 LINESTRING (176.93431 -39.33569, 176.93792 -39... "
+ "Empty GeoDataFrame\n",
+ "Columns: [site_id, orientation, along_dist, along_dist_norm, beach_slope, cil, ciu, trend, n_points, n_points_nonan, r2_score, mae, mse, rmse, intercept, geometry]\n",
+ "Index: []"
]
},
- "execution_count": 47,
+ "execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "transects.beach_slope.update(slope_est)\n",
- "transects.cil.update({k: v[0] for k,v in cis.items()})\n",
- "transects.ciu.update({k: v[1] for k,v in cis.items()})\n",
- "transects[transects.index.isin(slope_est.keys())]"
+ "new_transects = transects[transects.index.str.startswith(\"nzd\") & (transects.index > \"nzd0562\") & transects.beach_slope.isna()]\n",
+ "new_transects"
]
},
{
"cell_type": "code",
- "execution_count": 50,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
- "✔️ 4.06 s (2024-12-03T11:34:29/2024-12-03T11:34:33)
"
+ "✔️ 3.11 ms (2024-12-03T11:48:24/2024-12-03T11:48:24)
"
],
"text/plain": [
""
@@ -1189,7 +509,64 @@
}
],
"source": [
- "transects.to_file(\"transects_extended.geojson\")"
+ "if len(new_transects):\n",
+ " for site_id in tqdm(new_transects.site_id.unique()):\n",
+ " df = pd.read_csv(f\"data/{site_id}/transect_time_series.csv\")\n",
+ " df.index = pd.to_datetime(df.dates)\n",
+ " df.drop(columns=[\"dates\", \"satname\"], inplace=True)\n",
+ " tides = pd.read_csv(\"data/{site_id}/tides.csv\")\n",
+ " tides.dates = pd.to_datetime(tides.dates)\n",
+ " tides.set_index(\"dates\", inplace=True)\n",
+ " assert all(pd.to_datetime(df.index).round(\"10min\") == tides.index)\n",
+ " # slope estimation settings\n",
+ " days_in_year = 365.2425\n",
+ " seconds_in_day = 24*3600\n",
+ " settings_slope = {'slope_min': 0.01, # minimum slope to trial\n",
+ " 'slope_max': 0.2, # maximum slope to trial\n",
+ " 'delta_slope': 0.005, # slope increment\n",
+ " 'date_range': [1999,2020], # range of dates over which to perform the analysis\n",
+ " 'n_days': 8, # sampling period [days]\n",
+ " 'n0': 50, # parameter for Nyquist criterium in Lomb-Scargle transforms\n",
+ " 'freqs_cutoff': 1./(seconds_in_day*30), # 1 month frequency\n",
+ " 'delta_f': 100*1e-10, # deltaf for identifying peak tidal frequency band\n",
+ " 'prc_conf': 0.05, # percentage above minimum to define confidence bands in energy curve\n",
+ " }\n",
+ " settings_slope['date_range'] = [pytz.utc.localize(datetime(settings_slope['date_range'][0],5,1)),\n",
+ " pytz.utc.localize(datetime(settings_slope['date_range'][1],1,1))]\n",
+ " beach_slopes = SDS_slope.range_slopes(settings_slope['slope_min'], settings_slope['slope_max'], settings_slope['delta_slope'])\n",
+ "\n",
+ " t = np.array([_.timestamp() for _ in df.index]).astype('float64')\n",
+ " delta_t = np.diff(t)\n",
+ " fig, ax = plt.subplots(1,1,figsize=(12,3), tight_layout=True)\n",
+ " ax.grid(which='major', linestyle=':', color='0.5')\n",
+ " bins = np.arange(np.min(delta_t)/seconds_in_day, np.max(delta_t)/seconds_in_day+1,1)-0.5\n",
+ " ax.hist(delta_t/seconds_in_day, bins=bins, ec='k', width=1);\n",
+ " ax.set(xlabel='timestep [days]', ylabel='counts',\n",
+ " xticks=7*np.arange(0,20),\n",
+ " xlim=[0,50], title='Timestep distribution');\n",
+ "\n",
+ " # find tidal peak frequency (can choose 7 or 8 in this case)\n",
+ " settings_slope['n_days'] = 7\n",
+ " settings_slope['freqs_max'] = SDS_slope.find_tide_peak(df.index,tides.tide,settings_slope)\n",
+ " # estimate beach-face slopes along the transects\n",
+ " slope_est, cis = dict([]), dict([])\n",
+ " for key in df.keys():\n",
+ " # remove NaNs\n",
+ " idx_nan = np.isnan(df[key])\n",
+ " dates = [df.index[_] for _ in np.where(~idx_nan)[0]]\n",
+ " tide = tides.tide.to_numpy()[~idx_nan]\n",
+ " composite = df[key][~idx_nan]\n",
+ " # apply tidal correction\n",
+ " tsall = SDS_slope.tide_correct(composite,tide,beach_slopes)\n",
+ " title = 'Transect %s'%key\n",
+ " SDS_slope.plot_spectrum_all(dates,composite,tsall,settings_slope, title)\n",
+ " slope_est[key],cis[key] = SDS_slope.integrate_power_spectrum(dates,tsall,settings_slope)\n",
+ " print('Beach slope at transect %s: %.3f'%(key, slope_est[key]))\n",
+ " transects.beach_slope.update(slope_est)\n",
+ " transects.cil.update({k: v[0] for k,v in cis.items()})\n",
+ " transects.ciu.update({k: v[1] for k,v in cis.items()})\n",
+ " transects[transects.index.isin(slope_est.keys())]\n",
+ " transects.to_file(\"transects_extended.geojson\")"
]
}
],
diff --git a/update.sh b/update.sh
index 71e5553e..40cb83d2 100755
--- a/update.sh
+++ b/update.sh
@@ -1,7 +1,7 @@
#!/bin/bash
git pull
./batch_process.py || (echo "Batch process failed" && exit 1)
-jupyter nbconvert --to notebook --execute --inplace tidal_correction.ipynb linear_models.ipynb
+jupyter nbconvert --to notebook --execute --inplace tidal_correction.ipynb slope_estimation.ipynb linear_models.ipynb
git add .
git commit -am "auto update" --author="coastsat-bot "
git push