Update database structure

db_json_indexing.py

@@ -0,0 +1,35 @@
+# %% -*- encoding: utf-8 -*-
+'''
+@File    :   db_json_indexing.py
+@Time    :   2023/11/27
+@Author  :   Mingyu Li
+@Contact :   [email protected]
+'''
+
+# %%
+import h5py
+import pandas as pd
+from tqdm import tqdm
+import json
+
+indexing = []
+with h5py.File('db_filter.hdf5', 'r') as f:
+    for instrument in tqdm(f.keys()):
+
+        # sorting the bands by wavelength
+        bands = list(f[instrument].keys())
+        wavelength = [f[instrument][band].attrs['w_pivot'] for band in bands]
+        ddff = pd.DataFrame({"w":wavelength, "f":bands})
+        ddff.sort_values(by=['w'], inplace=True)
+        bands = ddff['f'].values
+
+        # indexing the bands
+        indexing.append({
+            "value":instrument,
+            "label":instrument,
+            "dialog":True,
+            "children":[{"value":f"{instrument}.{band}", "label":band, "checked":False} for band in bands]})
+
+# Save to json
+with open('indexing.json', 'w') as outfile:
+    json.dump(indexing, outfile)

myfilter_frontend/js/custom.js

@@ -91,7 +91,7 @@ const app = Vue.createApp({
             if (this.filter.length === 0) {
                 this.filter = this.filterInit
             }
-            this.filterLink = `https://preview.lmytime.com/getfilter?${this.filter.join('&')}`
+            this.filterLink = `/getfilter?${this.filter.join('&')}`
             this.g.updateOptions({
                 'file': this.filterLink
             });
@@ -223,7 +223,7 @@ const app = Vue.createApp({
 
         if (this.cus == 'JWST') {
             this.title = this.cus + " Filter";
-            this.cusComment = "This is " + this.cus + " customamized version.<br>Go to general version <a target='_blank' href='https://preview.lmytime.com/myfilter'>here</a>."
+            this.cusComment = "This is " + this.cus + " customamized version.<br>Go to general version <a target='_blank' href='/myfilter'>here</a>."
             this.filterInitDefault = ["JWST/NIRCam.F090W", "JWST/NIRCam.F115W", "JWST/NIRCam.F150W", "JWST/NIRCam.F200W",
                 "JWST/NIRCam.F277W", "JWST/NIRCam.F356W", "JWST/NIRCam.F444W"]
             this.zInitDefault = 6.0
@@ -293,7 +293,7 @@ const app = Vue.createApp({
         };
 
         this.g = new Dygraph(document.getElementById("dygraph"),
-            `https://preview.lmytime.com/getfilter?${this.filterInit.join('&')}`, {
+            `/getfilter?${this.filterInit.join('&')}`, {
             title: this.title,
             xlabel: 'Wavelength [Å]',
             ylabel: 'Transmittance',

run.py

@@ -1,41 +1,41 @@
 from scipy import interpolate
 import pandas as pd
 import numpy as np
-from flask import Flask, request, redirect, send_file, render_template, send_from_directory
-import requests, io
+from flask import Flask, request, send_file, render_template
+import io
 from flask_cors import CORS
+
 app = Flask(__name__, static_url_path='/myfilter', template_folder='./myfilter_frontend', static_folder='./myfilter_frontend')
 CORS(app)
 
 
-def combine_filter(urls):
-    filter = {band: pd.read_csv(urls[band], header=None, delim_whitespace=True, names=['w', 't'], comment='#').apply(
-        pd.to_numeric, errors='ignore') for band in urls}
+def combine_filter(bands, precision=2000):
+    bands_raw = {band: db_filter[band] for band in bands}
 
-    filter_interpolate = {}
+    bands_interpolate = {}
     _wavelength = []
-    for filname in filter:
-        w = filter[filname]['w']
+    for band_name, band_data in bands_raw.items():
+        w = band_data[:, 0]
+        t = band_data[:, 1]
         _wavelength = _wavelength + list(w)
-        t = filter[filname]['t']
-        f = interpolate.interp1d(w, t, kind='slinear',
+        inp_t = interpolate.interp1d(w, t, kind='slinear',
                                  bounds_error=False, fill_value=0)
-        filter_interpolate[filname] = f
+        bands_interpolate[band_name] = inp_t
+
+    # prepare final data
     wmin, wmax = min(_wavelength), max(_wavelength)
-    step = (wmax+wmin)/2000
-    final_t = {'w': np.arange(wmin, wmax+step, step=step)}
-    for filname in filter_interpolate:
-        fil_interpolate = filter_interpolate[filname]
-        final_t[filname] = fil_interpolate(final_t['w'])
-    final = pd.DataFrame(final_t)
-    return final.to_csv(index=None)
+    step = (wmax+wmin)/precision
+    final_wavelength = np.arange(wmin, wmax+step, step=step)
+    final_data = {band_name: band_interpolate(final_wavelength) for band_name, band_interpolate in bands_interpolate.items()}
+    final_dataframe = pd.DataFrame({'w': final_wavelength, **final_data})
+
+    return final_dataframe.to_csv(index=None)
 
 
 @app.route('/getfilter', methods=['GET'])
 def get_filter():
-    urls = {fil: "./filters/"+fil+'.dat' for fil in request.args}
+    file = combine_filter(request.args)
 
-    file = combine_filter(urls)
     proxy = io.StringIO(file)
     mem = io.BytesIO()
     mem.write(proxy.getvalue().encode())
@@ -49,5 +49,11 @@ def index():
 
 if __name__ == "__main__":
     from waitress import serve
+    import h5py
+
+    # Read the filter db
+    db_filter = h5py.File('db_filter.hdf5', 'r')
+
+    # Run the server
     # serve(app, host="0.0.0.0", port=3688)
     app.run(debug=True, port=9899)

store_db2hdf5.py

@@ -0,0 +1,34 @@
+# %% -*- encoding: utf-8 -*-
+'''
+@File    :   store_db2hdf5.py
+@Time    :   2023/11/27
+@Author  :   Mingyu Li
+@Contact :   [email protected]
+'''
+
+# %%
+import os
+import h5py
+import pandas as pd
+from glob import glob
+from tqdm import tqdm
+from pyphot import Filter
+
+# Get all instrument names
+instruments = sorted(os.listdir("filters"))
+
+# Write to hdf5
+with h5py.File("db_filter.hdf5", "w") as f:
+    for instrument in tqdm(instruments):
+        f.create_group(instrument)
+        files = glob("filters/"+instrument+"/*.dat")
+        for file in files:
+            band = file.split("/")[-1][:-4]
+            df = pd.read_csv(file, header=None, delim_whitespace=True, names=['wavelength', 'transmission'], comment='#').apply(pd.to_numeric, errors='ignore')
+            f[instrument].create_dataset(band, data=df)
+            fil = Filter(df['wavelength'], df['transmission'], name=band, dtype='photon', unit='Angstrom')
+            f[instrument][band].attrs['w_pivot'] = fil.lpivot.value
+            f[instrument][band].attrs['w_min'] = fil.lmin
+            f[instrument][band].attrs['w_max'] = fil.lmax
+            f[instrument][band].attrs['w_width'] = fil.width.value
+# %%