Merge branch 'main' into 28-locit-solver-for-antenna-positions

.readthedocs.yaml

@@ -10,7 +10,11 @@ build:
   tools:
     python: "3.7"
 
+python:
+  install:
+    - requirements: docs/sphinx.txt
+
 # Configuration for Sphinx documentation
 sphinx:
    configuration: docs/conf.py
-   fail_on_warning: true
+   fail_on_warning: False

docs/conf.py

@@ -68,7 +68,7 @@
 autoapi_options = ['show-module-summary']
 autoapi_template_dir = '_templates'
 autoapi_keep_files = True
-autoapi_ignore = ['*/_panel_classes/*','*/_utils/*','*/visualization/*','*/__pycache__/*']
+autoapi_ignore = ['*/_panel_classes/*','*/_utils/*','*/visualization/*','*/__pycache__/*','*/data/_*']
 #autoapi_ignore = ['*/_panel_classes/*','*/_utils/*','*/docs/*','*/visualization/*','*/__pycache__/*']
 
 # Napoleon settings
@@ -137,7 +137,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+#html_static_path = ['_static']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.

docs/tutorial_vla.ipynb

@@ -94,7 +94,7 @@
     }
    ],
    "source": [
-    "astrohack.gdown_utils.download('ea25_cal_small_after_fixed.split.ms', folder='data', unpack=True)"
+    "astrohack.data.datasets.download('ea25_cal_small_after_fixed.split.ms', folder='data', unpack=True)"
    ]
   },
   {
@@ -3923,7 +3923,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.15"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,

docs/visualization_tutorial.ipynb

@@ -137,12 +137,11 @@
     }
    ],
    "source": [
-    "from astrohack import gdown_utils\n",
     "from astrohack.astrohack_client import astrohack_local_client\n",
     "\n",
     "ms_name = 'data/ea25_cal_small_after_fixed.split.ms'\n",
     "\n",
-    "gdown_utils.download('ea25_cal_small_before_fixed.split.ms', folder='data', unpack=True)\n",
+    "astrohack.data.datasets.download('ea25_cal_small_before_fixed.split.ms', folder='data')\n",
     "\n",
     "client = astrohack_local_client(cores=2, memory_limit='8GB')"
    ]
@@ -326,13 +325,15 @@
    },
    "outputs": [],
    "source": [
-    "holog_mds.plot_diagnostics(holog_exports,         # Directory to contain the plots\n",
-    "                           ant_id='ea25',         # Plotting antenna ea25\n",
-    "                           ddi=0,                 # Plotting DDI 0\n",
-    "                           map_id=None,           # Plotting all mappings\n",
-    "                           complex_split='polar', # Split complex data to polar (amplitude + phase)\n",
-    "                           display=True,          # Display results in Notebook cell\n",
-    "                           parallel=False)        # Don't plots in parallel"
+    "holog_mds.plot_diagnostics(\n",
+    "    holog_exports,         # Directory to contain the plots\n",
+    "    ant_id='ea25',         # Plotting antenna ea25\n",
+    "    ddi=0,                 # Plotting DDI 0\n",
+    "    map_id=None,           # Plotting all mappings\n",
+    "    complex_split='polar', # Split complex data to polar (amplitude + phase)\n",
+    "    display=True,          # Display results in Notebook cell\n",
+    "    parallel=False         # Don't plots in parallel\n",
+    ")        "
    ]
   },
   {
@@ -352,13 +353,16 @@
    },
    "outputs": [],
    "source": [
-    "holog_mds.plot_diagnostics(holog_exports,             # Directory to contain the plots\n",
-    "                           ant_id=None,               # Plotting all antennae\n",
-    "                           ddi=None,                  # Plotting all DDIs\n",
-    "                           map_id=None,               # Plotting all mappings\n",
-    "                           complex_split='cartesian', # Split comples data to cartesian (real + imaginary)\n",
-    "                           display=False,             # don't display results in Notebook cell\n",
-    "                           parallel=True)             # Do plots in parallel\n",
+    "holog_mds.plot_diagnostics(\n",
+    "    holog_exports,             # Directory to contain the plots\n",
+    "    ant_id=None,               # Plotting all antennae\n",
+    "    ddi=None,                  # Plotting all DDIs\n",
+    "    map_id=None,               # Plotting all mappings\n",
+    "    complex_split='cartesian', # Split comples data to cartesian (real + imaginary)\n",
+    "    display=False,             # don't display results in Notebook cell\n",
+    "    parallel=True              # Do plots in parallel\n",
+    ")             \n",
+    "\n",
     "# All exported files\n",
     "print()\n",
     "print('Created files:')\n",
@@ -402,13 +406,14 @@
     "if os.path.exists(image_file):\n",
     "    image_mds = open_image(image_file)\n",
     "else:\n",
-    "    image_mds = holog(holog_name=holog_file,\n",
-    "                      overwrite=True,\n",
-    "                      phase_fit=True,\n",
-    "                      apply_mask=True,\n",
-    "                      to_stokes=True,\n",
-    "                      parallel=True\n",
-    "                     )\n"
+    "    image_mds = holog(\n",
+    "        holog_name=holog_file,\n",
+    "        overwrite=True,\n",
+    "        phase_fit=True,\n",
+    "        apply_mask=True,\n",
+    "        to_stokes=True,\n",
+    "        parallel=True\n",
+    ")\n"
    ]
   },
   {
@@ -798,7 +803,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.15"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,

src/astrohack/__init__.py

@@ -10,4 +10,4 @@
 from .panel import *
 from .combine import *
 from .mds import *
-from .gdown_utils import *
+from .data import *

src/astrohack/_utils/_holog.py

@@ -191,8 +191,6 @@ def _holog_chunk(holog_chunk_params):
     else:
         radius = radius_u
 
-    logger.debug(radius)
-
     if holog_chunk_params['apply_mask']:
     # Masking Aperture image
         mask = _mask_circular_disk(

src/astrohack/data/__init__.py

@@ -0,0 +1,3 @@
+from .datasets import *
+from ._dropbox import *
+from ._google_drive import *

src/astrohack/data/_dropbox.py

@@ -0,0 +1,83 @@
+import os
+import shutil
+import requests
+import zipfile
+
+from tqdm import tqdm
+from astrohack._utils._logger._astrohack_logger import _get_astrohack_logger
+
+FILE_ID = {
+  'ea25_cal_small_before_fixed.split.ms':
+  {
+    'file':'ea25_cal_small_before_fixed.split.ms.zip',
+    'id':'m2qnd2w6g9fhdxyzi6h7f',
+    'rlkey':'d4dgztykxpnnqrei7jhb1cu7m'
+  },
+  'ea25_cal_small_after_fixed.split.ms':
+  {
+    'file':'ea25_cal_small_after_fixed.split.ms.zip',
+    'id':'o3tl05e3qa440s4rk5owf',
+    'rlkey':'hoxte3zzeqgkju2ywnif2t7ko'
+  },
+  'J1924-2914.ms.calibrated.split.SPW3':
+  {
+    'file':'J1924-2914.ms.calibrated.split.SPW3.zip',
+    'id':'kyrwc5y6u7lxbmqw7fveh',
+    'rlkey':'r23qakcm24bid2x2cojsd96gs'
+  },
+  'extract_holog_verification.json':
+  {
+    'file': 'extract_holog_verification.json',
+    'id':'6pzucjd48a4n0eb74wys9',
+    'rlkey':'azuynw358zxvse9i225sbl59s'
+  },
+  'holog_numerical_verification.json':
+  {
+    'file':'holog_numerical_verification.json',
+    'id':'x69700pznt7uktwprdqpk',
+    'rlkey':'bxn9me7dgnxrtzvvay7xgicmi'
+  },
+
+}
+
+def download(file, folder='.'):
+  logger = _get_astrohack_logger()
+
+  if os.path.exists('/'.join((folder, file))):
+    logger.info("File exists.")
+    return
+
+  if file not in FILE_ID.keys():
+    logger.info("Requested file not found")
+
+    return 
+
+  fullname=FILE_ID[file]['file']
+  id=FILE_ID[file]['id']
+  rlkey=FILE_ID[file]['rlkey']
+
+  url = 'https://www.dropbox.com/scl/fi/{id}/{file}?rlkey={rlkey}'.format(id=id, file=fullname, rlkey=rlkey)
+
+  headers = {'user-agent': 'Wget/1.16 (linux-gnu)'}
+
+  r = requests.get(url, stream=True, headers=headers)
+  total = int(r.headers.get('content-length', 0))
+
+  fullname = '/'.join((folder, fullname))
+
+  with open(fullname, 'wb') as fd, tqdm(
+    desc=fullname,
+    total=total,
+    unit='iB',
+    unit_scale=True,
+    unit_divisor=1024) as bar:
+      for chunk in r.iter_content(chunk_size=1024):
+        if chunk:
+          size=fd.write(chunk)
+          bar.update(size)
+
+  if zipfile.is_zipfile(fullname):                
+    shutil.unpack_archive(filename=fullname, extract_dir=folder)
+
+    # Let's clean up after ourselves
+    os.remove(fullname)

src/astrohack/gdown_utils.py → src/astrohack/data/_google_drive.py

@@ -5,10 +5,11 @@
 import json
 
 from astrohack._utils._tools import _remove_suffix
+from astrohack._utils._logger._astrohack_logger import _get_astrohack_logger
 
 from prettytable import PrettyTable
 
-gdown_ids = {
+FILE_ID = {
     'ea25_cal_small_before_fixed.split.ms':'1oydlR7kA7F4n0i9KF9HgRc2jq1ziUslt',
     'ea25_cal_small_after_fixed.split.ms':'1TATMxKTFYIEO-l9L3jdYj62lZ8TZex4T',
     'J1924-2914.ms.calibrated.split.SPW3': '1OSDjWM1IskPOlC0w1wVBqsTp8JAbNGzL',
@@ -46,29 +47,31 @@ def build_folder_structure(dataname, resultname):
 def download(file, folder='.', unpack=False):
     """ Allows access to stakeholder and unit testing data and configuration files via gdown.
 
-    :param file: File to download for gdirve storage. A list of the available measurement sets can be accessed via `astrohack.gdown_utils.list_datasets()`. 
+    :param file: File to download for gdirve storage. A list of the available measurement sets can be accessed via `astrohack.datasets.list_datasets()`. 
     :type file: str
     :param folder: Destination folder if not the current directory, defaults to '.'
     :type folder: str, optional
     :param unpack: Unzip file, defaults to False
     :type unpack: bool, optional
     """
+    logger = _get_astrohack_logger()
+    logger.warning("The google-drive download option is deprecated adn will be removed soon!! Please use the dropbox option in download.")
 
-    if file == 'vla-test': 
-        matched = [(key, value) for key, value in gdown_ids.items() if re.search(r"^vla.+(before|after).split.+(holog|image|panel|point).*zarr$", key)]
+    if file == 'vla-test':
+        matched = [(key, value) for key, value in FILE_ID.items() if re.search(r"^vla.+(before|after).split.+(holog|image|panel|point).*zarr$", key)]
         files = files = list(dict(matched).keys())
 
     elif file == 'alma-test':
-        matched = [(key, value) for key, value in gdown_ids.items() if re.search(r"^alma.split.+(holog|image|panel|point).*zarr$", key)]
+        matched = [(key, value) for key, value in FILE_ID.items() if re.search(r"^alma.split.+(holog|image|panel|point).*zarr$", key)]
         files = list(dict(matched).keys())
 
     else:
         files = [file]
 
     for file in files:
-        assert file in gdown_ids, "File {file} not available. Available files are:".format(file=file) + str(gdown_ids.keys())
+        assert file in FILE_ID, "File {file} not available. Available files are:".format(file=file) + str(FILE_ID.keys())
 
-        id = gdown_ids[file]
+        id = FILE_ID[file]
         create_folder(folder)
 
         fullname = os.path.join(folder, file)
@@ -91,9 +94,9 @@ def download(file, folder='.', unpack=False):
 
 # DEPRECATED
 def gdown_data(ms_name, download_folder='.'):
-    assert ms_name in gdown_ids, "Measurement set not available. Available measurement sets are:" + str(gdown_ids.keys())
+    assert ms_name in FILE_ID, "Measurement set not available. Available measurement sets are:" + str(FILE_ID.keys())
 
-    id = gdown_ids[ms_name]
+    id = FILE_ID[ms_name]
     create_folder(download_folder)
     check_download(ms_name, download_folder, id)
 
@@ -102,7 +105,7 @@ def list_datasets():
     table.field_names = ["Measurement Table", "Description"]
     table.align = "l"
 
-    for key, _ in gdown_ids.items():
+    for key, _ in FILE_ID.items():
         basename = key.split('.')[0]
         file = ''.join((basename, '.json'))
         path = os.path.dirname(__file__)       

src/astrohack/data/datasets.py

@@ -0,0 +1,20 @@
+import os
+import astrohack
+
+from astrohack._utils._logger._astrohack_logger import _get_astrohack_logger
+
+def download(file, folder='.', unpack=False, source='dropbox'):
+  logger = _get_astrohack_logger()
+
+  if os.path.exists(folder) == False:
+    os.mkdir(folder)
+
+  if source == 'gdrive':
+    astrohack.data._google_drive.download(file=file, folder=folder, unpack=unpack)
+
+  elif source == 'dropbox':
+    astrohack.data._dropbox.download(file=file, folder=folder)
+
+  else:
+    logger.error("unknown source or issue found")
+    pass

stakeholder_test/alma_stakeholder_test.ipynb

@@ -23,15 +23,11 @@
    "source": [
     "import os\n",
     "import json\n",
-    "import gdown\n",
     "import shutil\n",
     "import astrohack\n",
     "\n",
     "import matplotlib.pyplot as plt\n",
     "\n",
-    "from astrohack.gdown_utils import gdown_data\n",
-    "from astrohack.gdown_utils import build_folder_structure\n",
-    "\n",
     "from astrohack.dio import open_pointing\n",
     "\n",
     "plt.rcParams['figure.figsize'] = [20, 20]"
@@ -46,17 +42,20 @@
    },
    "outputs": [],
    "source": [
-    "# Build folder structure & download \n",
+    "def build_folder_structure(paths):\n",
+    "    for path in paths:\n",
+    "        if not os.path.exists(path):\n",
+    "            os.makedirs(path) \n",
     "\n",
     "datafolder = 'data'\n",
     "resultsfolder = 'results'\n",
     "\n",
-    "build_folder_structure(datafolder, resultsfolder)\n",
-    "\n",
+    "# Build folder structure & download \n",
+    "build_folder_structure(['data', 'results'])\n",
     "\n",
-    "astrohack.gdown_utils.download('J1924-2914.ms.calibrated.split.SPW3', folder=datafolder, unpack=True)\n",
-    "astrohack.gdown_utils.download(file='extract_holog_verification.json')\n",
-    "astrohack.gdown_utils.download(file='holog_numerical_verification.json')\n"
+    "astrohack.data.datasets.download('J1924-2914.ms.calibrated.split.SPW3', folder=datafolder, unpack=True)\n",
+    "astrohack.data.datasets.download(file='extract_holog_verification.json')\n",
+    "astrohack.data.datasets.download(file='holog_numerical_verification.json')"
    ]
   },
   {
@@ -406,7 +405,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.17"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,