Skip to content

Commit

Permalink
add lincSun
Browse files Browse the repository at this point in the history
  • Loading branch information
@peijin94
peijin94 committed Dec 4, 2023
1 parent acb87be commit beb4538
Show file tree
Hide file tree
Showing 6 changed files with 348 additions and 6 deletions.
49 changes: 49 additions & 0 deletions paper.bib
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,52 @@
@article{offringa2014wsclean,
title = {WSCLEAN: an implementation of a fast, generic wide-field imager for radio astronomy},
author = {Offringa, AR and McKinley, Benjamin and Hurley-Walker, Natasha and Briggs, FH and Wayth, RB and Kaplan, DL and Bell, ME and Feng, Lu and Neben, AR and Hughes, JD and others},
journal = {Monthly Notices of the Royal Astronomical Society},
volume = {444},
number = {1},
pages = {606-619},
year = {2014},
publisher = {The Royal Astronomical Society}
}

@ARTICLE{GasperinAteam:2020,
author = {{de Gasperin}, F. and {Vink}, J. and {McKean}, J.~P. and {Asgekar}, A. and {Avruch}, I. and {Bentum}, M.~J. and {Blaauw}, R. and {Bonafede}, A. and {Broderick}, J.~W. and {Br{\"u}ggen}, M. and {Breitling}, F. and {Brouw}, W.~N. and {Butcher}, H.~R. and {Ciardi}, B. and {Cuciti}, V. and {de Vos}, M. and {Duscha}, S. and {Eisl{\"o}ffel}, J. and {Engels}, D. and {Fallows}, R.~A. and {Franzen}, T.~M.~O. and {Garrett}, M.~A. and {Gunst}, A.~W. and {H{\"o}randel}, J. and {Heald}, G. and {Hoeft}, M. and {Iacobelli}, M. and {Koopmans}, L.~V.~E. and {Krankowski}, A. and {Maat}, P. and {Mann}, G. and {Mevius}, M. and {Miley}, G. and {Morganti}, R. and {Nelles}, A. and {Norden}, M.~J. and {Offringa}, A.~R. and {Orr{\'u}}, E. and {Paas}, H. and {Pandey}, V.~N. and {Pandey-Pommier}, M. and {Pekal}, R. and {Pizzo}, R. and {Reich}, W. and {Rowlinson}, A. and {Rottgering}, H.~J.~A. and {Schwarz}, D.~J. and {Shulevski}, A. and {Smirnov}, O. and {Sobey}, C. and {Soida}, M. and {Steinmetz}, M. and {Tagger}, M. and {Toribio}, M.~C. and {van Ardenne}, A. and {van der Horst}, A.~J. and {van Haarlem}, M.~P. and {van Weeren}, R.~J. and {Vocks}, C. and {Wucknitz}, O. and {Zarka}, P. and {Zucca}, P.},
title = "{Cassiopeia A, Cygnus A, Taurus A, and Virgo A at ultra-low radio frequencies}",
journal = {\aap},
keywords = {radio continuum: general, techniques: interferometric, Astrophysics - High Energy Astrophysical Phenomena, Astrophysics - Astrophysics of Galaxies, Astrophysics - Instrumentation and Methods for Astrophysics},
year = 2020,
month = mar,
volume = {635},
eid = {A150},
pages = {A150},
doi = {10.1051/0004-6361/201936844},
archivePrefix = {arXiv},
eprint = {2002.10431},
primaryClass = {astro-ph.HE},
adsurl = {https://ui.adsabs.harvard.edu/abs/2020A&A...635A.150D},
adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}



@ARTICLE{GasperinCalib:2019,
author = {{de Gasperin}, F. and {Dijkema}, T.~J. and {Drabent}, A. and {Mevius}, M. and {Rafferty}, D. and {van Weeren}, R. and {Br{\"u}ggen}, M. and {Callingham}, J.~R. and {Emig}, K.~L. and {Heald}, G. and {Intema}, H.~T. and {Morabito}, L.~K. and {Offringa}, A.~R. and {Oonk}, R. and {Orr{\`u}}, E. and {R{\"o}ttgering}, H. and {Sabater}, J. and {Shimwell}, T. and {Shulevski}, A. and {Williams}, W.},
title = "{Systematic effects in LOFAR data: A unified calibration strategy}",
journal = {\aap},
keywords = {surveys, catalogs, radio continuum: general, techniques: interferometric, Astrophysics - Instrumentation and Methods for Astrophysics},
year = 2019,
month = feb,
volume = {622},
eid = {A5},
pages = {A5},
doi = {10.1051/0004-6361/201833867},
archivePrefix = {arXiv},
eprint = {1811.07954},
primaryClass = {astro-ph.IM},
adsurl = {https://ui.adsabs.harvard.edu/abs/2019A&A...622A...5D},
adsnote = {Provided by the SAO/NASA Astrophysics Data System}
}



@ARTICLE{vanHaarlem:2013,
Expand Down
22 changes: 19 additions & 3 deletions paper.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,19 +44,27 @@ The cutting edge performance of LOFAR brings benefit in multiple science field i
ASTRON has developed a set of software for LOFAR data reduction, while most of which are dedicated for astromical observations.
To exploite the resolution power of LOFAR for solar and spaceweather studies, we build this toolset `lofarSun` based on the original LOFAR data processing procedures.

# Modules
Documentation is available at [https://lofar-sun-tools.readthedocs.io/en/latest/](https://lofar-sun-tools.readthedocs.io/en/latest/)

# Data processing procedures

## Dynamic spectrum (beamformed data)

`lofarSun.BF`

The RFI flagging [@zhang:2023]
The RFI flagging uses a feature matching based method [@zhang:2023] available both on CPU and GPU

## Interferometric imaging

The preprocessing is based on the LINC pipeline [@GasperinCalib:2019]. Correcting the amplitude and phase of solar radio observation with the measurements of A-team sources reference the skymodel [@GasperinAteam:2019].
The observaiton of solar interferometry is calibrated with A team sources. During observation, there are simutanously two beams, one pointing at the target (Sun), one pointing at the calibrator.

After pre-processing, the 2DiFFT and de-convolution is done with `WSClean` [@offringa2014wsclean], then we have the imaging in astromocal coordinates.

Then we do post processing with `lofarSun` on the images.
In this package, the corresponding module is
`lofarSun.IM`

Uses the result of a pipeline based on LINC, and do the post processings

## Command-Line Interface tools

Expand All @@ -67,6 +75,14 @@ Some useful tools to do interactive inspections for the data processing

# Acknowledgements

LOFAR is the LOw Frequency ARray designed and constructed by ASTRON. It has observing, data processing, and data storage facilities in several countries, which are owned by various parties (each with their own funding sources), and are collectively operated by the ILT foundation under a joint scientific policy. The ILT resources have benefited from the following recent major funding sources: CNRS-INSU, Observatoire de Paris and
Universite d’Orleans, France; BMBF, MIWF-NRW, MPG, Germany; Science Foundation Ireland (SFI), Department of Business, Enterprise and Innovation
(DBEI), Ireland; NWO, The Netherlands; The Science and Technology Facilities
Council, UK; Ministry of Science and Higher Education, Poland; Istituto
Nazionale di Astrofisica (INAF). This research has made use of the University of
Hertfordshire high-performance computing facility (https://uhhpc.herts.ac.uk/) and the LOFAR-UK compute facility, located at the University of
Hertfordshire and supported by STFC (ST/P000096/1).

We acknowledge contributions from ASTRON and LOFAR SSWKSP for building LOFAR data processing pipeline and operating the observation.

# References
208 changes: 208 additions & 0 deletions utils/IM/LINC/applyLBA.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
{
"msin": [
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB009_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB010_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB011_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB012_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB013_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB014_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB015_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB016_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB017_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB019_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB020_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB021_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB022_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB023_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB024_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB025_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB026_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB027_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB028_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB029_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB030_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB031_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB032_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB033_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB034_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB035_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB036_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB037_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB038_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB039_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB040_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB041_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB042_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB043_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB044_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB045_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB046_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB047_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB048_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB049_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB050_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB051_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB052_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB053_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB054_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB055_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB056_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB057_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB058_uv.MS"
},
{
"class": "Directory",
"path": "/scratch/project_2008326/peijin/E20220523/MSprep/L861398_SAP000_SB059_uv.MS"
}
],
"solutions": {
"class": "File",
"path": "/scratch/project_2008326/peijin/E20220523/results/solutions.h5"
}
}
15 changes: 12 additions & 3 deletions utils/IM/LINC/lincSun/steps/find_skymodel.cwl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ outputs:
- id: skymodel_file
type: File
outputBinding:
glob: calibrator.skymodel
glob: 'calibrator.skymodel'
- id: target_source
type: string
outputBinding:
Expand All @@ -33,6 +33,8 @@ baseCommand:
hints:
DockerRequirement:
dockerPull: astronrd/linc
NetworkAccess:
networkAccess: false

stdout: target_source
requirements:
Expand All @@ -50,12 +52,19 @@ requirements:
'Tau': 'TauAGG',
'Cyg': 'CygAGG'
} # LBA
CAL_MAP = {
'CasA': 'CasA_4_patch',
'VirA': 'VirA_4_patch',
'Tau': 'TauAGG',
'Cyg': 'CygAGG'
} # LBA
CAL_MAP = {
'CasA': 'CasA',
'VirA': 'VirA',
'Tau': 'TauA',
'Cyg': 'CygA'
} # LBA
os.system('cp '+sys.argv[2]+' ./calibrator.skymodel')
} # HBA

os.system('cp '+sys.argv[2]+' $PWD/calibrator.skymodel')
ms_in = table(sys.argv[1].split(",")[0] + '/POINTING', ack=False)
print(CAL_MAP[ms_in[0]['NAME']])
30 changes: 30 additions & 0 deletions utils/IM/LINC/linc_run_appl.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=20
#SBATCH -J LapplPJ
#SBATCH --account=project_2008326
#SBATCH --mem=128G
#SBATCH --partition=small
#SBATCH --time=23:59:00
#SBATCH [email protected]

#source /home/pjer1316/conda_start.sh
#source /projappl/project_2008326/linc_proj/bin/activate

mytmpdir=/dev/shm/run_hba_calibrator/
export SINGULARITY_TMPDIR=$mytmpdir
export TMPDIR=$mytmpdir
export TEMP=$mytmpdir
export TMP=$mytmpdir

alias python=python3

singularity exec -B /scratch/project_2008326/ -B $PWD \
/scratch/project_2008326/linc_latest.sif \
cwltool --no-container --verbose \
--basedir /scratch/project_2008326/ \
--outdir /scratch/project_2008326/peijin/E20220523/results/ \
--log-dir /scratch/project_2008326/peijin/E20220523/logs/ \
--tmpdir-prefix $mytmpdir \
/scratch/project_2008326/peijin/lincSun/workflow/apply_cal.cwl \
/scratch/project_2008326/peijin/E20220523/applyLBA.json
Loading

0 comments on commit beb4538

Please sign in to comment.