Adding BPOLY calibration option to CARACal

caracal/schema/crosscal_schema.yml

@@ -181,6 +181,31 @@ mapping:
             example: ""
             required: false 
             type: str
+          degamp:
+            desc: Polynomial degree for BPOLY amplitude solution
+            type: int
+            required: false
+            example: "3"
+          degphase:
+            desc: Polynomial degree for BPOLY phase solution
+            type: int
+            required: false
+            example: "3"
+          visnorm:
+            desc: Normalize data prior to BPOLY solution
+            type: bool
+            required: false
+            example: "False"
+          maskcenter:
+            desc: Number of channels in BPOLY to avoid in center of band
+            type: int
+            required: false
+            example: "0"
+          maskedge:
+            desc: Percent of channels in BPOLY to avoid at each band edge
+            type: int
+            required: false
+            example: "0"
           plotgains:
             desc: Plot gains with ragavi-gains. The .html plots are located in <output>/diagnostic_plots/crosscal/.
             type: bool

caracal/workers/crosscal_worker.py

@@ -80,6 +80,12 @@ def exists(outdir, path):
         "cab": "cab/casa_bandpass",
         "field": "bpcal",
     },
+    "P": {
+        "name": "bpoly_cal",
+        "interp": "linear",
+        "cab": "cab/casa_bandpass",
+        "field": "bpcal",
+    },
     "A": {
         "name": "auto_flagging",
         "cab": "cab/casa_flagdata",
@@ -184,6 +190,23 @@ def do_KGBF(i):
             params["fillgaps"] = config[ftype]["b_fillgaps"]
             params["uvrange"] = config["uvrange"]
             params["scan"] = config[ftype]["scanselection"]
+        elif term == "P":
+            # adding a symlink to caltables for bpoly reference and applycal
+            src_ = os.path.abspath(os.path.join(pipeline.msdir, msname))
+            dst_ = os.path.join(pipeline.caltables, msname)
+            if not os.path.exists(dst_):
+                os.symlink(src_, dst_)
+            # parameters
+            params["bandtype"] = 'BPOLY'
+            params["solnorm"] = config[ftype]["b_solnorm"]
+            params["fillgaps"] = config[ftype]["b_fillgaps"]
+            params["uvrange"] = config["uvrange"]
+            params["scan"] = config[ftype]["scanselection"]
+            params["degamp"] = config[ftype]["degamp"]
+            params["degphase"] = config[ftype]["degphase"]
+            params["visnorm"] = config[ftype]["visnorm"]
+            params["maskcenter"] = config[ftype]["maskcenter"]
+            params["maskedge"] = config[ftype]["maskedge"]
         elif term == "K":
             params["gaintype"] = term
             params["scan"] = config[ftype]["scanselection"]
@@ -246,7 +269,9 @@ def do_KGBF(i):
 
         # Assume gains were plotted when they were created
         if config[ftype]["plotgains"] and not can_reuse:
-            plotgains(recipe, pipeline, field_id if term != "F" else None, caltable, iobs, term=term)
+            # BPOLY gain plots break because of new table format, no gains plots for the moment
+            if term != "P":
+                plotgains(recipe, pipeline, field_id if term != "F" else None, caltable, iobs, term=term)
 
         fields.append(",".join(field))
         interps.append(interp)
@@ -327,7 +352,7 @@ def do_IA(i):
     # terms that need an apply
     groups_apply = list(filter(lambda g: g, re.findall("([AI]+)?", order)))
     # terms that need a solve
-    groups_solve = list(filter(lambda g: g, re.findall("([KGBF]+)?", order)))
+    groups_solve = list(filter(lambda g: g, re.findall("([KGBPF]+)?", order)))
     # Order has to start with solve group.
     # TODO(sphe) in the philosophy of giving user enough roap to hang themselves
     # Release II will allow both starting with I/A in case
@@ -460,7 +485,7 @@ def applycal(order, msname, recipe, gaintable, interp, gainfield, field, pipelin
     """
 
     gaintables, interps, fields = get_caltab_final(order, gaintable, interp,
-                                                   gainfield, field)
+            gainfield, field)
 
     step = "apply_gains-%s-%s-%d" % (field, label, i)
     recipe.add("cab/casa_applycal", step, {
@@ -595,7 +620,8 @@ def worker(pipeline, recipe, config):
                         "row-chunks": sdm.dismissable(config['set_model']["meerkat_crystalball_row_chunks"]),
                         "model-chunks": sdm.dismissable(config['set_model']["meerkat_crystalball_model_chunks"]),
                         "num-sources": sdm.dismissable(config['set_model']['meerkat_crystalball_num_sources']),
-                    }
+                   }
+
                 elif modelpoint:  # spectral model if specified in our standard
                     opts = {
                         "vis": msname,
@@ -655,7 +681,7 @@ def worker(pipeline, recipe, config):
                          "nearest", "xcal", pipeline, i, calmode=calmode, label=label)
             if "target" in config["apply_cal"]["applyto"]:
                 applycal(primary_order, msname, recipe, copy.deepcopy(gaintables), copy.deepcopy(interps),
-                         "nearest", "target", pipeline, i, calmode=calmode, label=label)
+                         gainfields, "target", pipeline, i, calmode=calmode, label=label)
         else:
             primary = solve(msname, msinfo, recipe, config, pipeline, i,
                             prefix_msbase, label=label, ftype="primary")
@@ -683,7 +709,7 @@ def worker(pipeline, recipe, config):
                          "nearest", "xcal", pipeline, i, calmode=calmode, label=label)
             if "target" in config["apply_cal"]["applyto"]:
                 applycal(secondary_order, msname, recipe, copy.deepcopy(gaintables), interps,
-                         "nearest", "target", pipeline, i, calmode=calmode, label=label)
+                         gainfields, "target", pipeline, i, calmode=calmode, label=label)
 
         if {"gcal", "fcal", "target"}.intersection(config["apply_cal"]["applyto"]):
             substep = 'save-{0:s}-ms{1:d}'.format(flags_after_worker, i)

docs/wiki/README.md

@@ -0,0 +1 @@
+Images for wiki pages

docs/wiki/run-3c39-32k-bpoly.yml

@@ -0,0 +1,117 @@
+schema_version: 1.0.4
+
+
+## Generate MS for calibration
+# Compulsory worker to set up data/input/output directories.
+# The prefix used for the output data products (e.g., diagnostic plots, images, etc.).
+general:
+  prefix: 3C39
+  title: 3C39
+  # Location where CARACal will write and expect to find .MS files
+  msdir: msdir-bpoly
+  # This directory and the input .MS files within it can be read-only
+  rawdatadir: ms-orig
+  # Location where CARACal writes output products
+  output: output-3c39-32k-bpoly
+
+# Compulsory worker to specify the input .MS files.
+getdata:
+  dataid: [1627186165_sdp_l0]
+  extension: ms
+
+# Compulsory worker to set up target/calibrators names.
+obsconf:
+  refant: auto
+  obsinfo:
+    enable: True
+    plotelev:
+      enable: True
+      plotter: plotms
+
+
+## Processing calibrators
+# Narrow band zoom modes channelise the 32768 channels
+# For calibration average down to the wideband 4096 channels
+# Factor 8 averaging if all channels are available
+transform__calibrators:
+  enable: true
+  label_out: cal
+  field: calibrators
+  split_field:
+    enable: true
+    chan_avg: 8
+    col: data
+
+# Prepare the calibrators-only .MS files for processing.
+prep__calibrators:
+  enable: true
+  label_in: cal
+  field: calibrators
+  clearcal: true
+  specweights:
+    enable: true
+    mode: uniform
+
+
+## Initial inspection and flagging
+# Make an inital amplitude vs. channel plot of the bandpass calibrator averaging all the data
+# together in time.
+# Next plot phase vs. channel.
+# Now plot amplitude and phase as a function of time.
+# 0a. Flag out band edges
+# 0b. Automated flagging for shadowing, elevation and extreme outliers
+# 0c. Flagging of identified bad data
+# Check that bad data is gone
+flag__calibrators:
+  enable: true
+  label_in: cal
+  field: calibrators
+  flag_autocorr:
+    enable: true
+  flag_shadow:
+    enable: true
+  flag_spw:
+    enable: true
+    # for narrow band only flag out the band edges
+    chans: '*:1630MHz~1657MHz,*:1673MHz~1700MHz'
+
+# Derive the cross-calibration tables and apply them to the calibrators.
+# Carry out Cross calibration of the data (delay, bandpass and gain calibration).
+crosscal:
+  enable: true
+  label_in: cal
+  label_cal: 1gc
+  uvrange: '>150'
+  set_model:
+    enable: true
+    meerkat_skymodel: true
+  primary:
+    reuse_existing_gains: false
+    order: KGPAKGP
+    combine: ["scan", "spw", "scan,spw", null, "", "", "scan,spw"]
+    solint: [inf, inf, inf, null, int, int, inf]
+    calmode: [ap, ap, ap, null, ap, ap, ap]
+    b_fillgaps: 70
+    # degamp: 2
+    degamp: 3
+    degphase: 3
+    visnorm: false
+    plotgains: false
+  # the default rime is 'KGB', so if you are using the 'P' option
+  # you need to specify it for all the calibrators, else the pipeline
+  # will attempt to run 'B' which will cause and error, because it does not exist
+  secondary:
+    reuse_existing_gains: false
+    order: KGAKF
+    apply: P
+    combine: ["scan", "spw", null, "", ""]
+    solint: [inf, inf, null, 60s, 60s]
+    calmode: [ap, ap, null, ap, ap]
+#     plotgains: true
+    plotgains: false
+  apply_cal:
+    applyto:
+      - fcal
+      - gcal
+
+# -fin-

docs/wiki/run-3c39-32k-smooth.yml

@@ -0,0 +1,110 @@
+schema_version: 1.0.4
+
+
+## Generate MS for calibration
+# Compulsory worker to set up data/input/output directories.
+# The prefix used for the output data products (e.g., diagnostic plots, images, etc.).
+general:
+  prefix: 3C39
+  title: 3C39
+  # Location where CARACal will write and expect to find .MS files
+  msdir: msdir-smooth
+  # This directory and the input .MS files within it can be read-only
+  rawdatadir: ms-orig
+  # Location where CARACal writes output products
+  output: output-3c39-32k-smooth
+
+# Compulsory worker to specify the input .MS files.
+getdata:
+  dataid: [1627186165_sdp_l0]
+  extension: ms
+
+# Compulsory worker to set up target/calibrators names.
+obsconf:
+  refant: auto
+  obsinfo:
+    enable: True
+    plotelev:
+      enable: True
+      plotter: plotms
+
+
+## Processing calibrators
+# Narrow band zoom modes channelise the 32768 channels
+# For calibration average down to the wideband 4096 channels
+# Factor 8 averaging if all channels are available
+transform__calibrators:
+  enable: true
+  label_out: cal
+  field: calibrators
+  split_field:
+    enable: true
+    chan_avg: 8
+    col: data
+
+# Prepare the calibrators-only .MS files for processing.
+prep__calibrators:
+  enable: true
+  label_in: cal
+  field: calibrators
+  clearcal: true
+  specweights:
+    enable: true
+    mode: uniform
+
+
+## Initial inspection and flagging
+# Make an inital amplitude vs. channel plot of the bandpass calibrator averaging all the data
+# together in time.
+# Next plot phase vs. channel.
+# Now plot amplitude and phase as a function of time.
+# 0a. Flag out band edges
+# 0b. Automated flagging for shadowing, elevation and extreme outliers
+# 0c. Flagging of identified bad data
+# Check that bad data is gone
+flag__calibrators:
+  enable: true
+  label_in: cal
+  field: calibrators
+  flag_autocorr:
+    enable: true
+  flag_shadow:
+    enable: true
+  flag_spw:
+    enable: true
+    # for narrow band only flag out the band edges
+    chans: '*:1630MHz~1657MHz,*:1673MHz~1700MHz'
+
+# Derive the cross-calibration tables and apply them to the calibrators.
+# Carry out Cross calibration of the data (delay, bandpass and gain calibration).
+crosscal:
+  enable: true
+  label_in: cal
+  label_cal: 1gc
+  uvrange: '>150'
+  set_model:
+    enable: true
+    meerkat_skymodel: true
+  primary:
+    reuse_existing_gains: true
+    order: KGBAKGB
+    combine: ["scan", "spw", "scan,spw", null, "", "", "scan,spw"]
+    solint: [inf, inf, inf, null, int, int, inf]
+    calmode: [ap, ap, ap, null, ap, ap, ap]
+    b_fillgaps: 70
+    b_smoothwindow: 32
+    plotgains: true
+  secondary:
+    reuse_existing_gains: true
+    order: KGAKF
+    apply: B
+    combine: ["scan", "spw", null, "", ""]
+    solint: [inf, inf, null, 60s, 60s]
+    calmode: [ap, ap, null, ap, ap]
+    plotgains: true
+  apply_cal:
+    applyto:
+      - fcal
+      - gcal
+
+# -fin-