Merge pull request #393 from LSSTDESC/u/cwalter/input-templates

Make config template input-type independent.

.github/workflows/ci.yml

@@ -93,9 +93,8 @@ jobs:
                 export RUBIN_SIM_DATA_DIR=`pwd`/rubin_sim_data
                 export SIMS_SED_LIBRARY_DIR=`pwd`/rubin_sim_data/sims_sed_library
                 # Just check that they haven't gotten stale and produce errors.
-                cd config
+                cd examples
                 galsim flat.yaml image.counts_per_pixel=500
                 galsim flat_with_sed.yaml image.counts_per_pixel=5
-                galsim imsim-skycat.yaml image.nobjects=10
-                cd ../examples
-                galsim imsim-user.yaml image.nobjects=10
+                galsim imsim-user-instcat.yaml image.nobjects=10
+                galsim imsim-user-skycat.yaml image.nobjects=10

config/imsim-config-instcat.yaml

@@ -0,0 +1,376 @@
+
+# This config file configures the imSim modules with default behavior. This file
+# is designed to be used as a template and some values can or should be
+# overridden.
+#
+# At least the following value(s) should generally be overridden in order to use
+# this file:
+#
+# input.instance_catalog.filename
+#
+
+
+# This tells GalSim which things to import to register extra config types
+# Mostly we need the imsim repo.  But could add others here as well.
+# E.g. galsim_extra has some useful routines we might want to use.  (But I don't here.)
+modules:
+    - imsim
+    - astropy  # I don't really understand why this is required.  But if only have astropy.time
+               # then I get a NameError when trying to parse astropy.time.Time.
+    - astropy.time  # Need this for an Eval.  Tell GalSim to have it loaded.
+
+# Anything set here can be used in any Eval (usually a string that starts with $) as a variable.
+# This is often convenient, so they can be set just once (probably from the command line) and
+# used in multiple places.
+eval_variables:
+    # The first letters of these variables indicates the type.
+    # c = CelestialCoord
+    # a = Angle
+    # s = string
+    # f = float
+    # So to use one of these in an Eval string, you would write just boresight,
+    # not cboresight.
+    cboresight:
+        type: RADec
+        ra:
+            type: Degrees
+            theta: { type: OpsimData, field: fieldRA }
+        dec:
+            type: Degrees
+            theta: { type: OpsimData, field: fieldDec }
+
+    aazimuth:
+        type: Degrees
+        theta: { type: OpsimData, field: azimuth }
+    aaltitude:
+        type: Degrees
+        theta: { type: OpsimData, field: altitude }
+    arotTelPos:
+        type: Degrees
+        theta: { type: OpsimData, field: rotTelPos }
+
+    sband: { type: OpsimData, field: band }
+
+    fexptime: { type: OpsimData, field: exptime }
+
+
+# Any input data is set here.  These are read in at the start of the program and
+# potentially updated for each output file. Also includes things that need some
+# set up at the start of an exposure, like the atmospheric PSF.
+input:
+
+    instance_catalog:
+        # This should be overridden below or on the command line.
+        file_name: default_catalog_file.txt
+        sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR')
+
+    opsim_data:
+        # Read the visit meta data by default we use the same file as the input.
+        # However you could specify a opsim file or a another instance catalog
+        # instead.
+        file_name: '@input.instance_catalog.file_name'
+
+
+    telescope:
+        file_name:
+            type: FormattedStr
+            format : LSST_%s.yaml
+            items:
+                - { type: OpsimData, field: band }
+        rotTelPos: $rotTelPos
+
+    sky_model:
+        # Use the rubin_sim.skybrightness model to compute the sky
+        # background level.
+        exptime: $exptime
+        mjd: { type: OpsimData, field: mjd }
+
+    atm_psf:
+        # This enables the AtmosphericPSF type for the PSF
+
+        # The first 4 items are required.
+        airmass: { type: OpsimData, field: airmass }
+        rawSeeing:  { type: OpsimData, field: rawSeeing }
+        band:  { type: OpsimData, field: band }
+        boresight: "$boresight"
+
+        # Optional parameters:  (Unless otherwise stated, these are the default values.)
+        t0: 0               # seconds
+        exptime: $exptime
+        kcrit: 0.2          # in units of 1/r0
+        screen_size: 409.6  # Default=812.2, which takes a lot of memory, so use this for testing.
+        screen_scale: 0.1   # meters
+        doOpt: False
+        nproc: 1            # Default (None) means one proc per screen.
+
+        # TODO:
+        #save_file: {}        # This is currently set of the imsim command line, so we could keep
+                              # doing that, but we could also base this name off of
+                              # input.instance_catalog.file_name, or even just have `save: True`
+                              # and let the atm_psf figure out a unique name from the input params.
+
+    tree_rings:
+        # This enables TreeRingCenter and TreeRungFunc, which are stored as a dict here based
+        # on the detector name, so the right value can be accessed for each object.
+        # This file lives in the imsim repo.
+        file_name: "tree_ring_parameters_2018-04-26.txt"
+        # Can limit the detectors to read in.  This is handy while debugging, since it takes
+        # half a minute or so to read all 189 detectors (the default behavior).
+        # only_dets: [R22_S11]
+
+    checkpoint:
+        dir: checkpoint
+        file_name:
+          type: FormattedStr
+          format : checkpoint_%08d-%s.hdf
+          items:
+              - { type: OpsimData, field: observationId }
+              - "$det_name"
+
+    vignetting:
+        file_name: LSSTCam_vignetting_data.json
+
+# Define how the objects are placed on the image.  This refers to a single CCD.
+image:
+    type: LSST_Image
+
+    random_seed: { type: OpsimData, field: seed }
+
+    nproc: 1    # Can set this to -1 to use all CPUs if you want.  That probably won't be
+                # particularly efficient until we switch to using MakePhot rather than DrawImage.
+                # Right now, there is a significant overhead passing the drawn stamps back to
+                # the main process for all of the (many) vert faint sources, which only shoot
+                # a small numer of photons.
+
+    xsize: "$xsize"
+    ysize: "$ysize"
+
+    bandpass: { type: OpsimBandpass }
+
+    wcs:
+        type: Batoid
+
+        # These are required:
+        camera: "@output.camera"
+        boresight: "$boresight"
+
+        obstime:
+            type: Eval
+            str: "astropy.time.Time(mjd_val, format='mjd', scale='tai')"
+            fmjd_val: { type: OpsimData, field: mjd }
+
+        det_name: $det_name
+        wavelength: "$(@image.bandpass).effective_wavelength"
+
+        # The rest can be omitted, since these are the default values, but shown here
+        # for reference.
+        temperature: 280  # Kelvin
+        pressure: 72.7    # kPa
+        H2O_pressure: 1.0 # kPa
+        order: 3          # Order of the SIP polynomial
+
+    camera: "@output.camera"
+
+    noise:
+        type: CCD
+        gain: 1
+        read_noise: 0  # The read noise is applied later.  Only sky noise here.
+
+    sky_level: { type: SkyLevel }  # Computed from input.sky_model.
+
+    apply_sky_gradient: True
+
+    use_flux_sky_areas: False   # This means don't bother with the BFE from sky flux when computing
+                                # the pixel areas for drawing the sky.
+                                # This is the default.  But set to true to get BF in sky.
+                                # TODO: The True option is a one step flux calculation, so it's not
+                                # yet right for doing flats.  Aspirationally, I'd like it to be
+                                # possible to make flats simply by setting this to true and
+                                # cranking up the sky flux to something appropriate.
+
+    sensor:
+        type: Silicon
+        strength: 1.0
+        index_key: image_num    # This tells GalSim that this only changes with a new image num.
+        treering_center: { type: TreeRingCenter, det_name: $det_name }
+        treering_func: { type: TreeRingFunc, det_name: $det_name }
+
+
+# Define the PSF to use in the normal case (LSST_Silicon will override if doing a saturated star)
+psf:
+    type: Convolve
+    items:
+        -
+            # Note: This type requires the "input" type atm_psf.
+            #       It's not really an input in the normal sense, since it doesn't read
+            #       in a file.  But it needs to do some setup at the start, which all
+            #       objects will use.  So functionally, this works like an input type.
+            #       All of the relevant configurable parameters are set there.
+            type: AtmosphericPSF
+        -
+            # An additional Gaussian profile to represent contributions of physical effects
+            # not otherwise explicitly modeled in either the Optical or Atmospheric parts.
+            # This value of 0.3 arcsec is appropriate when doOpt=True and sensor effects are
+            # being modeled. If this is not the case, then it may be appropriate to increase
+            # this value to account for the missing contribution of these effects.
+            type: Gaussian
+            fwhm: 0.3
+
+stamp:
+    type: LSST_Silicon
+
+    fft_sb_thresh: 2.e5   # When to switch to fft and a simpler PSF and skip silicon
+    max_flux_simple: 100  # When to switch to simple SED
+    airmass: { type: OpsimData, field: airmass }
+    rawSeeing:  { type: OpsimData, field: rawSeeing }
+    band:  { type: OpsimData, field: band }
+    camera: "@output.camera"
+    det_name: "$det_name"  # This is automatically defined by the LSST_CCD output type.
+
+    diffraction_psf:
+      exptime: { type: OpsimData, field: exptime }
+      azimuth:
+        type: Degrees
+        theta: { type: OpsimData, field: azimuth }
+      altitude:
+        type: Degrees
+        theta: { type: OpsimData, field: altitude }
+      rotTelPos:
+        type: Degrees
+        theta: { type: OpsimData, field: rotTelPos }
+
+    photon_ops:
+        -
+            type: TimeSampler
+            t0: 0.0
+            exptime: $exptime
+        -
+            type: PupilAnnulusSampler
+            R_outer: 4.18
+            R_inner: 2.55  # M1 inner diameter is 2.558, but we need a bit of slack for off-axis rays
+        -
+            type: PhotonDCR
+            base_wavelength: $bandpass.effective_wavelength
+            latitude: -30.24463 degrees
+            HA:
+                type: Degrees
+                theta: { type: OpsimData, field: HA }
+        -
+            type: RubinDiffractionOptics
+            boresight: "$boresight"
+            camera: "@output.camera"
+            altitude: $altitude
+            azimuth: $azimuth
+        -
+            # Note: If FocusDepth is before Refraction, then the depth is the amount of focus
+            #       change required relative to the rays coming to a focus at the surface.
+            #       If FocusDepth is after Refraction, then the depth is the actual depth in
+            #       the silicon where the (refracted) rays come to a focus.
+            type: FocusDepth
+            depth:
+                type: Eval
+                str: depth_dict[band]
+                # TODO: Figure out the depth to use for other bands.  Josh found -0.6 for y.
+                # These numbers are in units of pixels.
+                ddepth_dict: {'u':0, 'g':0, 'r':0, 'i':0, 'z':0, 'y':-0.6}
+                sband: { type: OpsimData, field: band }
+        -
+            type: Refraction
+            index_ratio: 3.9  # TODO: This is what Josh used for y band.
+                              # I assume it's wavelength dependent...
+                              # Probably need to use the same kind of pattern as above for depth.
+
+    world_pos:
+        type: InstCatWorldPos
+
+# This defines both the output files and some basic things about the overall exposure/fov.
+output:
+    type: LSST_CCD
+    nproc: 1    # Change this to work on multiple CCDs at once.
+    nfiles: 1   # Default is all 189 CCDs.  Set to 1 while testing.
+
+    camera: LsstCam
+
+    exptime: $exptime
+
+    cosmic_ray_rate: 0.2
+
+    det_num:
+        type: Sequence
+        nitems: 189
+        first: 94  # Can set first to something if you want to do a specific sensor.
+
+    dir: fits
+    file_name:
+        type: FormattedStr
+        format : eimage_%08d-%1d-%s-%s-det%03d.fits
+        items:
+            - { type: OpsimData, field: observationId }
+            - { type: OpsimData, field: snap }
+            - { type: OpsimData, field: band }
+            - "$det_name"   # A value stored in the dict by LSST_CCD
+            - "@output.det_num"
+
+    readout:
+        # Convert from e-image to realized amp images
+        readout_time: 3.
+        dark_current: 0.02
+        bias_level: 1000.
+        pcti: 1.e-6
+        scti: 1.e-6
+
+        file_name:
+            type: FormattedStr
+            format : amp_%08d-%1d-%s-%s-det%03d.fits.fz
+            items:
+                - { type: OpsimData, field: observationId }
+                - { type: OpsimData, field: snap }
+                - { type: OpsimData, field: band }
+                - "$det_name"
+                - "@output.det_num"
+
+    truth:
+        dir: output
+        file_name:
+            type: FormattedStr
+            format : centroid_%08d-%1d-%s-%s-det%03d.txt.gz
+            items:
+                - { type: OpsimData, field: observationId }
+                - { type: OpsimData, field: snap }
+                - { type: OpsimData, field: band }
+                - "$det_name"
+                - "@output.det_num"
+        columns:
+            object_id: "@object_id"
+            ra: "$sky_pos.ra.deg"
+            dec: "$sky_pos.dec.deg"
+            x: "$image_pos.x"
+            y: "$image_pos.y"
+            # TODO: ... more probably
+
+gal:
+    type: InstCatObj
+
+################################################################
+#  Above is template material.
+#
+#  The following entires are added  to the base configuration above.
+################################################################
+
+# input.instance_catalog:
+#     # This should be overridden below or on the command line.
+#     file_name: default_catalog_file.txt
+#     sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR')
+
+# input.opsim_data:
+#     # Read the visit meta data by default we use the same file as the input.
+#     # However you could specify a opsim file or a another instance catalog
+#     # instead.
+#     file_name: '@input.instance_catalog.file_name'
+
+# Define the galaxy type and positions to use
+# gal:
+#     type: InstCatObj
+
+# stamp.world_pos:
+#     type: InstCatWorldPos