From 9b96db3406946817e70cda3ac7b743f3cd48210c Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Mon, 31 Jul 2023 15:35:12 -0400 Subject: [PATCH 01/24] Try using nested templates. --- config/imsim-config.yaml | 13 ------------- config/imsim-instcat.yaml | 26 ++++++++++++++++++++++++++ imsim/templates.py | 11 +++++++---- 3 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 config/imsim-instcat.yaml diff --git a/config/imsim-config.yaml b/config/imsim-config.yaml index 3ae87ade..5372fd8f 100644 --- a/config/imsim-config.yaml +++ b/config/imsim-config.yaml @@ -52,19 +52,6 @@ eval_variables: # updated for each output file. # Also includes things that need some set up at the start of an exposure, like the atmospheric PSF. input: - instance_catalog: - # This enables InstCat types - file_name: default_catalog_file.txt # This should be overridden in either the user config file or - # on the command line. See imsim-user.yaml for instructions. - sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') - - opsim_data: - # Read the visit meta data from the opsim db file. Note: We - # want some OpsimData items at file scope, in particular to - # define the WCS, which is needed by the InstCat items. - file_name: small_opsim.db - visit: 19500 - snap: 1 telescope: file_name: diff --git a/config/imsim-instcat.yaml b/config/imsim-instcat.yaml new file mode 100644 index 00000000..38972765 --- /dev/null +++ b/config/imsim-instcat.yaml @@ -0,0 +1,26 @@ +# This template can be used for reading the instance catalog format. + +# Use imSim custom modules +modules: + - imsim + +# Get most of the configuration from the imSim config-template +template: imsim-config + +# Set up the input keywords needed for instance catalogs. By default use the +# same file for the metadata information. + +input.instance_catalog: + # This should be overridden in either the user config file or + # on the command line. See imsim-user.yaml for instructions. + + file_name: default_catalog_file.txt + sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') + +input.opsim_data: + # Read the visit meta data by default we use the same file as the input. + # However you could specify a opsim file or a another instance catalog + # instead. + + file_name: "@instance_catalog.file_name" + diff --git a/imsim/templates.py b/imsim/templates.py index 93f4b994..7ac9dfa3 100644 --- a/imsim/templates.py +++ b/imsim/templates.py @@ -1,11 +1,14 @@ -# Register some configuration files with default desired imSim behavior with convenient names. -# These live in the imsim/config directory, which gets installed with imsim. -# Users can just use the given aliases rather than specifying the full location (which will -# typically be somewhere in the depths of a python site-packages directory). +# Register some configuration files with default desired imSim behavior with +# convenient names. These live in the imsim/config directory, which gets +# installed with imsim. Users can just use the given aliases rather than +# specifying the full location (which will typically be somewhere in the depths +# of a python site-packages directory). import os import galsim from .meta_data import config_dir galsim.config.RegisterTemplate('imsim-config', os.path.join(config_dir, 'imsim-config.yaml')) +galsim.config.RegisterTemplate('imsim-instcat', os.path.join(config_dir, 'imsim-instcat.yaml')) +galsim.config.RegisterTemplate('imsim-skycat', os.path.join(config_dir, 'imsim-skycat.yaml')) From 999444c9fdb95dedeae71668b022aa976c2a18f8 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 15:02:28 -0400 Subject: [PATCH 02/24] This doesn't actually work yet. Need to wait for GalSim v2.5. --- imsim/templates.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imsim/templates.py b/imsim/templates.py index 7ac9dfa3..83123665 100644 --- a/imsim/templates.py +++ b/imsim/templates.py @@ -10,5 +10,4 @@ from .meta_data import config_dir galsim.config.RegisterTemplate('imsim-config', os.path.join(config_dir, 'imsim-config.yaml')) -galsim.config.RegisterTemplate('imsim-instcat', os.path.join(config_dir, 'imsim-instcat.yaml')) -galsim.config.RegisterTemplate('imsim-skycat', os.path.join(config_dir, 'imsim-skycat.yaml')) + From d7138d69e82dbfe2beca273700224fb5da7983a4 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 15:19:10 -0400 Subject: [PATCH 03/24] Remove blank line. --- imsim/templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imsim/templates.py b/imsim/templates.py index 83123665..6adfe9ae 100644 --- a/imsim/templates.py +++ b/imsim/templates.py @@ -10,4 +10,3 @@ from .meta_data import config_dir galsim.config.RegisterTemplate('imsim-config', os.path.join(config_dir, 'imsim-config.yaml')) - From e3cc28e1248cd51a070d13c68694509e64a31556 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 15:37:43 -0400 Subject: [PATCH 04/24] Rename to be instcat specific --- examples/{imsim-user.yaml => imsim-user-instcat.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{imsim-user.yaml => imsim-user-instcat.yaml} (100%) diff --git a/examples/imsim-user.yaml b/examples/imsim-user-instcat.yaml similarity index 100% rename from examples/imsim-user.yaml rename to examples/imsim-user-instcat.yaml From a1db8b6cf05c0b1c22029a161af60cd5ccb48d67 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 15:59:41 -0400 Subject: [PATCH 05/24] Remove items specific to intCats --- config/imsim-config.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/config/imsim-config.yaml b/config/imsim-config.yaml index 5372fd8f..529237d3 100644 --- a/config/imsim-config.yaml +++ b/config/imsim-config.yaml @@ -198,11 +198,6 @@ psf: type: Gaussian fwhm: 0.3 -# Define the galaxy (or delta function) to use -gal: - type: InstCatObj - -# This has most of the code from GalsimSiliconInterpreter. stamp: type: LSST_Silicon @@ -226,9 +221,6 @@ stamp: type: Degrees theta: { type: OpsimData, field: rotTelPos } - world_pos: - type: InstCatWorldPos - photon_ops: - type: TimeSampler From bc200af5e9f32c0b09a400c36c79fe9181072759 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:02:56 -0400 Subject: [PATCH 06/24] Make template config files. These are templates that the user can copy and add to for each instance type. With GalSim 2.5 these will become templates. --- config/imsim-instcat.yaml | 27 +++++++++++++------- config/imsim-skycat.yaml | 52 +++++++++++++++++---------------------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/config/imsim-instcat.yaml b/config/imsim-instcat.yaml index 38972765..4b644103 100644 --- a/config/imsim-instcat.yaml +++ b/config/imsim-instcat.yaml @@ -1,4 +1,5 @@ -# This template can be used for reading the instance catalog format. +# This template file can be used for reading the instance catalog format. Make +# a copy and modify the area below the double ### line. # Use imSim custom modules modules: @@ -7,13 +8,8 @@ modules: # Get most of the configuration from the imSim config-template template: imsim-config -# Set up the input keywords needed for instance catalogs. By default use the -# same file for the metadata information. - input.instance_catalog: - # This should be overridden in either the user config file or - # on the command line. See imsim-user.yaml for instructions. - + # This should be overridden below or on the command line. file_name: default_catalog_file.txt sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') @@ -22,5 +18,18 @@ input.opsim_data: # However you could specify a opsim file or a another instance catalog # instead. - file_name: "@instance_catalog.file_name" - + file_name: '@input.instance_catalog.file_name' + +################################################################ +# Above is template material. +# +# Make changes below. Soon you will be able to include the above +# information via a single template command instead. +################################################################ +################################################################ + +# Put your own commands below here. For example +# input.instance_catalog.file_name: ./imsim_cat_197356.txt +# input.instance_catalog.sort_mag: False +# input.tree_rings.only_dets: [R22_S11] +# image.nobjects: 5 \ No newline at end of file diff --git a/config/imsim-skycat.yaml b/config/imsim-skycat.yaml index fc03f762..6bec77b6 100644 --- a/config/imsim-skycat.yaml +++ b/config/imsim-skycat.yaml @@ -1,44 +1,36 @@ # Use baseline configuration modules: - imsim -template: imsim-config -# Disable inputs from instance catalogs. -input.instance_catalog: "" +template: imsim-config # Use skyCatalogs for obtaining the objects to render. input.sky_catalog: - file_name: ../tests/data/sky_cat_9683.yaml - obj_types: [galaxy] # restrict to galaxies to avoid bright stars + file_name: default_sky_cat.yaml band: { type: OpsimData, field: band } mjd: { type: OpsimData, field: mjd } -input.opsim_data.file_name: ../tests/data/small_opsim_9683.db -input.opsim_data.visit: 449053 - -input.tree_rings.only_dets: [R22_S11, R22_S12] -input.checkpoint: "" - -# Disable the atmospheric PSF to run faster for testing. -input.atm_psf: "" - -image.nobjects: 1000 -image.random_seed: 42 - -psf: - type: Convolve - items: - - - type: Gaussian - fwhm: 0.8 - - type: Gaussian - fwhm: 0.3 +input.opsim_data.file_name: default_opsim.db -# Override the InstCat* defaults in the config/imsim-config.yaml baseline. +# Define the galaxy type and positions to use gal.type: SkyCatObj stamp.world_pos.type: SkyCatWorldPos -output.dir: output # default `fits` -output.det_num.first: 94 -output.nproc: 2 -output.nfiles: 2 +################################################################ +# Above is template material. +# +# Make changes below. Soon you will be able to include the above +# information via a single template command instead. +################################################################ +################################################################ + +# Put your own commands below here. For example + +# input.sky_catalog.file_name: ../tests/data/sky_cat_9683.yaml +# input.opsim_data.file_name: ../tests/data/small_opsim_9683.db +# input.opsim_data.visit: 449053 +# input.tree_rings.only_dets: [R22_S11, R22_S12] +# output.det_num.first: 94 +# output.nproc: 2 +# output.nfiles: 2 +# image.nobjects: 1000 From 2260384bde7f73e76eafa4b42f887f5f4c410f1d Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:04:35 -0400 Subject: [PATCH 07/24] Galaxy type piece somehow missed by git.. --- config/imsim-instcat.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/imsim-instcat.yaml b/config/imsim-instcat.yaml index 4b644103..80b280e3 100644 --- a/config/imsim-instcat.yaml +++ b/config/imsim-instcat.yaml @@ -20,6 +20,10 @@ input.opsim_data: file_name: '@input.instance_catalog.file_name' +# Define the galaxy type and positions to use +gal.type: InstCatObj +stamp.world_pos.type: InstCatWorldPos + ################################################################ # Above is template material. # From e90505144e6b67ff5999c201dddf71df82007331 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:06:14 -0400 Subject: [PATCH 08/24] Add examples using templates. Including adding a new skycatalog template. --- examples/imsim-user-instcat.yaml | 31 +++++++++++++----- examples/imsim-user-skycat.yaml | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 examples/imsim-user-skycat.yaml diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index ea78c321..ff0eb28f 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -2,19 +2,34 @@ modules: - imsim -# Get most of the configuration from the imSim named template +# Get most of the configuration from the imSim config-template template: imsim-config -input.instance_catalog.file_name: $os.environ.get('IMSIM_HOME')+'/imSim/examples/example_instance_catalog.txt' -input.instance_catalog.sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') +input.instance_catalog: + # This should be overridden below or on the command line. + file_name: default_catalog_file.txt + sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') -input.instance_catalog.sort_mag: False +input.opsim_data: + # Read the visit meta data by default we use the same file as the input. + # However you could specify a opsim file or a another instance catalog + # instead. -input.tree_rings.only_dets: [R22_S11] + file_name: '@input.instance_catalog.file_name' -input.opsim_data: - file_name: $os.environ.get('IMSIM_HOME')+'/imSim/examples/example_instance_catalog.txt' +################################################################ +# Above is template material. +# +# Make changes below. Soon you will be able to include the above +# information via a single template command instead. +################################################################ +################################################################ -#image.nobjects: 5 + +input.instance_catalog.file_name: $os.environ.get('IMSIM_HOME')+'/imSim/examples/example_instance_catalog.txt' + +input.instance_catalog.sort_mag: False +input.tree_rings.only_dets: [R22_S11] +image.nobjects: 10 output.dir: output diff --git a/examples/imsim-user-skycat.yaml b/examples/imsim-user-skycat.yaml new file mode 100644 index 00000000..6e9cea50 --- /dev/null +++ b/examples/imsim-user-skycat.yaml @@ -0,0 +1,55 @@ +# Use baseline configuration +modules: + - imsim + +template: imsim-config + +# Use skyCatalogs for obtaining the objects to render. +input.sky_catalog: + file_name: default_sky_cat.yaml + band: { type: OpsimData, field: band } + mjd: { type: OpsimData, field: mjd } + +input.opsim_data.file_name: default_opsim.db + +# Define the galaxy type and positions to use +gal.type: SkyCatObj +stamp.world_pos.type: SkyCatWorldPos + +################################################################ +# Above is template material. +# +# Make changes below. Soon you will be able to include the above +# information via a single template command instead. +################################################################ +################################################################ + +# Restrict to galaxies to avoid bright stars +input.sky_catalog.obj_types: [galaxy] + +# Disable the atmospheric PSF to run faster for testing. +input.atm_psf: "" +psf: + type: Convolve + items: + - + type: Gaussian + fwhm: 0.8 + - type: Gaussian + fwhm: 0.3 + +# Turn off check pointing for this example. +input.checkpoint: "" + +input.sky_catalog.file_name: ../tests/data/sky_cat_9683.yaml +input.opsim_data.file_name: ../tests/data/small_opsim_9683.db +input.opsim_data.visit: 449053 +input.tree_rings.only_dets: [R22_S11, R22_S12] +image.nobjects: 1000 + +output.dir: output # default `fits` +output.det_num.first: 94 +output.nproc: 2 +output.nfiles: 2 + +image.random_seed: 42 From 59976f2d6ea26de2d16c0269134ea6cb423727ab Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:07:32 -0400 Subject: [PATCH 09/24] Make the CI use examples. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41fcf8bb..ee5f7d3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,6 @@ jobs: cd config galsim flat.yaml image.counts_per_pixel=500 galsim flat_with_sed.yaml image.counts_per_pixel=5 - galsim imsim-skycat.yaml image.nobjects=10 cd ../examples - galsim imsim-user.yaml image.nobjects=10 + galsim imsim-user-instcat.yaml image.nobjects=10 + galsim imsim-user-skycat.yaml image.nobjects=10 From 7b960e3dfe6ccfc0f321c014f0563b1fdf72513f Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:12:02 -0400 Subject: [PATCH 10/24] Remove early text about DC2/previous imSim. Make clear what the file is for. --- config/imsim-config.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/config/imsim-config.yaml b/config/imsim-config.yaml index 529237d3..11901b08 100644 --- a/config/imsim-config.yaml +++ b/config/imsim-config.yaml @@ -1,7 +1,6 @@ -# This config file is intended to approximately reproduce what imSim does for e.g. DC2. -# It won't match in detail, since our random number treatment is different. -# And not all features are implemented yet, but the main functionality should be pretty close. - +# This config file is configures the imSim modules with default behavior. +# This file is designed to be used as a template and the values can be +# overridden. # This tells GalSim which things to import to register extra config types # Mostly we need the imsim repo. But could add others here as well. @@ -47,10 +46,9 @@ eval_variables: fexptime: { type: OpsimData, field: exptime } - -# Any input data is set here. These are read in at the start of the program and potentially -# updated for each output file. -# Also includes things that need some set up at the start of an exposure, like the atmospheric PSF. +# Any input data is set here. These are read in at the start of the program and +# potentially updated for each output file. Also includes things that need some +# set up at the start of an exposure, like the atmospheric PSF. input: telescope: From 6d8bbf0457af3c00ba9217f484bc7a4f89f7f06a Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:19:54 -0400 Subject: [PATCH 11/24] Remove white space --- config/imsim-instcat.yaml | 1 - examples/imsim-user-instcat.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/config/imsim-instcat.yaml b/config/imsim-instcat.yaml index 80b280e3..a6bfdbb9 100644 --- a/config/imsim-instcat.yaml +++ b/config/imsim-instcat.yaml @@ -17,7 +17,6 @@ input.opsim_data: # Read the visit meta data by default we use the same file as the input. # However you could specify a opsim file or a another instance catalog # instead. - file_name: '@input.instance_catalog.file_name' # Define the galaxy type and positions to use diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index ff0eb28f..585e49eb 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -14,7 +14,6 @@ input.opsim_data: # Read the visit meta data by default we use the same file as the input. # However you could specify a opsim file or a another instance catalog # instead. - file_name: '@input.instance_catalog.file_name' ################################################################ From 50337543e217c210616c31306f4b3e5ad884df14 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:22:40 -0400 Subject: [PATCH 12/24] Copied before modifying a line. --- examples/imsim-user-instcat.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index 585e49eb..05bc1b43 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -16,6 +16,10 @@ input.opsim_data: # instead. file_name: '@input.instance_catalog.file_name' +# Define the galaxy type and positions to use +gal.type: InstCatObj +stamp.world_pos.type: InstCatWorldPos + ################################################################ # Above is template material. # From 624efdac6098b68f3b716d0098ccf32cb16edac7 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:46:39 -0400 Subject: [PATCH 13/24] YAML keys are fiddly Can't redefine if not used before. But, careful not to wipe out whole section. --- config/imsim-instcat.yaml | 7 +++++-- config/imsim-skycat.yaml | 7 +++++-- examples/imsim-user-instcat.yaml | 7 +++++-- examples/imsim-user-skycat.yaml | 7 +++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/config/imsim-instcat.yaml b/config/imsim-instcat.yaml index a6bfdbb9..76ee95b1 100644 --- a/config/imsim-instcat.yaml +++ b/config/imsim-instcat.yaml @@ -20,8 +20,11 @@ input.opsim_data: file_name: '@input.instance_catalog.file_name' # Define the galaxy type and positions to use -gal.type: InstCatObj -stamp.world_pos.type: InstCatWorldPos +gal: + type: InstCatObj + +stamp.world_pos: + type: InstCatWorldPos ################################################################ # Above is template material. diff --git a/config/imsim-skycat.yaml b/config/imsim-skycat.yaml index 6bec77b6..aa6b2581 100644 --- a/config/imsim-skycat.yaml +++ b/config/imsim-skycat.yaml @@ -13,8 +13,11 @@ input.sky_catalog: input.opsim_data.file_name: default_opsim.db # Define the galaxy type and positions to use -gal.type: SkyCatObj -stamp.world_pos.type: SkyCatWorldPos +gal: + type: SkyCatObj + +stamp.world_pos: + type: SkyCatWorldPos ################################################################ # Above is template material. diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index 05bc1b43..2ea0f698 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -17,8 +17,11 @@ input.opsim_data: file_name: '@input.instance_catalog.file_name' # Define the galaxy type and positions to use -gal.type: InstCatObj -stamp.world_pos.type: InstCatWorldPos +gal: + type: InstCatObj + +stamp.world_pos: + type: InstCatWorldPos ################################################################ # Above is template material. diff --git a/examples/imsim-user-skycat.yaml b/examples/imsim-user-skycat.yaml index 6e9cea50..bba97244 100644 --- a/examples/imsim-user-skycat.yaml +++ b/examples/imsim-user-skycat.yaml @@ -13,8 +13,11 @@ input.sky_catalog: input.opsim_data.file_name: default_opsim.db # Define the galaxy type and positions to use -gal.type: SkyCatObj -stamp.world_pos.type: SkyCatWorldPos +gal: + type: SkyCatObj + +stamp.world_pos: + type: SkyCatWorldPos ################################################################ # Above is template material. From 1884db0d0a93206df92cc8e4cb88a0f28389ae27 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 16:58:45 -0400 Subject: [PATCH 14/24] Can't modify unset YAML items. --- config/imsim-skycat.yaml | 3 ++- examples/imsim-user-skycat.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/imsim-skycat.yaml b/config/imsim-skycat.yaml index aa6b2581..3a8daa21 100644 --- a/config/imsim-skycat.yaml +++ b/config/imsim-skycat.yaml @@ -10,7 +10,8 @@ input.sky_catalog: band: { type: OpsimData, field: band } mjd: { type: OpsimData, field: mjd } -input.opsim_data.file_name: default_opsim.db +input.opsim_data: + file_name: default_opsim.db # Define the galaxy type and positions to use gal: diff --git a/examples/imsim-user-skycat.yaml b/examples/imsim-user-skycat.yaml index bba97244..a00a8fe5 100644 --- a/examples/imsim-user-skycat.yaml +++ b/examples/imsim-user-skycat.yaml @@ -10,7 +10,8 @@ input.sky_catalog: band: { type: OpsimData, field: band } mjd: { type: OpsimData, field: mjd } -input.opsim_data.file_name: default_opsim.db +input.opsim_data: + file_name: default_opsim.db # Define the galaxy type and positions to use gal: From c3189023622994b137abf5bd646ced2c17a4e937 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Tue, 1 Aug 2023 17:02:34 -0400 Subject: [PATCH 15/24] Extend readme file explanation. --- examples/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index a5c14cee..aee15b08 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,10 +3,17 @@ allow you to test running imSim and show you examples of the config language. _Simple example of running and imSim config file in this directory:_ -To test that the your installed version of imSim is working you can run one the files in this directory. If you have setup imSim properly, from any working area you should be able to: +To test that the your installed version of imSim is working you can run one the +files in this directory. If you have setup imSim properly, from any working +area you should be able to: ``` galsim $IMSIM_HOME/imSim/examples/imsim-user.yaml ``` and the program should run to completion without errors. + +This directory also contains an example of running using skyCatalogs. A small +skyCatalog is distributed for testing in the distribution. To make your own +YAML files: Copy one of the templates from the "config" directory and add +your commands to the bottom of the file like in these examples. From 930c95fbe1c82e4c3d76baa68d842adae8cbf9e1 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Wed, 2 Aug 2023 11:54:27 -0400 Subject: [PATCH 16/24] Change to use specific templates. Until GalSim supports nested templates, manually copy the information into the config files. Also, rename files and register them with the template system. --- config/imsim-config-instcat.yaml | 355 +++++++++++++++++++++++++++++++ config/imsim-config-skycat.yaml | 354 ++++++++++++++++++++++++++++++ config/imsim-instcat.yaml | 41 ---- config/imsim-skycat.yaml | 40 ---- examples/imsim-user-instcat.yaml | 30 +-- examples/imsim-user-skycat.yaml | 37 ++-- imsim/templates.py | 2 + 7 files changed, 735 insertions(+), 124 deletions(-) create mode 100644 config/imsim-config-instcat.yaml create mode 100644 config/imsim-config-skycat.yaml delete mode 100644 config/imsim-instcat.yaml delete mode 100644 config/imsim-skycat.yaml diff --git a/config/imsim-config-instcat.yaml b/config/imsim-config-instcat.yaml new file mode 100644 index 00000000..cdb6717c --- /dev/null +++ b/config/imsim-config-instcat.yaml @@ -0,0 +1,355 @@ + +# This config file is configures the imSim modules with default behavior. +# This file is designed to be used as a template and the values can be +# overridden. + +# This tells GalSim which things to import to register extra config types +# Mostly we need the imsim repo. But could add others here as well. +# E.g. galsim_extra has some useful routines we might want to use. (But I don't here.) +modules: + - imsim + - astropy # I don't really understand why this is required. But if only have astropy.time + # then I get a NameError when trying to parse astropy.time.Time. + - astropy.time # Need this for an Eval. Tell GalSim to have it loaded. + +# Anything set here can be used in any Eval (usually a string that starts with $) as a variable. +# This is often convenient, so they can be set just once (probably from the command line) and +# used in multiple places. +eval_variables: + # The first letters of these variables indicates the type. + # c = CelestialCoord + # a = Angle + # s = string + # f = float + # So to use one of these in an Eval string, you would write just boresight, + # not cboresight. + cboresight: + type: RADec + ra: + type: Degrees + theta: { type: OpsimData, field: fieldRA } + dec: + type: Degrees + theta: { type: OpsimData, field: fieldDec } + + aazimuth: + type: Degrees + theta: { type: OpsimData, field: azimuth } + aaltitude: + type: Degrees + theta: { type: OpsimData, field: altitude } + arotTelPos: + type: Degrees + theta: { type: OpsimData, field: rotTelPos } + + sband: { type: OpsimData, field: band } + + fexptime: { type: OpsimData, field: exptime } + + +# Any input data is set here. These are read in at the start of the program and +# potentially updated for each output file. Also includes things that need some +# set up at the start of an exposure, like the atmospheric PSF. +input: + + telescope: + file_name: + type: FormattedStr + format : LSST_%s.yaml + items: + - { type: OpsimData, field: band } + rotTelPos: $rotTelPos + + sky_model: + # Use the rubin_sim.skybrightness model to compute the sky + # background level. + exptime: $exptime + mjd: { type: OpsimData, field: mjd } + + atm_psf: + # This enables the AtmosphericPSF type for the PSF + + # The first 4 items are required. + airmass: { type: OpsimData, field: airmass } + rawSeeing: { type: OpsimData, field: rawSeeing } + band: { type: OpsimData, field: band } + boresight: "$boresight" + + # Optional parameters: (Unless otherwise stated, these are the default values.) + t0: 0 # seconds + exptime: $exptime + kcrit: 0.2 # in units of 1/r0 + screen_size: 409.6 # Default=812.2, which takes a lot of memory, so use this for testing. + screen_scale: 0.1 # meters + doOpt: False + nproc: 1 # Default (None) means one proc per screen. + + # TODO: + #save_file: {} # This is currently set of the imsim command line, so we could keep + # doing that, but we could also base this name off of + # input.instance_catalog.file_name, or even just have `save: True` + # and let the atm_psf figure out a unique name from the input params. + + tree_rings: + # This enables TreeRingCenter and TreeRungFunc, which are stored as a dict here based + # on the detector name, so the right value can be accessed for each object. + # This file lives in the imsim repo. + file_name: "tree_ring_parameters_2018-04-26.txt" + # Can limit the detectors to read in. This is handy while debugging, since it takes + # half a minute or so to read all 189 detectors (the default behavior). + # only_dets: [R22_S11] + + checkpoint: + dir: checkpoint + file_name: + type: FormattedStr + format : checkpoint_%08d-%s.hdf + items: + - { type: OpsimData, field: observationId } + - "$det_name" + + vignetting: + file_name: LSSTCam_vignetting_data.json + +# Define how the objects are placed on the image. This refers to a single CCD. +image: + type: LSST_Image + + random_seed: { type: OpsimData, field: seed } + + nproc: 1 # Can set this to -1 to use all CPUs if you want. That probably won't be + # particularly efficient until we switch to using MakePhot rather than DrawImage. + # Right now, there is a significant overhead passing the drawn stamps back to + # the main process for all of the (many) vert faint sources, which only shoot + # a small numer of photons. + + xsize: "$xsize" + ysize: "$ysize" + + bandpass: { type: OpsimBandpass } + + wcs: + type: Batoid + + # These are required: + camera: "@output.camera" + boresight: "$boresight" + + obstime: + type: Eval + str: "astropy.time.Time(mjd_val, format='mjd', scale='tai')" + fmjd_val: { type: OpsimData, field: mjd } + + det_name: $det_name + wavelength: "$(@image.bandpass).effective_wavelength" + + # The rest can be omitted, since these are the default values, but shown here + # for reference. + temperature: 280 # Kelvin + pressure: 72.7 # kPa + H2O_pressure: 1.0 # kPa + order: 3 # Order of the SIP polynomial + + camera: "@output.camera" + + noise: + type: CCD + gain: 1 + read_noise: 0 # The read noise is applied later. Only sky noise here. + + sky_level: { type: SkyLevel } # Computed from input.sky_model. + + apply_sky_gradient: True + + use_flux_sky_areas: False # This means don't bother with the BFE from sky flux when computing + # the pixel areas for drawing the sky. + # This is the default. But set to true to get BF in sky. + # TODO: The True option is a one step flux calculation, so it's not + # yet right for doing flats. Aspirationally, I'd like it to be + # possible to make flats simply by setting this to true and + # cranking up the sky flux to something appropriate. + + sensor: + type: Silicon + strength: 1.0 + index_key: image_num # This tells GalSim that this only changes with a new image num. + treering_center: { type: TreeRingCenter, det_name: $det_name } + treering_func: { type: TreeRingFunc, det_name: $det_name } + + +# Define the PSF to use in the normal case (LSST_Silicon will override if doing a saturated star) +psf: + type: Convolve + items: + - + # Note: This type requires the "input" type atm_psf. + # It's not really an input in the normal sense, since it doesn't read + # in a file. But it needs to do some setup at the start, which all + # objects will use. So functionally, this works like an input type. + # All of the relevant configurable parameters are set there. + type: AtmosphericPSF + - + # An additional Gaussian profile to represent contributions of physical effects + # not otherwise explicitly modeled in either the Optical or Atmospheric parts. + # This value of 0.3 arcsec is appropriate when doOpt=True and sensor effects are + # being modeled. If this is not the case, then it may be appropriate to increase + # this value to account for the missing contribution of these effects. + type: Gaussian + fwhm: 0.3 + +stamp: + type: LSST_Silicon + + fft_sb_thresh: 2.e5 # When to switch to fft and a simpler PSF and skip silicon + max_flux_simple: 100 # When to switch to simple SED + airmass: { type: OpsimData, field: airmass } + rawSeeing: { type: OpsimData, field: rawSeeing } + band: { type: OpsimData, field: band } + camera: "@output.camera" + det_name: "$det_name" # This is automatically defined by the LSST_CCD output type. + + diffraction_psf: + exptime: { type: OpsimData, field: exptime } + azimuth: + type: Degrees + theta: { type: OpsimData, field: azimuth } + altitude: + type: Degrees + theta: { type: OpsimData, field: altitude } + rotTelPos: + type: Degrees + theta: { type: OpsimData, field: rotTelPos } + + photon_ops: + - + type: TimeSampler + t0: 0.0 + exptime: $exptime + - + type: PupilAnnulusSampler + R_outer: 4.18 + R_inner: 2.55 # M1 inner diameter is 2.558, but we need a bit of slack for off-axis rays + - + type: PhotonDCR + base_wavelength: $bandpass.effective_wavelength + latitude: -30.24463 degrees + HA: + type: Degrees + theta: { type: OpsimData, field: HA } + - + type: RubinDiffractionOptics + boresight: "$boresight" + camera: "@output.camera" + altitude: $altitude + azimuth: $azimuth + - + # Note: If FocusDepth is before Refraction, then the depth is the amount of focus + # change required relative to the rays coming to a focus at the surface. + # If FocusDepth is after Refraction, then the depth is the actual depth in + # the silicon where the (refracted) rays come to a focus. + type: FocusDepth + depth: + type: Eval + str: depth_dict[band] + # TODO: Figure out the depth to use for other bands. Josh found -0.6 for y. + # These numbers are in units of pixels. + ddepth_dict: {'u':0, 'g':0, 'r':0, 'i':0, 'z':0, 'y':-0.6} + sband: { type: OpsimData, field: band } + - + type: Refraction + index_ratio: 3.9 # TODO: This is what Josh used for y band. + # I assume it's wavelength dependent... + # Probably need to use the same kind of pattern as above for depth. + + +# This defines both the output files and some basic things about the overall exposure/fov. +output: + type: LSST_CCD + nproc: 1 # Change this to work on multiple CCDs at once. + nfiles: 1 # Default is all 189 CCDs. Set to 1 while testing. + + camera: LsstCam + + exptime: $exptime + + cosmic_ray_rate: 0.2 + + det_num: + type: Sequence + nitems: 189 + first: 94 # Can set first to something if you want to do a specific sensor. + + dir: fits + file_name: + type: FormattedStr + format : eimage_%08d-%1d-%s-%s-det%03d.fits + items: + - { type: OpsimData, field: observationId } + - { type: OpsimData, field: snap } + - { type: OpsimData, field: band } + - "$det_name" # A value stored in the dict by LSST_CCD + - "@output.det_num" + + readout: + # Convert from e-image to realized amp images + readout_time: 3. + dark_current: 0.02 + bias_level: 1000. + pcti: 1.e-6 + scti: 1.e-6 + + file_name: + type: FormattedStr + format : amp_%08d-%1d-%s-%s-det%03d.fits.fz + items: + - { type: OpsimData, field: observationId } + - { type: OpsimData, field: snap } + - { type: OpsimData, field: band } + - "$det_name" + - "@output.det_num" + + truth: + dir: output + file_name: + type: FormattedStr + format : centroid_%08d-%1d-%s-%s-det%03d.txt.gz + items: + - { type: OpsimData, field: observationId } + - { type: OpsimData, field: snap } + - { type: OpsimData, field: band } + - "$det_name" + - "@output.det_num" + columns: + object_id: "@object_id" + ra: "$sky_pos.ra.deg" + dec: "$sky_pos.dec.deg" + x: "$image_pos.x" + y: "$image_pos.y" + # TODO: ... more probably + + +################################################################ +# Above is template material. +# +# Make changes specific to this input method below. +# When GalSim allows nested templates we will replace this with a +# template call +################################################################ + +input.instance_catalog: + # This should be overridden below or on the command line. + file_name: default_catalog_file.txt + sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') + +input.opsim_data: + # Read the visit meta data by default we use the same file as the input. + # However you could specify a opsim file or a another instance catalog + # instead. + file_name: '@input.instance_catalog.file_name' + +# Define the galaxy type and positions to use +gal: + type: InstCatObj + +stamp.world_pos: + type: InstCatWorldPos diff --git a/config/imsim-config-skycat.yaml b/config/imsim-config-skycat.yaml new file mode 100644 index 00000000..40aef2cd --- /dev/null +++ b/config/imsim-config-skycat.yaml @@ -0,0 +1,354 @@ + +# This config file is configures the imSim modules with default behavior. +# This file is designed to be used as a template and the values can be +# overridden. + +# This tells GalSim which things to import to register extra config types +# Mostly we need the imsim repo. But could add others here as well. +# E.g. galsim_extra has some useful routines we might want to use. (But I don't here.) +modules: + - imsim + - astropy # I don't really understand why this is required. But if only have astropy.time + # then I get a NameError when trying to parse astropy.time.Time. + - astropy.time # Need this for an Eval. Tell GalSim to have it loaded. + +# Anything set here can be used in any Eval (usually a string that starts with $) as a variable. +# This is often convenient, so they can be set just once (probably from the command line) and +# used in multiple places. +eval_variables: + # The first letters of these variables indicates the type. + # c = CelestialCoord + # a = Angle + # s = string + # f = float + # So to use one of these in an Eval string, you would write just boresight, + # not cboresight. + cboresight: + type: RADec + ra: + type: Degrees + theta: { type: OpsimData, field: fieldRA } + dec: + type: Degrees + theta: { type: OpsimData, field: fieldDec } + + aazimuth: + type: Degrees + theta: { type: OpsimData, field: azimuth } + aaltitude: + type: Degrees + theta: { type: OpsimData, field: altitude } + arotTelPos: + type: Degrees + theta: { type: OpsimData, field: rotTelPos } + + sband: { type: OpsimData, field: band } + + fexptime: { type: OpsimData, field: exptime } + + +# Any input data is set here. These are read in at the start of the program and +# potentially updated for each output file. Also includes things that need some +# set up at the start of an exposure, like the atmospheric PSF. +input: + + telescope: + file_name: + type: FormattedStr + format : LSST_%s.yaml + items: + - { type: OpsimData, field: band } + rotTelPos: $rotTelPos + + sky_model: + # Use the rubin_sim.skybrightness model to compute the sky + # background level. + exptime: $exptime + mjd: { type: OpsimData, field: mjd } + + atm_psf: + # This enables the AtmosphericPSF type for the PSF + + # The first 4 items are required. + airmass: { type: OpsimData, field: airmass } + rawSeeing: { type: OpsimData, field: rawSeeing } + band: { type: OpsimData, field: band } + boresight: "$boresight" + + # Optional parameters: (Unless otherwise stated, these are the default values.) + t0: 0 # seconds + exptime: $exptime + kcrit: 0.2 # in units of 1/r0 + screen_size: 409.6 # Default=812.2, which takes a lot of memory, so use this for testing. + screen_scale: 0.1 # meters + doOpt: False + nproc: 1 # Default (None) means one proc per screen. + + # TODO: + #save_file: {} # This is currently set of the imsim command line, so we could keep + # doing that, but we could also base this name off of + # input.instance_catalog.file_name, or even just have `save: True` + # and let the atm_psf figure out a unique name from the input params. + + tree_rings: + # This enables TreeRingCenter and TreeRungFunc, which are stored as a dict here based + # on the detector name, so the right value can be accessed for each object. + # This file lives in the imsim repo. + file_name: "tree_ring_parameters_2018-04-26.txt" + # Can limit the detectors to read in. This is handy while debugging, since it takes + # half a minute or so to read all 189 detectors (the default behavior). + # only_dets: [R22_S11] + + checkpoint: + dir: checkpoint + file_name: + type: FormattedStr + format : checkpoint_%08d-%s.hdf + items: + - { type: OpsimData, field: observationId } + - "$det_name" + + vignetting: + file_name: LSSTCam_vignetting_data.json + +# Define how the objects are placed on the image. This refers to a single CCD. +image: + type: LSST_Image + + random_seed: { type: OpsimData, field: seed } + + nproc: 1 # Can set this to -1 to use all CPUs if you want. That probably won't be + # particularly efficient until we switch to using MakePhot rather than DrawImage. + # Right now, there is a significant overhead passing the drawn stamps back to + # the main process for all of the (many) vert faint sources, which only shoot + # a small numer of photons. + + xsize: "$xsize" + ysize: "$ysize" + + bandpass: { type: OpsimBandpass } + + wcs: + type: Batoid + + # These are required: + camera: "@output.camera" + boresight: "$boresight" + + obstime: + type: Eval + str: "astropy.time.Time(mjd_val, format='mjd', scale='tai')" + fmjd_val: { type: OpsimData, field: mjd } + + det_name: $det_name + wavelength: "$(@image.bandpass).effective_wavelength" + + # The rest can be omitted, since these are the default values, but shown here + # for reference. + temperature: 280 # Kelvin + pressure: 72.7 # kPa + H2O_pressure: 1.0 # kPa + order: 3 # Order of the SIP polynomial + + camera: "@output.camera" + + noise: + type: CCD + gain: 1 + read_noise: 0 # The read noise is applied later. Only sky noise here. + + sky_level: { type: SkyLevel } # Computed from input.sky_model. + + apply_sky_gradient: True + + use_flux_sky_areas: False # This means don't bother with the BFE from sky flux when computing + # the pixel areas for drawing the sky. + # This is the default. But set to true to get BF in sky. + # TODO: The True option is a one step flux calculation, so it's not + # yet right for doing flats. Aspirationally, I'd like it to be + # possible to make flats simply by setting this to true and + # cranking up the sky flux to something appropriate. + + sensor: + type: Silicon + strength: 1.0 + index_key: image_num # This tells GalSim that this only changes with a new image num. + treering_center: { type: TreeRingCenter, det_name: $det_name } + treering_func: { type: TreeRingFunc, det_name: $det_name } + + +# Define the PSF to use in the normal case (LSST_Silicon will override if doing a saturated star) +psf: + type: Convolve + items: + - + # Note: This type requires the "input" type atm_psf. + # It's not really an input in the normal sense, since it doesn't read + # in a file. But it needs to do some setup at the start, which all + # objects will use. So functionally, this works like an input type. + # All of the relevant configurable parameters are set there. + type: AtmosphericPSF + - + # An additional Gaussian profile to represent contributions of physical effects + # not otherwise explicitly modeled in either the Optical or Atmospheric parts. + # This value of 0.3 arcsec is appropriate when doOpt=True and sensor effects are + # being modeled. If this is not the case, then it may be appropriate to increase + # this value to account for the missing contribution of these effects. + type: Gaussian + fwhm: 0.3 + +stamp: + type: LSST_Silicon + + fft_sb_thresh: 2.e5 # When to switch to fft and a simpler PSF and skip silicon + max_flux_simple: 100 # When to switch to simple SED + airmass: { type: OpsimData, field: airmass } + rawSeeing: { type: OpsimData, field: rawSeeing } + band: { type: OpsimData, field: band } + camera: "@output.camera" + det_name: "$det_name" # This is automatically defined by the LSST_CCD output type. + + diffraction_psf: + exptime: { type: OpsimData, field: exptime } + azimuth: + type: Degrees + theta: { type: OpsimData, field: azimuth } + altitude: + type: Degrees + theta: { type: OpsimData, field: altitude } + rotTelPos: + type: Degrees + theta: { type: OpsimData, field: rotTelPos } + + photon_ops: + - + type: TimeSampler + t0: 0.0 + exptime: $exptime + - + type: PupilAnnulusSampler + R_outer: 4.18 + R_inner: 2.55 # M1 inner diameter is 2.558, but we need a bit of slack for off-axis rays + - + type: PhotonDCR + base_wavelength: $bandpass.effective_wavelength + latitude: -30.24463 degrees + HA: + type: Degrees + theta: { type: OpsimData, field: HA } + - + type: RubinDiffractionOptics + boresight: "$boresight" + camera: "@output.camera" + altitude: $altitude + azimuth: $azimuth + - + # Note: If FocusDepth is before Refraction, then the depth is the amount of focus + # change required relative to the rays coming to a focus at the surface. + # If FocusDepth is after Refraction, then the depth is the actual depth in + # the silicon where the (refracted) rays come to a focus. + type: FocusDepth + depth: + type: Eval + str: depth_dict[band] + # TODO: Figure out the depth to use for other bands. Josh found -0.6 for y. + # These numbers are in units of pixels. + ddepth_dict: {'u':0, 'g':0, 'r':0, 'i':0, 'z':0, 'y':-0.6} + sband: { type: OpsimData, field: band } + - + type: Refraction + index_ratio: 3.9 # TODO: This is what Josh used for y band. + # I assume it's wavelength dependent... + # Probably need to use the same kind of pattern as above for depth. + + +# This defines both the output files and some basic things about the overall exposure/fov. +output: + type: LSST_CCD + nproc: 1 # Change this to work on multiple CCDs at once. + nfiles: 1 # Default is all 189 CCDs. Set to 1 while testing. + + camera: LsstCam + + exptime: $exptime + + cosmic_ray_rate: 0.2 + + det_num: + type: Sequence + nitems: 189 + first: 94 # Can set first to something if you want to do a specific sensor. + + dir: fits + file_name: + type: FormattedStr + format : eimage_%08d-%1d-%s-%s-det%03d.fits + items: + - { type: OpsimData, field: observationId } + - { type: OpsimData, field: snap } + - { type: OpsimData, field: band } + - "$det_name" # A value stored in the dict by LSST_CCD + - "@output.det_num" + + readout: + # Convert from e-image to realized amp images + readout_time: 3. + dark_current: 0.02 + bias_level: 1000. + pcti: 1.e-6 + scti: 1.e-6 + + file_name: + type: FormattedStr + format : amp_%08d-%1d-%s-%s-det%03d.fits.fz + items: + - { type: OpsimData, field: observationId } + - { type: OpsimData, field: snap } + - { type: OpsimData, field: band } + - "$det_name" + - "@output.det_num" + + truth: + dir: output + file_name: + type: FormattedStr + format : centroid_%08d-%1d-%s-%s-det%03d.txt.gz + items: + - { type: OpsimData, field: observationId } + - { type: OpsimData, field: snap } + - { type: OpsimData, field: band } + - "$det_name" + - "@output.det_num" + columns: + object_id: "@object_id" + ra: "$sky_pos.ra.deg" + dec: "$sky_pos.dec.deg" + x: "$image_pos.x" + y: "$image_pos.y" + # TODO: ... more probably + + +################################################################ +# Above is template material. +# +# Make changes specific to this input method below. +# When GalSim allows nested templates we will replace this with a +# template call +################################################################ + +# Use skyCatalogs for obtaining the objects to render. + +input.sky_catalog: + file_name: default_sky_cat.yaml + band: { type: OpsimData, field: band } + mjd: { type: OpsimData, field: mjd } + +input.opsim_data: + file_name: default_opsim.db + +# Define the galaxy type and positions to use +gal: + type: SkyCatObj + +stamp.world_pos: + type: SkyCatWorldPos diff --git a/config/imsim-instcat.yaml b/config/imsim-instcat.yaml deleted file mode 100644 index 76ee95b1..00000000 --- a/config/imsim-instcat.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# This template file can be used for reading the instance catalog format. Make -# a copy and modify the area below the double ### line. - -# Use imSim custom modules -modules: - - imsim - -# Get most of the configuration from the imSim config-template -template: imsim-config - -input.instance_catalog: - # This should be overridden below or on the command line. - file_name: default_catalog_file.txt - sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') - -input.opsim_data: - # Read the visit meta data by default we use the same file as the input. - # However you could specify a opsim file or a another instance catalog - # instead. - file_name: '@input.instance_catalog.file_name' - -# Define the galaxy type and positions to use -gal: - type: InstCatObj - -stamp.world_pos: - type: InstCatWorldPos - -################################################################ -# Above is template material. -# -# Make changes below. Soon you will be able to include the above -# information via a single template command instead. -################################################################ -################################################################ - -# Put your own commands below here. For example -# input.instance_catalog.file_name: ./imsim_cat_197356.txt -# input.instance_catalog.sort_mag: False -# input.tree_rings.only_dets: [R22_S11] -# image.nobjects: 5 \ No newline at end of file diff --git a/config/imsim-skycat.yaml b/config/imsim-skycat.yaml deleted file mode 100644 index 3a8daa21..00000000 --- a/config/imsim-skycat.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Use baseline configuration -modules: - - imsim - -template: imsim-config - -# Use skyCatalogs for obtaining the objects to render. -input.sky_catalog: - file_name: default_sky_cat.yaml - band: { type: OpsimData, field: band } - mjd: { type: OpsimData, field: mjd } - -input.opsim_data: - file_name: default_opsim.db - -# Define the galaxy type and positions to use -gal: - type: SkyCatObj - -stamp.world_pos: - type: SkyCatWorldPos - -################################################################ -# Above is template material. -# -# Make changes below. Soon you will be able to include the above -# information via a single template command instead. -################################################################ -################################################################ - -# Put your own commands below here. For example - -# input.sky_catalog.file_name: ../tests/data/sky_cat_9683.yaml -# input.opsim_data.file_name: ../tests/data/small_opsim_9683.db -# input.opsim_data.visit: 449053 -# input.tree_rings.only_dets: [R22_S11, R22_S12] -# output.det_num.first: 94 -# output.nproc: 2 -# output.nfiles: 2 -# image.nobjects: 1000 diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index 2ea0f698..ade3a8f9 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -3,34 +3,20 @@ modules: - imsim # Get most of the configuration from the imSim config-template -template: imsim-config - -input.instance_catalog: - # This should be overridden below or on the command line. - file_name: default_catalog_file.txt - sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') - -input.opsim_data: - # Read the visit meta data by default we use the same file as the input. - # However you could specify a opsim file or a another instance catalog - # instead. - file_name: '@input.instance_catalog.file_name' - -# Define the galaxy type and positions to use -gal: - type: InstCatObj - -stamp.world_pos: - type: InstCatWorldPos +# for instance catalogs. +template: imsim-instcat-config ################################################################ # Above is template material. # -# Make changes below. Soon you will be able to include the above -# information via a single template command instead. -################################################################ +# Make your changes below. ################################################################ +# Put your own commands that override the defaults below here. For example +# input.instance_catalog.file_name: ./imsim_cat_197356.txt +# input.instance_catalog.sort_mag: False +# input.tree_rings.only_dets: [R22_S11] +# image.nobjects: 5 input.instance_catalog.file_name: $os.environ.get('IMSIM_HOME')+'/imSim/examples/example_instance_catalog.txt' diff --git a/examples/imsim-user-skycat.yaml b/examples/imsim-user-skycat.yaml index a00a8fe5..a9b22e30 100644 --- a/examples/imsim-user-skycat.yaml +++ b/examples/imsim-user-skycat.yaml @@ -1,33 +1,28 @@ -# Use baseline configuration +# Use imSim custom modules modules: - imsim -template: imsim-config - -# Use skyCatalogs for obtaining the objects to render. -input.sky_catalog: - file_name: default_sky_cat.yaml - band: { type: OpsimData, field: band } - mjd: { type: OpsimData, field: mjd } - -input.opsim_data: - file_name: default_opsim.db - -# Define the galaxy type and positions to use -gal: - type: SkyCatObj - -stamp.world_pos: - type: SkyCatWorldPos +# Get most of the configuration from the imSim config-template +# for skyCatalogs +template: imsim-skycat-config ################################################################ # Above is template material. # -# Make changes below. Soon you will be able to include the above -# information via a single template command instead. -################################################################ +# Make your changes below. ################################################################ +# Put your own commands below here. For example + +# input.sky_catalog.file_name: ../tests/data/sky_cat_9683.yaml +# input.opsim_data.file_name: ../tests/data/small_opsim_9683.db +# input.opsim_data.visit: 449053 +# input.tree_rings.only_dets: [R22_S11, R22_S12] +# output.det_num.first: 94 +# output.nproc: 2 +# output.nfiles: 2 +# image.nobjects: 1000 + # Restrict to galaxies to avoid bright stars input.sky_catalog.obj_types: [galaxy] diff --git a/imsim/templates.py b/imsim/templates.py index 6adfe9ae..270219fa 100644 --- a/imsim/templates.py +++ b/imsim/templates.py @@ -10,3 +10,5 @@ from .meta_data import config_dir galsim.config.RegisterTemplate('imsim-config', os.path.join(config_dir, 'imsim-config.yaml')) +galsim.config.RegisterTemplate('imsim-config-instcat', os.path.join(config_dir, 'imsim-config-instcat.yaml')) +galsim.config.RegisterTemplate('imsim-config-skycat', os.path.join(config_dir, 'imsim-config-skycat.yaml')) From 91e08b1e87d5ec04a9be9079667e7d6f58dbb0af Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Wed, 2 Aug 2023 12:07:59 -0400 Subject: [PATCH 17/24] File name fix. --- examples/imsim-user-instcat.yaml | 2 +- examples/imsim-user-skycat.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index ade3a8f9..b85d6a85 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -4,7 +4,7 @@ modules: # Get most of the configuration from the imSim config-template # for instance catalogs. -template: imsim-instcat-config +template: imsim-config-instcat ################################################################ # Above is template material. diff --git a/examples/imsim-user-skycat.yaml b/examples/imsim-user-skycat.yaml index a9b22e30..47a4ae36 100644 --- a/examples/imsim-user-skycat.yaml +++ b/examples/imsim-user-skycat.yaml @@ -4,7 +4,7 @@ modules: # Get most of the configuration from the imSim config-template # for skyCatalogs -template: imsim-skycat-config +template: imsim-config-skycat ################################################################ # Above is template material. From bd31fd445c16762569fc7bc732de1454561b4229 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Wed, 2 Aug 2023 14:06:36 -0400 Subject: [PATCH 18/24] Manually insert YAML elements. It seems appending YAML elements in . notation doesn't work. So, for now, manually insert the elements in the yaml sections. This is dangerous! Hopefully this will be replaced by a nested template soon. We have to be careful to not let config files get out of sync. Now nothing should be using the core imsim-config.yaml file. Also, update two tests to use the instcat config file. --- config/imsim-config-instcat.yaml | 47 +++++++++++++++++++++++--------- config/imsim-config-skycat.yaml | 40 +++++++++++++++++++-------- tests/test_header_keywords.py | 2 +- tests/test_object_positions.py | 2 +- 4 files changed, 65 insertions(+), 26 deletions(-) diff --git a/config/imsim-config-instcat.yaml b/config/imsim-config-instcat.yaml index cdb6717c..0c0e3818 100644 --- a/config/imsim-config-instcat.yaml +++ b/config/imsim-config-instcat.yaml @@ -52,6 +52,18 @@ eval_variables: # set up at the start of an exposure, like the atmospheric PSF. input: + instance_catalog: + # This should be overridden below or on the command line. + file_name: default_catalog_file.txt + sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') + + opsim_data: + # Read the visit meta data by default we use the same file as the input. + # However you could specify a opsim file or a another instance catalog + # instead. + file_name: '@input.instance_catalog.file_name' + + telescope: file_name: type: FormattedStr @@ -261,6 +273,8 @@ stamp: # I assume it's wavelength dependent... # Probably need to use the same kind of pattern as above for depth. + world_pos: + type: InstCatWorldPos # This defines both the output files and some basic things about the overall exposure/fov. output: @@ -327,6 +341,13 @@ output: y: "$image_pos.y" # TODO: ... more probably +gal: + type: InstCatObj + +# Including the material below doesn't work. I am going to manually inject the +# entries above. :( +# +# Hopefully this will be replaced soon. ################################################################ # Above is template material. @@ -336,20 +357,20 @@ output: # template call ################################################################ -input.instance_catalog: - # This should be overridden below or on the command line. - file_name: default_catalog_file.txt - sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') +# input.instance_catalog: +# # This should be overridden below or on the command line. +# file_name: default_catalog_file.txt +# sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') -input.opsim_data: - # Read the visit meta data by default we use the same file as the input. - # However you could specify a opsim file or a another instance catalog - # instead. - file_name: '@input.instance_catalog.file_name' +# input.opsim_data: +# # Read the visit meta data by default we use the same file as the input. +# # However you could specify a opsim file or a another instance catalog +# # instead. +# file_name: '@input.instance_catalog.file_name' # Define the galaxy type and positions to use -gal: - type: InstCatObj +# gal: +# type: InstCatObj -stamp.world_pos: - type: InstCatWorldPos +# stamp.world_pos: +# type: InstCatWorldPos diff --git a/config/imsim-config-skycat.yaml b/config/imsim-config-skycat.yaml index 40aef2cd..a08fde99 100644 --- a/config/imsim-config-skycat.yaml +++ b/config/imsim-config-skycat.yaml @@ -52,6 +52,14 @@ eval_variables: # set up at the start of an exposure, like the atmospheric PSF. input: + sky_catalog: + file_name: default_sky_cat.yaml + band: { type: OpsimData, field: band } + mjd: { type: OpsimData, field: mjd } + + opsim_data: + file_name: default_opsim.db + telescope: file_name: type: FormattedStr @@ -261,6 +269,9 @@ stamp: # I assume it's wavelength dependent... # Probably need to use the same kind of pattern as above for depth. + world_pos: + type: SkyCatWorldPos + # This defines both the output files and some basic things about the overall exposure/fov. output: @@ -327,6 +338,13 @@ output: y: "$image_pos.y" # TODO: ... more probably +gal: + type: SkyCatObj + +# Including the material below doesn't work. I am going to manually inject the +# entries above. :( +# +# Hopefully this will be replaced soon. ################################################################ # Above is template material. @@ -338,17 +356,17 @@ output: # Use skyCatalogs for obtaining the objects to render. -input.sky_catalog: - file_name: default_sky_cat.yaml - band: { type: OpsimData, field: band } - mjd: { type: OpsimData, field: mjd } +# input.sky_catalog: +# file_name: default_sky_cat.yaml +# band: { type: OpsimData, field: band } +# mjd: { type: OpsimData, field: mjd } -input.opsim_data: - file_name: default_opsim.db +# input.opsim_data: +# file_name: default_opsim.db -# Define the galaxy type and positions to use -gal: - type: SkyCatObj +# # Define the galaxy type and positions to use +# gal: +# type: SkyCatObj -stamp.world_pos: - type: SkyCatWorldPos +# stamp.world_pos: +# type: SkyCatWorldPos diff --git a/tests/test_header_keywords.py b/tests/test_header_keywords.py index 1502801f..62121704 100644 --- a/tests/test_header_keywords.py +++ b/tests/test_header_keywords.py @@ -12,7 +12,7 @@ def run_imsim(camera): imsim_dir = os.path.dirname(os.path.abspath(str(Path(__file__).parent))) os.environ['SIMS_SED_LIBRARY_DIR'] \ = os.path.join(imsim_dir, 'tests', 'data', 'test_sed_library') - template = os.path.join(imsim_dir, 'config', 'imsim-config.yaml') + template = os.path.join(imsim_dir, 'config', 'imsim-config-instcat.yaml') instcat_file = os.path.join(imsim_dir, 'tests', 'data', 'instcat_object_positions_test.txt') diff --git a/tests/test_object_positions.py b/tests/test_object_positions.py index 50fc9db0..5d1182e3 100644 --- a/tests/test_object_positions.py +++ b/tests/test_object_positions.py @@ -12,7 +12,7 @@ def run_imsim(camera): imsim_dir = os.path.dirname(os.path.abspath(str(Path(__file__).parent))) os.environ['SIMS_SED_LIBRARY_DIR'] \ = os.path.join(imsim_dir, 'tests', 'data', 'test_sed_library') - template = os.path.join(imsim_dir, 'config', 'imsim-config.yaml') + template = os.path.join(imsim_dir, 'config', 'imsim-config-instcat.yaml') instcat_file = os.path.join(imsim_dir, 'tests', 'data', 'instcat_object_positions_test.txt') From 26b072f54d3c1d337508b3b36f51755d4c836e96 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Fri, 4 Aug 2023 11:09:11 -0400 Subject: [PATCH 19/24] Update usage docs to use new template. --- doc/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage.rst b/doc/usage.rst index 345540ca..7fa0efc8 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -32,7 +32,7 @@ The rest of the file configures the input and output options. You should make a modules: - imsim - template: imsim-config + template: imsim-config-instcat input.instance_catalog.sed_dir: $os.environ.get('SIMS_SED_LIBRARY_DIR') From f8ab7cc43ca92149a140bf401b258d75f156683b Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Fri, 4 Aug 2023 11:49:15 -0400 Subject: [PATCH 20/24] Edit internal comments in instcat template. --- config/imsim-config-instcat.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/config/imsim-config-instcat.yaml b/config/imsim-config-instcat.yaml index 0c0e3818..e65e64b8 100644 --- a/config/imsim-config-instcat.yaml +++ b/config/imsim-config-instcat.yaml @@ -1,7 +1,14 @@ -# This config file is configures the imSim modules with default behavior. -# This file is designed to be used as a template and the values can be +# This config file configures the imSim modules with default behavior. This file +# is designed to be used as a template and some values can or should be # overridden. +# +# At least the following value(s) should generally be overridden in order to use +# this file: +# +# input.instance_catalog.filename +# + # This tells GalSim which things to import to register extra config types # Mostly we need the imsim repo. But could add others here as well. @@ -344,17 +351,10 @@ output: gal: type: InstCatObj -# Including the material below doesn't work. I am going to manually inject the -# entries above. :( -# -# Hopefully this will be replaced soon. - ################################################################ # Above is template material. # -# Make changes specific to this input method below. -# When GalSim allows nested templates we will replace this with a -# template call +# The following entires are added to the base configuration above. ################################################################ # input.instance_catalog: From ef2cd6b5916ba043d3fd49d129081500ff5c096c Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Fri, 4 Aug 2023 11:51:34 -0400 Subject: [PATCH 21/24] Edit comments in skycat template. --- config/imsim-config-skycat.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/config/imsim-config-skycat.yaml b/config/imsim-config-skycat.yaml index a08fde99..defb8f81 100644 --- a/config/imsim-config-skycat.yaml +++ b/config/imsim-config-skycat.yaml @@ -1,7 +1,15 @@ -# This config file is configures the imSim modules with default behavior. -# This file is designed to be used as a template and the values can be +# This config file configures the imSim modules with default behavior. This file +# is designed to be used as a template and some values can or should be # overridden. +# +# At least the following value(s) should generally be overridden in order to use +# this file: +# +# input.sky_catalog.file_name +# input.opsim_data.file_name +# + # This tells GalSim which things to import to register extra config types # Mostly we need the imsim repo. But could add others here as well. @@ -341,17 +349,10 @@ output: gal: type: SkyCatObj -# Including the material below doesn't work. I am going to manually inject the -# entries above. :( -# -# Hopefully this will be replaced soon. - ################################################################ # Above is template material. # -# Make changes specific to this input method below. -# When GalSim allows nested templates we will replace this with a -# template call +# The following entires are added to the base configuration above. ################################################################ # Use skyCatalogs for obtaining the objects to render. From 9c25890e9d443008ef6301e0d20ab42b2bc55c6d Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Fri, 4 Aug 2023 11:59:31 -0400 Subject: [PATCH 22/24] Edit example file comments --- examples/imsim-user-instcat.yaml | 2 -- examples/imsim-user-skycat.yaml | 13 ------------- 2 files changed, 15 deletions(-) diff --git a/examples/imsim-user-instcat.yaml b/examples/imsim-user-instcat.yaml index b85d6a85..1be69728 100644 --- a/examples/imsim-user-instcat.yaml +++ b/examples/imsim-user-instcat.yaml @@ -7,8 +7,6 @@ modules: template: imsim-config-instcat ################################################################ -# Above is template material. -# # Make your changes below. ################################################################ diff --git a/examples/imsim-user-skycat.yaml b/examples/imsim-user-skycat.yaml index 47a4ae36..31646ac5 100644 --- a/examples/imsim-user-skycat.yaml +++ b/examples/imsim-user-skycat.yaml @@ -7,22 +7,9 @@ modules: template: imsim-config-skycat ################################################################ -# Above is template material. -# # Make your changes below. ################################################################ -# Put your own commands below here. For example - -# input.sky_catalog.file_name: ../tests/data/sky_cat_9683.yaml -# input.opsim_data.file_name: ../tests/data/small_opsim_9683.db -# input.opsim_data.visit: 449053 -# input.tree_rings.only_dets: [R22_S11, R22_S12] -# output.det_num.first: 94 -# output.nproc: 2 -# output.nfiles: 2 -# image.nobjects: 1000 - # Restrict to galaxies to avoid bright stars input.sky_catalog.obj_types: [galaxy] From 272869dc59a76815cc39389145beef167436b3a3 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Fri, 4 Aug 2023 12:03:30 -0400 Subject: [PATCH 23/24] Move flat-*.yaml files to examples directory. Also update CI. --- .github/workflows/ci.yml | 3 +-- {config => examples}/.gitignore | 0 {config => examples}/flat.yaml | 0 {config => examples}/flat_with_sed.yaml | 0 4 files changed, 1 insertion(+), 2 deletions(-) rename {config => examples}/.gitignore (100%) rename {config => examples}/flat.yaml (100%) rename {config => examples}/flat_with_sed.yaml (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee5f7d3d..d484ba23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,9 +91,8 @@ jobs: export RUBIN_SIM_DATA_DIR=`pwd`/rubin_sim_data export SIMS_SED_LIBRARY_DIR=`pwd`/rubin_sim_data/sims_sed_library # Just check that they haven't gotten stale and produce errors. - cd config + cd examples galsim flat.yaml image.counts_per_pixel=500 galsim flat_with_sed.yaml image.counts_per_pixel=5 - cd ../examples galsim imsim-user-instcat.yaml image.nobjects=10 galsim imsim-user-skycat.yaml image.nobjects=10 diff --git a/config/.gitignore b/examples/.gitignore similarity index 100% rename from config/.gitignore rename to examples/.gitignore diff --git a/config/flat.yaml b/examples/flat.yaml similarity index 100% rename from config/flat.yaml rename to examples/flat.yaml diff --git a/config/flat_with_sed.yaml b/examples/flat_with_sed.yaml similarity index 100% rename from config/flat_with_sed.yaml rename to examples/flat_with_sed.yaml From 32ca0184fe604ba0a8250bb33d0f5b42c39746d5 Mon Sep 17 00:00:00 2001 From: Chris Walter Date: Fri, 4 Aug 2023 12:04:53 -0400 Subject: [PATCH 24/24] Update README file. --- examples/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/README.md b/examples/README.md index aee15b08..e7267fe3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,6 +14,4 @@ galsim $IMSIM_HOME/imSim/examples/imsim-user.yaml and the program should run to completion without errors. This directory also contains an example of running using skyCatalogs. A small -skyCatalog is distributed for testing in the distribution. To make your own -YAML files: Copy one of the templates from the "config" directory and add -your commands to the bottom of the file like in these examples. +skyCatalog is distributed for testing in the distribution. \ No newline at end of file