Merge branch 'main' into fix_stack_bug

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,20 +1,30 @@
 ---
 name: Bug Report
-about: Describe your issue here, including a minimum working example and all file downloads without authentication. We have limited time, so only easily reproduced issues can be fixed quickly.
+about: Describe your issue here, including a minimum working example and all file downloads without authentication.
+We cant fix your problem unless you help us by filling out the code blocks below.
 
 title: ''
 labels: ''
 assignees: ''
 
 ---
 
-__I have added a _complete_ minimum working example below including downloads of the files (that works without authentication)  and understand that without it my bug is very hard to be fix__
+Your problem. You don't have to write anything here for a bug, the code and error are what is important.
 
 ```julia
-mwe_here
+All the julia code needed to make the error goes here including download or generation of all data used
+
+This is the most important thing in the issue, we cant fix bugs without it.
+```
+
+```julia
+The full complete error from start to finish goes here
+
+This is the second most imortant thing in the issue, we can't be sure we fixed the same problem unless we can see this.
 ```
 
-__I have pasted a _complete_ error from function call down to the end__
 ```julia
-error here
+Paste the results of `versioninfo()` here.
+
+Paste the results of `] status Rasters` here.
 ```

.github/workflows/Documenter.yml

@@ -5,16 +5,32 @@ on:
     tags: [v*]
   pull_request:
     branches: [main]
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 jobs:
-  Documenter:
-    name: Documentation
+  build:
+    permissions:
+      contents: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/julia-buildpkg@latest
-      - uses: julia-actions/julia-docdeploy@latest
+      - uses: actions/checkout@v3
+      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/cache@v1
+        with:
+          cache-registries: "true"
+      - name: Install documentation dependencies
+        run: julia --project=docs -e 'using Pkg; pkg"dev ."; Pkg.instantiate()'
+      - name: Build and deploy
         env:
-          RASTERDATASOURCES_PATH: ".."
-          GKSwstype: "100"
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+          RASTERDATASOURCES_PATH: "."
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key
+          GKSwstype: "100" # https://discourse.julialang.org/t/generation-of-documentation-fails-qt-qpa-xcb-could-not-connect-to-display/60988
+          JULIA_DEBUG: "Documenter"
+          DATADEPS_ALWAYS_ACCEPT: true
+        run: |
+          julia --project=docs/ --color=yes docs/genfiles.jl
+          julia --project=docs/ --color=yes docs/make.jl

.github/workflows/ci.yml

@@ -13,10 +13,11 @@ jobs:
       fail-fast: true
       matrix:
         version:
+          - '1.8'
           - '1'
         os:
           - ubuntu-latest
-          # - macOS-latest
+          - macOS-latest
           # - windows-latest
         arch:
           - x64

.gitignore

@@ -1,7 +1,22 @@
 *.jl.*.cov
 *.jl.cov
 *.jl.mem
-docs/build
 test/data
 Manifest.toml
+
+*.shp
+*.nc
+*.tif
+
+tmp
+generated
+
 .vscode
+.DS_Store
+
+docs/docs
+docs/site
+docs/build
+docs/var
+deps/build.jl
+Manifest.toml

Project.toml

@@ -1,14 +1,12 @@
 name = "Rasters"
 uuid = "a3a2b9e3-a471-40c9-b274-f788e487c689"
 authors = ["Rafael Schouten <[email protected]>"]
-version = "0.5.3"
+version = "0.8.3"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
 ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
 ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
-CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
 DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
@@ -17,14 +15,11 @@ FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
 Flatten = "4c728ea3-d9ee-5c9a-9642-b6f7d7dc04fa"
 GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f"
 GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f"
-HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b"
 Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
-NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
-RasterDataSources = "3cb90ccd-e1b6-4867-9617-4276c8b2ca36"
 RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -36,7 +31,7 @@ ArchGDAL = "0.9, 0.10"
 ColorTypes = "0.10, 0.11"
 ConstructionBase = "1"
 CoordinateTransformations = "0.6.2"
-DimensionalData = "0.24"
+DimensionalData = "0.24.13"
 DiskArrays = "^0.3.3"
 Extents = "0.1"
 FillArrays = "0.12, 0.13, 1"
@@ -54,24 +49,40 @@ RecipesBase = "0.7, 0.8, 1.0"
 Reexport = "0.2, 1.0"
 Requires = "0.5, 1"
 Setfield = "0.6, 0.7, 0.8, 1"
-julia = "1"
+julia = "1.8"
 
 [extensions]
-RastersMakie = "Makie"
+RastersArchGDALExt = "ArchGDAL"
+RastersCoordinateTransformationsExt = "CoordinateTransformations"
+RastersHDF5Ext = "HDF5"
+RastersMakieExt = "Makie"
+RastersNCDatasetsExt = "NCDatasets"
+RastersRasterDataSourcesExt = "RasterDataSources"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
 CFTime = "179af706-886a-5703-950a-314cd64e0468"
+CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
+NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+RasterDataSources = "3cb90ccd-e1b6-4867-9617-4276c8b2ca36"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "CFTime", "DataFrames", "GeometryBasics", "Plots", "SafeTestsets", "Shapefile", "Statistics", "Test"]
+test = ["Aqua", "ArchGDAL", "CFTime", "CoordinateTransformations", "DataFrames", "GeometryBasics", "HDF5", "NCDatasets", "Plots", "RasterDataSources", "SafeTestsets", "Shapefile", "Statistics", "Test"]
 
 [weakdeps]
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
+CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
+NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
+RasterDataSources = "3cb90ccd-e1b6-4867-9617-4276c8b2ca36"

README.md

@@ -1,22 +1,52 @@
 # Rasters
 
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/rafaqz/Rasters.jl/blob/main/LICENSE)
 [![](https://img.shields.io/badge/docs-stable-blue.svg)](https://rafaqz.github.io/Rasters.jl/stable)
 [![](https://img.shields.io/badge/docs-dev-blue.svg)](https://rafaqz.github.io/Rasters.jl/dev)
 [![CI](https://github.com/rafaqz/Rasters.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/rafaqz/Rasters.jl/actions/workflows/ci.yml)
 [![Codecov](https://codecov.io/gh/rafaqz/Rasters.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/rafaqz/Rasters.jl)
 [![Aqua.jl Quality Assurance](https://img.shields.io/badge/Aquajl-%F0%9F%8C%A2-aqua.svg)](https://github.com/JuliaTesting/Aqua.jl)
+[![Downloads](https://shields.io/endpoint?url=https://pkgs.genieframework.com/api/v1/badge/Rasters&label=Downloads)](https://pkgs.genieframework.com?packages=Rasters)
+
+<img src="https://rafaqz.github.io/Rasters.jl/dev/scripts/generated/basics/aus_trim.png" align="right" width="30%"></img>
 
 [Rasters.jl](https://rafaqz.github.io/Rasters.jl/dev) defines common types and methods for reading, writing and
 manipulating rasterized spatial data. 
 
 These currently include raster arrays like GeoTIFF and NetCDF, R grd files, 
 multi-layered stacks, and multi-file series of arrays and stacks. 
 
-![EarthEnv HabitatHeterogeneity layers trimmed to Australia](https://rafaqz.github.io/Rasters.jl/stable/trim_example_after.png)
 
-_A RasterStack of EarthEnv HabitatHeterogeneity layers, trimmed to Australia and plotted with Plots.jl_
+## :warning: Packages extensions and Rasters 0.8 and onwards 
+
+On Julia 1.9 we can put additional packages in extensions, so the code only loads when
+you load a specific package. Rasters.jl was always intended to work like this,
+and its finally possible. This reduced package `using` time from many seconds to well under a second.
+
+But, it means you have to manually load packages you need for each backend or additional
+functionality.
+
+For example, to use the GDAL backend, and download files, you now need to do:
+
+```julia
+using Rasters, ArchGDAL, RasterDataSources
+```
+
+where previously it was just `using Rasters`.
+
+Sources and packages needed:
+- `:gdal`: `using ArchGDAL`
+- `:netcdf`: `using NCDatasets`
+- `:grd`: built-in.
+- `:smap`: `using HDF5`
+- `:grib`: not yet finished.
+
+Other functionality in extensions:
+- Raster data downloads, like `Worldclim{Climate}`: `using RasterDataSources`
+- Makie plots: `using Makie`
+- Coordinate transformations for gdal rasters: `using CoordinateTransformations`
 
-## Quick start
+# Quick start
 Install the package by typing:
 
 ```julia

docs/Project.toml

@@ -1,13 +1,19 @@
 [deps]
-CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
 CFTime = "179af706-886a-5703-950a-314cd64e0468"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433"
 GBIF2 = "dedd4f52-e074-43bf-924d-d6bce14ad628"
 GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f"
+HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
 Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
+NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 RasterDataSources = "3cb90ccd-e1b6-4867-9617-4276c8b2ca36"
+Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
 Shapefile = "8e980c4a-a4fe-5da2-b3a7-4b4b0353a2f4"

docs/_overrides/partials/source.html

@@ -0,0 +1,30 @@
+{% import "partials/language.html" as lang with context %}
+<a href="{{ config.repo_url }}" title="{{ lang.t('source.link.title') }}" class="md-source" data-md-component="source">
+  <div class="md-source__icon md-icon">
+    {% set icon = config.theme.icon.repo or "fontawesome/brands/git-alt" %}
+    {% include ".icons/" ~ icon ~ ".svg" %}
+  </div>
+  <div class="md-source__repository">
+    {{ config.repo_name }}
+  </div>
+</a>
+{% if config.theme.twitter_url %}
+<a href="{{ config.theme.twitter_url }}" title="Go to Twitter" class="md-source">
+  <div class="md-source__icon md-icon">
+    {% include ".icons/fontawesome/brands/twitter.svg" %}
+  </div>
+  <div class="md-source__repository">
+    {{ config.theme.twitter_name }}
+  </div>
+</a>
+{% endif %}
+{% if config.theme.sponsor_url %}
+<a href="{{ config.theme.sponsor_url }}" title="Go to Sponsor" class="md-source">
+  <div class="md-source__icon md-icon">
+    {% include ".icons/fontawesome/regular/heart.svg" %}
+  </div>
+  <div class="md-source__repository">
+    {{ config.theme.sponsor_name }}
+  </div>
+</a>
+{% endif %}

docs/genfiles.jl

@@ -0,0 +1,28 @@
+using Documenter, DocumenterMarkdown
+using Literate
+
+get_example_path(p) = joinpath(@__DIR__, ".", "scripts", p)
+OUTPUT = joinpath(@__DIR__, "src", "scripts", "generated")
+
+folders = readdir(joinpath(@__DIR__, ".", "scripts"))
+setdiff!(folders, [".DS_Store"])
+
+function getfiles()
+    srcsfiles = []
+    for f in folders
+        names = readdir(joinpath(@__DIR__, ".", "scripts", f))
+        setdiff!(names, [".DS_Store"])
+        fpaths  = "$(f)/" .* names
+        srcsfiles = vcat(srcsfiles, fpaths...)
+    end
+    return srcsfiles
+end
+
+srcsfiles = getfiles()
+
+for (d, paths) in (("tutorial", srcsfiles),)
+    for p in paths
+    Literate.markdown(get_example_path(p), joinpath(OUTPUT, dirname(p));
+            documenter=true)
+    end
+end

docs/make.jl

@@ -1,6 +1,7 @@
-using Documenter, Rasters, Plots, Logging, Statistics, Dates
+using Documenter, Rasters, Plots, Logging, Statistics, Dates, 
+    RasterDataSources, ArchGDAL, NCDatasets, HDF5, CoordinateTransformations
 import Makie, CairoMakie
-
+using DocumenterMarkdown
 using Rasters.LookupArrays, Rasters.Dimensions
 
 ENV["GKSwstype"] = "100"
@@ -20,16 +21,38 @@ flush_info_and_warnings()
 Logging.disable_logging(Logging.Warn)
 
 # Make the docs, without running the tests again
+# We need to explicitly add all the extensions here
 makedocs(
-    modules = [Rasters],
+    modules = [
+        Rasters,
+        Base.get_extension(Rasters, :RastersArchGDALExt),
+        Base.get_extension(Rasters, :RastersCoordinateTransformationsExt),
+        Base.get_extension(Rasters, :RastersHDF5Ext),
+        Base.get_extension(Rasters, :RastersMakieExt),
+        Base.get_extension(Rasters, :RastersNCDatasetsExt),
+        Base.get_extension(Rasters, :RastersRasterDataSourcesExt),
+    ],
     sitename = "Rasters.jl",
-    strict = true,
-    clean = false,
+    authors="Rafael Schouten et al.",
+    clean=true,
+    doctest=true,
+    strict=[
+        :doctest,
+        :linkcheck,
+        :parse_error,
+        :example_block,
+        # Other available options are
+        # :autodocs_block, :cross_references, :docs_block, :eval_block, :example_block,
+        # :footnote, :meta_block, :missing_docs, :setup_block
+    ], checkdocs=:all, format=Markdown(), draft=false,
+    build=joinpath(@__DIR__, "docs")
 )
 
 # Enable logging to console again
 Logging.disable_logging(Logging.BelowMinLevel)
 
-deploydocs(
-    repo = "github.com/rafaqz/Rasters.jl.git",
-)
+deploydocs(; repo="github.com/rafaqz/Rasters.jl.git", push_preview=true,
+    deps=Deps.pip("mkdocs", "pygments", "python-markdown-math", "mkdocs-material",
+        "pymdown-extensions", "mkdocstrings", "mknotebooks",
+        "pytkdocs_tweaks", "mkdocs_include_exclude_files", "jinja2", "mkdocs-video"),
+    make=() -> run(`mkdocs build`), target="site", devbranch="main")