Added demicomplete pakckage structure (docs, test..) and some potenti…

…aly useful code

.github/workflows/CompatHelper.yml

@@ -0,0 +1,17 @@
+name: CompatHelper
+
+on:
+  schedule:
+    - cron: '00 00 * * *'
+
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+    tags: '*'
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.6' # Replace this with the minimum Julia version that your package supports. E.g. if your package requires Julia 1.5 or higher, change this to '1.5'.
+          - '1' # Leave this line unchanged. '1' will automatically expand to the latest stable 1.x release of Julia.
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v3
+        with:
+          file: lcov.info
+  docs:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1'
+      - run: |
+          julia --project=docs -e '
+            using Pkg
+            Pkg.develop(PackageSpec(path=pwd()))
+            Pkg.instantiate()'
+      - run: |
+          julia --project=docs -e '
+            using Documenter: doctest
+            using MrFOR_resources_init_fr     # change MyAwesomePackage to the name of your package
+            doctest(MrFor_resources_init_fr)' # change MyAwesomePackage to the name of your package
+      - run: julia --project=docs docs/make.jl
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v1

Project.toml

@@ -2,3 +2,6 @@ name = "MrFOR_resources_init_fr"
 uuid = "78598d3f-d65b-4630-9136-a3bfe5c9de2f"
 authors = ["Antonello Lobianco <[email protected]>"]
 version = "0.1.0"
+
+[deps]
+MrFOR_resources = "731fd045-8cb1-4036-8c04-a2a477f3265a"

README.md

@@ -1,2 +1,11 @@
 # MrFOR_resources_init_fr
 Initialisation package of the resource module of MrFOR specific for France
+
+For timber Volumes, we'll be using the National Inventory row data.
+
+[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://ecoformod.github.io/MrFOR_resources_init_fr.jl/stable)
+[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://ecoformod.github.io/MrFOR_resources_init_fr.jl/dev)
+[![Build status (Github Actions)](https://github.com/ecoformod/Resourceload_france.jl/workflows/CI/badge.svg)](https://github.com/ecoformod/MrFOR_resources_init_fr.jl/actions)
+[![codecov.io](http://codecov.io/github/ecoformod/MrFOR_resources_init_fr.jl/coverage.svg?branch=main)](http://codecov.io/github/ecoformod/MrFOR_resources_init_fr.jl?branch=main)
+
+

docs/.gitignore

@@ -0,0 +1,2 @@
+build/
+site/

docs/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"

docs/make.jl

@@ -0,0 +1,18 @@
+using Documenter
+using MrFOR_resources_init_fr
+
+push!(LOAD_PATH,"../src/")
+makedocs(sitename="MrFOR_resources_init_fr.jl Documentation",
+         pages = [
+            "Index" => "index.md",
+            "An other page" => "anotherPage.md",
+         ],
+         format = Documenter.HTML(prettyurls = false)
+)
+# Documenter can also automatically deploy documentation to gh-pages.
+# See "Hosting Documentation" and deploydocs() in the Documenter manual
+# for more information.
+deploydocs(
+    repo = "github.com/sylvaticus/MrFOR_resources_init_fr.jl.git",
+    devbranch = "main"
+)

docs/src/anotherPage.md

@@ -0,0 +1,48 @@
+# The MrFOR_resources_init_fr Module
+
+
+```@docs
+MrFOR_resources_init_fr
+```
+
+## Module Index
+
+```@index
+Modules = [MrFOR_resources_init_fr]
+Order   = [:constant, :type, :function, :macro]
+```
+## Detailed API
+
+```@autodocs
+Modules = [MrFOR_resources_init_fr]
+Order   = [:constant, :type, :function, :macro]
+```
+
+# Some manual code that is executed during doc compilation
+
+```@setup abc
+using DataFrames
+println("This is printed during doc compilation")
+@info
+a = [1,2]
+b = a .+ 1
+```
+
+```@example abc
+b # hide
+```
+
+```@example abc
+DataFrame(A=[1,2,3],B=[10,20,30]) # hide
+```
+
+
+Test 
+
+```@eval
+using DataFrames, Latexify
+df = DataFrame(a=[1,2,3],b=[10,20,30])
+nothing
+mdtable(df,latex=false)
+```
+

docs/src/index.md

@@ -0,0 +1,3 @@
+# MrFOR_resources_init_fr.jl
+
+Documentation for [MrFOR_resources_init_fr.jl](https://github.com/sylvaticus/MrFOR_resources_init_fr.jl/)

src/MrFOR_resources_init_fr.jl

@@ -1,5 +1,32 @@
+
+"""
+    MrFOR_resources_init_fr
+
+Here you are, you found an awesome package :-)
+
+"""
 module MrFOR_resources_init_fr
 
-greet() = print("Hello World!")
+export data_path
+
+using Statistics, Random, Downloads # Standard Library (shipped with Julia)
+using FTPClient
+using DataFrames, CSV, Plots, DataDeps, ZipFile, DataStructures, JLD2, Pipe
+using GeoFormatTypes, ArchGDAL, Rasters, RasterDataSources
+using BetaML
+#import DecisionTree
+#plotlyjs()
+#gr() 
+#pythonplot()
+Random.seed!(123) # fix random seed
+
+(X,Y) = 1,2 # Workaround for Rasters.jl bug https://github.com/rafaqz/DimensionalData.jl/issues/493
+# Note that the first dimension is X (the cols!) and second is Y (the row!), i.e. opposite of matrix ordering!
+
+data_path   = joinpath(@__DIR__,"data")
+ENV["RASTERDATASOURCES_PATH"] = data_path
+include("getdata.jl")
+
+
 
 end # module MrFOR_resources_init_fr

src/crunchdata.jl

@@ -0,0 +1,137 @@
+
+
+function impute_X(data_path,X;force=false,verbosity=1)
+    verbosity >= 1 && println("Iputing missing values..")
+    dest = joinpath(data_path,"temp","full_X.jld")
+
+    if ispath(dest) && !force 
+        return JLD2.load(dest, "full_X")
+    end
+
+    if !isdir(joinpath(data_path,"temp"))
+        mkdir(joinpath(data_path,"temp"))
+    end
+
+    #mod = RFImputer(n_trees=30,max_depth=10,recursive_passages=2)
+    mod = UniversalImputer(estimator=DecisionTree.RandomForestRegressor(max_depth=10,n_trees=10),
+                       fit_function = DecisionTree.fit!, predict_function=DecisionTree.predict,
+                       recursive_passages=3)
+    X_full_matrix = fit!(mod,Matrix(X))
+    X_full = DataFrame(X_full_matrix,names(X))
+
+    JLD2.jldopen(dest, "w") do f
+        f["full_X"] = X_full
+    end
+
+    return X_full
+end
+
+
+function get_nn_trained_model(xtrain,ytrain,xval,yval;force=false,model_file="models.jld",maxepochs=5,scmodel=Scaler(),verbosity=STD)
+    #model_file="models.jld"
+    #force = false
+    #xtrain = xtrain_s
+    #sditrain = sditrain_s
+    #stemntrain = stemntrain_s
+    #epochs=[5,5,5]
+    #verbosity = STD
+
+    if !force 
+        models = model_load(model_file)
+        haskey(models,"bml_nnmodel") && return models["bml_nnmodel"]
+    end
+
+    # Prestep: scaling
+    xtrain_s     = predict(scmodel,xtrain)
+    xval_s       = predict(scmodel,xval)
+
+    verbosity >= STD && @info "Training vol model.."
+    #=
+    # Define the Artificial Neural Network model
+    l1_age  = ReplicatorLayer(8)
+    l1_sp   = ReplicatorLayer(8)
+    l1_soil = DenseLayer(44,60, f=relu)
+    l1_oth  = DenseLayer(4,6, f=relu)
+    l1      = GroupedLayer([l1_age,l1_sp,l1_soil,l1_oth])
+
+    l2_age  = DenseLayer(8,3, f=relu)
+    l2_sp   = DenseLayer(8,3, f=relu)
+    l2_soil = DenseLayer(60,5, f=relu)
+    l2_oth  = ReplicatorLayer(6)
+    l2      = GroupedLayer([l2_age,l2_sp,l2_soil,l2_oth])
+
+    l3_age  = ReplicatorLayer(3)
+    l3_sp   = ReplicatorLayer(3)
+    l3_soiloth = DenseLayer(11,11)
+    l3      =  GroupedLayer([l3_age,l3_sp,l3_soiloth])
+
+    l4 = DenseLayer(17,17, f=relu)
+    l5 = DenseLayer(17,1, f=relu)
+    nnm = NeuralNetworkEstimator(layers=[l1,l2,l3,l4,l5], batch_size=16, epochs=1, verbosity=verbosity) 
+    =#
+    nnm = NeuralNetworkEstimator( batch_size=16, epochs=1, verbosity=verbosity)
+
+    rmes_train = Float64[]
+    rmes_test  = Float64[] 
+    for e in 1:maxepochs
+        verbosity >= STD && @info "Epoch $e ..."
+        # Train the model (using the ADAM optimizer by default)
+        fit!(nnm,xtrain_s,ytrain) # Fit the model to the (scaled) data
+        ŷtrain         = predict(nnm,xtrain_s) 
+        ŷval           = predict(nnm,xval_s ) 
+        rme_train      = relative_mean_error(ytrain,ŷtrain)  # 0.1517 # 0.1384 # 0.165
+        rme_test       = relative_mean_error(yval,ŷval) 
+        push!(rmes_train,rme_train)
+        push!(rmes_test,rme_test)
+    end
+    display(plot([rmes_train[2:end] rmes_test[2:end]],title="Rel mean error per epoch", labels=["train rme" "test rme"]))
+    display(plot(info(nnm)["loss_per_epoch"][2:end],title="Loss per epoch", label=nothing))
+
+    model_save(model_file,true;bml_nnmodel=nnm)
+    return nnm
+
+end
+
+
+function get_rf_trained_model(xtrain,ytrain;force=false,model_file="models.jld",verbosity=STD)
+    #DecisionTree.fit!(rfmodel,fit!(Scaler(),xtrain),ytrain)
+    #ŷtrain     = DecisionTree.predict(rfmodel,fit!(Scaler(),xtrain)) 
+    #ŷtest      = DecisionTree.predict(rfmodel,fit!(Scaler(),xtest)) 
+
+    models = model_load(model_file)
+    (haskey(models,"bml_rfmodel") && !force ) && return models["bml_rfmodel"]
+    # Using Random Forest model
+    rfmodel = RandomForestEstimator(max_depth=20,verbosity=verbosity)
+    # rfmodel = DecisionTree.RandomForestRegressor()
+    fit!(rfmodel,xtrain,ytrain)
+    model_save("models.jld",false;bml_rfmodel=rfmodel)
+    return rfmodel
+
+end
+
+
+function get_estvol(x_s,mod,nAgeClasses,nSpgrClasses;force=false,data_file="estvol.csv")
+    if ispath(data_file) && !force 
+        return CSV.read(data_file,DataFrame)
+    end
+    estvol = DataFrame(r=Int64[],c=Int64[],ecoreg=Int64[],spgr=Int64[],agegr=Int64[],estvol=Float64[])
+    x_mod = copy(x_s)
+    for px in 1:size(x_mod,1), is in 1:nSpgrClasses, ia in 1:nAgeClasses, #size(X_full,1)
+        r = X_full.R[px]
+        c = X_full.C[px]
+        ecoreg_px = ecoregion_full[px]
+        spgr = is
+        agegr = ia
+        # modifying the record that we use as feature to predict the volumes
+        x_mod[px,1:nAgeClasses] .= 0.0
+        x_mod[px,1+nAgeClasses:nAgeClasses+nSpgrClasses] .= 0.0
+        x_mod[px,ia] = 1.0
+        x_mod[px,is+nAgeClasses] = 1.0
+
+        evol = predict(mod,x_mod[px,:]')[1]
+        #evol= 1.0
+        push!(estvol,[r,c,ecoreg_px,spgr,agegr,evol])
+    end
+    CSV.write("estvol.csv",estvol)
+    return estvol
+end